diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 3a596a9552d70..b5f5a752e0a46 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -1,5 +1,5 @@ name: Integration tests -on: +on: push: branches: - 'master' @@ -23,9 +23,32 @@ permissions: contents: read jobs: + changes: + runs-on: ubuntu-latest + outputs: + backend: ${{ steps.filter.outputs.backend_any_changed }} + frontend: ${{ steps.filter.outputs.frontend_any_changed }} + steps: + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + - uses: tj-actions/changed-files@90a06d6ba9543371ab4df8eeca0be07ca6054959 # v42.0.2 + id: filter + with: + # Any file which is not under docs/, ui/ or is not a markdown file is counted as a backend file + files_yaml: | + backend: + - '!ui/**' + - '!**.md' + - '!**/*.md' + - '!docs/**' + frontend: + - 'ui/**' + - Dockerfile check-go: name: Ensure Go modules synchronicity + if: ${{ needs.changes.outputs.backend == 'true' }} runs-on: ubuntu-22.04 + needs: + - changes steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 @@ -36,14 +59,17 @@ jobs: - name: Download all Go modules run: | go mod download - - name: Check for tidyness of go.mod and go.sum + - name: Check for tidiness of go.mod and go.sum run: | go mod tidy git diff --exit-code -- . build-go: name: Build & cache Go code + if: ${{ needs.changes.outputs.backend == 'true' }} runs-on: ubuntu-22.04 + needs: + - changes steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 @@ -67,7 +93,10 @@ jobs: contents: read # for actions/checkout to fetch code pull-requests: read # for golangci/golangci-lint-action to fetch pull requests name: Lint Go code + if: ${{ needs.changes.outputs.backend == 'true' }} runs-on: ubuntu-22.04 + needs: + - changes steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 @@ -83,12 +112,14 @@ jobs: test-go: name: Run unit tests for Go packages + if: ${{ needs.changes.outputs.backend == 'true' }} runs-on: ubuntu-22.04 needs: - build-go + - changes env: GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }} + GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }} steps: - name: Create checkout directory run: mkdir -p ~/go/src/github.com/argoproj @@ -150,12 +181,14 @@ jobs: test-go-race: name: Run unit tests with -race for Go packages + if: ${{ needs.changes.outputs.backend == 'true' }} runs-on: ubuntu-22.04 needs: - build-go + - changes env: GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }} + GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }} steps: - name: Create checkout directory run: mkdir -p ~/go/src/github.com/argoproj @@ -212,7 +245,10 @@ jobs: codegen: name: Check changes to generated code + if: ${{ needs.changes.outputs.backend == 'true' }} runs-on: ubuntu-22.04 + needs: + - changes steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 @@ -260,14 +296,17 @@ jobs: build-ui: name: Build, test & lint UI code + if: ${{ needs.changes.outputs.frontend == 'true' }} runs-on: ubuntu-22.04 + needs: + - changes steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: Setup NodeJS uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1 with: - node-version: '20.7.0' + node-version: '21.6.1' - name: Restore node dependency cache id: cache-dependencies uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 @@ -292,10 +331,12 @@ jobs: analyze: name: Process & analyze test artifacts + if: ${{ needs.changes.outputs.backend == 'true' || needs.changes.outputs.frontend == 'true' }} runs-on: ubuntu-22.04 needs: - test-go - build-ui + - changes env: sonar_secret: ${{ secrets.SONAR_TOKEN }} steps: @@ -315,7 +356,7 @@ jobs: - name: Create test-results directory run: | mkdir -p test-results - - name: Get code coverage artifiact + - name: Get code coverage artifact uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: code-coverage @@ -336,35 +377,37 @@ jobs: SCANNER_PATH: /tmp/cache/scanner OS: linux run: | - # We do not use the provided action, because it does contain an old - # version of the scanner, and also takes time to build. - set -e - mkdir -p ${SCANNER_PATH} - export SONAR_USER_HOME=${SCANNER_PATH}/.sonar - if [[ ! -x "${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner" ]]; then - curl -Ol https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SCANNER_VERSION}-${OS}.zip - unzip -qq -o sonar-scanner-cli-${SCANNER_VERSION}-${OS}.zip -d ${SCANNER_PATH} - fi - - chmod +x ${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner - chmod +x ${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/jre/bin/java - - # Explicitly set NODE_MODULES - export NODE_MODULES=${PWD}/ui/node_modules - export NODE_PATH=${PWD}/ui/node_modules - - ${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner + # We do not use the provided action, because it does contain an old + # version of the scanner, and also takes time to build. + set -e + mkdir -p ${SCANNER_PATH} + export SONAR_USER_HOME=${SCANNER_PATH}/.sonar + if [[ ! -x "${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner" ]]; then + curl -Ol https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SCANNER_VERSION}-${OS}.zip + unzip -qq -o sonar-scanner-cli-${SCANNER_VERSION}-${OS}.zip -d ${SCANNER_PATH} + fi + + chmod +x ${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner + chmod +x ${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/jre/bin/java + + # Explicitly set NODE_MODULES + export NODE_MODULES=${PWD}/ui/node_modules + export NODE_PATH=${PWD}/ui/node_modules + + ${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner if: env.sonar_secret != '' test-e2e: name: Run end-to-end tests + if: ${{ needs.changes.outputs.backend == 'true' }} runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: - k3s-version: [v1.28.2, v1.27.6, v1.26.9, v1.25.14] - needs: + k3s-version: [v1.29.1, v1.28.6, v1.27.10, v1.26.13, v1.25.16] + needs: - build-go + - changes env: GOPATH: /home/runner/go ARGOCD_FAKE_IN_CLUSTER: "true" @@ -377,7 +420,7 @@ jobs: ARGOCD_APPLICATION_NAMESPACES: "argocd-e2e-external,argocd-e2e-external-2" ARGOCD_SERVER: "127.0.0.1:8088" GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }} + GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }} steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 @@ -427,7 +470,7 @@ jobs: git config --global user.email "john.doe@example.com" - name: Pull Docker image required for tests run: | - docker pull ghcr.io/dexidp/dex:v2.37.0 + docker pull ghcr.io/dexidp/dex:v2.38.0 docker pull argoproj/argo-cd-ci-builder:v1.0.0 docker pull redis:7.0.14-alpine - name: Create target directory for binaries in the build-process @@ -462,3 +505,26 @@ jobs: name: e2e-server-k8s${{ matrix.k3s-version }}.log path: /tmp/e2e-server.log if: ${{ failure() }} + + # workaround for status checks -- check this one job instead of each individual E2E job in the matrix + # this allows us to skip the entire matrix when it doesn't need to run while still having accurate status checks + # see: + # https://github.com/argoproj/argo-workflows/pull/12006 + # https://github.com/orgs/community/discussions/9141#discussioncomment-2296809 + # https://github.com/orgs/community/discussions/26822#discussioncomment-3305794 + test-e2e-composite-result: + name: E2E Tests - Composite result + if: ${{ always() }} + needs: + - test-e2e + - changes + runs-on: ubuntu-22.04 + steps: + - run: | + result="${{ needs.test-e2e.result }}" + # mark as successful even if skipped + if [[ $result == "success" || $result == "skipped" ]]; then + exit 0 + else + exit 1 + fi \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ae5174659cf40..567ab8a23ab31 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -87,6 +87,14 @@ jobs: echo "KUBECTL_VERSION=$(go list -m k8s.io/client-go | head -n 1 | rev | cut -d' ' -f1 | rev)" >> $GITHUB_ENV echo "GIT_TREE_STATE=$(if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi)" >> $GITHUB_ENV + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@4d9e71b726748f254fe64fa44d273194bd18ec91 + with: + large-packages: false + docker-images: false + swap-storage: false + tool-cache: false + - name: Run GoReleaser uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0 id: run-goreleaser diff --git a/CODEOWNERS b/CODEOWNERS index ec72eccbf416e..83bb38871d96d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,9 +2,10 @@ ** @argoproj/argocd-approvers # Docs -/docs/** @argoproj/argocd-approvers @argoproj/argocd-approvers-docs -/USERS.md @argoproj/argocd-approvers @argoproj/argocd-approvers-docs -/mkdocs.yml @argoproj/argocd-approvers @argoproj/argocd-approvers-docs +/docs/** @argoproj/argocd-approvers @argoproj/argocd-approvers-docs +/USERS.md @argoproj/argocd-approvers @argoproj/argocd-approvers-docs +/README.md @argoproj/argocd-approvers @argoproj/argocd-approvers-docs +/mkdocs.yml @argoproj/argocd-approvers @argoproj/argocd-approvers-docs # CI /.github/** @argoproj/argocd-approvers @argoproj/argocd-approvers-ci diff --git a/Dockerfile b/Dockerfile index 461a42305f3ae..44202104d356b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,7 +83,7 @@ WORKDIR /home/argocd #################################################################################################### # Argo CD UI stage #################################################################################################### -FROM --platform=$BUILDPLATFORM docker.io/library/node:20.6.1@sha256:14bd39208dbc0eb171cbfb26ccb9ac09fa1b2eba04ccd528ab5d12983fd9ee24 AS argocd-ui +FROM --platform=$BUILDPLATFORM docker.io/library/node:21.6.2@sha256:65998e325b06014d4f1417a8a6afb1540d1ac66521cca76f2221a6953947f9ee AS argocd-ui WORKDIR /src COPY ["ui/package.json", "ui/yarn.lock", "./"] diff --git a/Makefile b/Makefile index 8bd9a49b6bb02..84282a8b25bf7 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,8 @@ DOCKER_WORKDIR?=/go/src/github.com/argoproj/argo-cd ARGOCD_PROCFILE?=Procfile -# Strict mode has been disabled in latest versions of mkdocs-material. -# Thus pointing to the older image of mkdocs-material matching the version used by argo-cd. -MKDOCS_DOCKER_IMAGE?=squidfunk/mkdocs-material:4.1.1 +# pointing to python 3.7 to match https://github.com/argoproj/argo-cd/blob/master/.readthedocs.yml +MKDOCS_DOCKER_IMAGE?=python:3.7-alpine MKDOCS_RUN_ARGS?= # Configuration for building argocd-test-tools image @@ -49,7 +48,7 @@ ARGOCD_E2E_DEX_PORT?=5556 ARGOCD_E2E_YARN_HOST?=localhost ARGOCD_E2E_DISABLE_AUTH?= -ARGOCD_E2E_TEST_TIMEOUT?=60m +ARGOCD_E2E_TEST_TIMEOUT?=90m ARGOCD_IN_CI?=false ARGOCD_TEST_E2E?=true @@ -521,7 +520,7 @@ build-docs-local: .PHONY: build-docs build-docs: - docker run ${MKDOCS_RUN_ARGS} --rm -it -v ${CURRENT_DIR}:/docs --entrypoint "" ${MKDOCS_DOCKER_IMAGE} sh -c 'pip install -r docs/requirements.txt; mkdocs build' + docker run ${MKDOCS_RUN_ARGS} --rm -it -v ${CURRENT_DIR}:/docs -w /docs --entrypoint "" ${MKDOCS_DOCKER_IMAGE} sh -c 'pip install -r docs/requirements.txt; mkdocs build' .PHONY: serve-docs-local serve-docs-local: diff --git a/Procfile b/Procfile index 3bc2de5eca5e0..4862b0230062f 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,4 @@ -controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-application-controller $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''} --server-side-diff-enabled=${ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF:-'false'}" +controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "HOSTNAME=testappcontroller-1 FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-application-controller $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''} --server-side-diff-enabled=${ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF:-'false'}" api-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-server $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --disable-auth=${ARGOCD_E2E_DISABLE_AUTH:-'true'} --insecure --dex-server http://localhost:${ARGOCD_E2E_DEX_PORT:-5556} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --port ${ARGOCD_E2E_APISERVER_PORT:-8080} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''}" dex: sh -c "ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/v2/cmd gendexcfg -o `pwd`/dist/dex.yaml && (test -f dist/dex.yaml || { echo 'Failed to generate dex configuration'; exit 1; }) && docker run --rm -p ${ARGOCD_E2E_DEX_PORT:-5556}:${ARGOCD_E2E_DEX_PORT:-5556} -v `pwd`/dist/dex.yaml:/dex.yaml ghcr.io/dexidp/dex:$(grep "image: ghcr.io/dexidp/dex" manifests/base/dex/argocd-dex-server-deployment.yaml | cut -d':' -f3) dex serve /dex.yaml" redis: bash -c "if [ \"$ARGOCD_REDIS_LOCAL\" = 'true' ]; then redis-server --save '' --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}; else docker run --rm --name argocd-redis -i -p ${ARGOCD_E2E_REDIS_PORT:-6379}:${ARGOCD_E2E_REDIS_PORT:-6379} docker.io/library/redis:$(grep "image: redis" manifests/base/redis/argocd-redis-deployment.yaml | cut -d':' -f3) --save '' --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}; fi" diff --git a/README.md b/README.md index ef5664de5b5b7..707848191c830 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ **Social:** [![Twitter Follow](https://img.shields.io/twitter/follow/argoproj?style=social)](https://twitter.com/argoproj) [![Slack](https://img.shields.io/badge/slack-argoproj-brightgreen.svg?logo=slack)](https://argoproj.github.io/community/join-slack) +[![LinkedIn](https://img.shields.io/badge/LinkedIn-argoproj-blue.svg?logo=linkedin)](https://www.linkedin.com/company/argoproj/) # Argo CD - Declarative Continuous Delivery for Kubernetes @@ -85,4 +86,5 @@ Participation in the Argo CD project is governed by the [CNCF Code of Conduct](h 1. [Getting Started with ArgoCD for GitOps Deployments](https://youtu.be/AvLuplh1skA) 1. [Using Argo CD & Datree for Stable Kubernetes CI/CD Deployments](https://youtu.be/17894DTru2Y) 1. [How to create Argo CD Applications Automatically using ApplicationSet? "Automation of GitOps"](https://amralaayassen.medium.com/how-to-create-argocd-applications-automatically-using-applicationset-automation-of-the-gitops-59455eaf4f72) +1. [Progressive Delivery with Service Mesh – Argo Rollouts with Istio](https://www.cncf.io/blog/2022/12/16/progressive-delivery-with-service-mesh-argo-rollouts-with-istio/) diff --git a/USERS.md b/USERS.md index 60dd3b881c10b..6a09724c7d7b7 100644 --- a/USERS.md +++ b/USERS.md @@ -20,6 +20,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Allianz Direct](https://www.allianzdirect.de/) 1. [Amadeus IT Group](https://amadeus.com/) 1. [Ambassador Labs](https://www.getambassador.io/) +1. [Ancestry](https://www.ancestry.com/) 1. [ANSTO - Australian Synchrotron](https://www.synchrotron.org.au/) 1. [Ant Group](https://www.antgroup.com/) 1. [AppDirect](https://www.appdirect.com) @@ -44,14 +45,14 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Camptocamp](https://camptocamp.com) 1. [Candis](https://www.candis.io) 1. [Capital One](https://www.capitalone.com) -1. [CARFAX](https://www.carfax.com) 1. [CARFAX Europe](https://www.carfax.eu) +1. [CARFAX](https://www.carfax.com) 1. [Carrefour Group](https://www.carrefour.com) 1. [Casavo](https://casavo.com) 1. [Celonis](https://www.celonis.com/) 1. [CERN](https://home.cern/) -1. [Chargetrip](https://chargetrip.com) 1. [Chainnodes](https://chainnodes.org) +1. [Chargetrip](https://chargetrip.com) 1. [Chime](https://www.chime.com) 1. [Cisco ET&I](https://eti.cisco.com/) 1. [Cloud Posse](https://www.cloudposse.com/) @@ -94,6 +95,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Fave](https://myfave.com) 1. [Flexport](https://www.flexport.com/) 1. [Flip](https://flip.id) +1. [Fly Security](https://www.flysecurity.com.br/) 1. [Fonoa](https://www.fonoa.com/) 1. [Fortra](https://www.fortra.com) 1. [freee](https://corp.freee.co.jp/en/company/) @@ -112,8 +114,8 @@ Currently, the following organizations are **officially** using Argo CD: 1. [GlueOps](https://glueops.dev) 1. [GMETRI](https://gmetri.com/) 1. [Gojek](https://www.gojek.io/) -1. [GoTo](https://www.goto.com/) 1. [GoTo Financial](https://gotofinancial.com/) +1. [GoTo](https://www.goto.com/) 1. [Greenpass](https://www.greenpass.com.br/) 1. [Gridfuse](https://gridfuse.com/) 1. [Groww](https://groww.in) @@ -126,6 +128,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Hiya](https://hiya.com) 1. [Honestbank](https://honestbank.com) 1. [Hostinger](https://www.hostinger.com) +1. [IABAI](https://www.iab.ai) 1. [IBM](https://www.ibm.com/) 1. [Ibotta](https://home.ibotta.com) 1. [IITS-Consulting](https://iits-consulting.de) @@ -187,6 +190,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Natura &Co](https://naturaeco.com/) 1. [Nethopper](https://nethopper.io) 1. [New Relic](https://newrelic.com/) +1. [Nextbasket](https://nextbasket.com) 1. [Nextdoor](https://nextdoor.com/) 1. [Nikkei](https://www.nikkei.co.jp/nikkeiinfo/en/) 1. [Nitro](https://gonitro.com) @@ -213,6 +217,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [PagerDuty](https://www.pagerduty.com/) 1. [Pandosearch](https://www.pandosearch.com/en/home) 1. [Patreon](https://www.patreon.com/) +1. [PayIt](https://payitgov.com/) 1. [PayPay](https://paypay.ne.jp/) 1. [Peloton Interactive](https://www.onepeloton.com/) 1. [Percona](https://percona.com/) @@ -220,6 +225,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Pigment](https://www.gopigment.com/) 1. [Pipefy](https://www.pipefy.com/) 1. [Pismo](https://pismo.io/) +1. [PITS Globale Datenrettungsdienste](https://www.pitsdatenrettung.de/) 1. [Platform9 Systems](https://platform9.com/) 1. [Polarpoint.io](https://polarpoint.io) 1. [PostFinance](https://github.com/postfinance) @@ -241,9 +247,11 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Reenigne Cloud](https://reenigne.ca) 1. [reev.com](https://www.reev.com/) 1. [RightRev](https://rightrev.com/) +1. [Rijkswaterstaat](https://www.rijkswaterstaat.nl/en) 1. [Rise](https://www.risecard.eu/) 1. [Riskified](https://www.riskified.com/) 1. [Robotinfra](https://www.robotinfra.com) +1. [Rocket.Chat](https://rocket.chat) 1. [Rubin Observatory](https://www.lsst.org) 1. [Saildrone](https://www.saildrone.com/) 1. [Salad Technologies](https://salad.com/) @@ -269,6 +277,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Splunk](https://splunk.com/) 1. [Spores Labs](https://spores.app) 1. [Statsig](https://statsig.com) +1. [SternumIOT](https://sternumiot.com) 1. [StreamNative](https://streamnative.io) 1. [Stuart](https://stuart.com/) 1. [Sumo Logic](https://sumologic.com/) @@ -282,6 +291,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Tamkeen Technologies](https://tamkeentech.sa/) 1. [Techcombank](https://www.techcombank.com.vn/trang-chu) 1. [Technacy](https://www.technacy.it/) +1. [Telavita](https://www.telavita.com.br/) 1. [Tesla](https://tesla.com/) 1. [The Scale Factory](https://www.scalefactory.com/) 1. [ThousandEyes](https://www.thousandeyes.com/) diff --git a/assets/badge.svg b/assets/badge.svg index cc216ccdd1508..f1dab6b6cb711 100644 --- a/assets/badge.svg +++ b/assets/badge.svg @@ -5,6 +5,7 @@ + @@ -14,6 +15,7 @@ + diff --git a/assets/swagger.json b/assets/swagger.json index 3c7c3d687f38a..1ecc59993e088 100644 --- a/assets/swagger.json +++ b/assets/swagger.json @@ -5700,6 +5700,10 @@ "type": "string", "title": "ClusterName contains AWS cluster name" }, + "profile": { + "description": "Profile contains optional role ARN. If set then AWS IAM Authenticator uses the profile to perform cluster operations instead of the default AWS credential provider chain.", + "type": "string" + }, "roleARN": { "description": "RoleARN contains optional role ARN. If set then AWS IAM Authenticator assume a role to perform cluster operations instead of the default AWS credential provider chain.", "type": "string" @@ -6456,6 +6460,10 @@ "type": "string" } }, + "labelWithoutSelector": { + "type": "boolean", + "title": "LabelWithoutSelector specifies whether to apply common labels to resource selectors or not" + }, "namePrefix": { "type": "string", "title": "NamePrefix is a prefix appended to resources for Kustomize apps" diff --git a/cmd/argocd-application-controller/commands/argocd_application_controller.go b/cmd/argocd-application-controller/commands/argocd_application_controller.go index d5ef88a1702b6..3c7fe8bbac107 100644 --- a/cmd/argocd-application-controller/commands/argocd_application_controller.go +++ b/cmd/argocd-application-controller/commands/argocd_application_controller.go @@ -10,8 +10,6 @@ import ( "github.com/redis/go-redis/v9" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - kubeerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -26,7 +24,6 @@ import ( cacheutil "github.com/argoproj/argo-cd/v2/util/cache" appstatecache "github.com/argoproj/argo-cd/v2/util/cache/appstate" "github.com/argoproj/argo-cd/v2/util/cli" - "github.com/argoproj/argo-cd/v2/util/db" "github.com/argoproj/argo-cd/v2/util/env" "github.com/argoproj/argo-cd/v2/util/errors" kubeutil "github.com/argoproj/argo-cd/v2/util/kube" @@ -50,6 +47,7 @@ func NewCommand() *cobra.Command { clientConfig clientcmd.ClientConfig appResyncPeriod int64 appHardResyncPeriod int64 + appResyncJitter int64 repoErrorGracePeriod int64 repoServerAddress string repoServerTimeoutSeconds int @@ -146,7 +144,8 @@ func NewCommand() *cobra.Command { appController.InvalidateProjectsCache() })) kubectl := kubeutil.NewKubectl() - clusterFilter := getClusterFilter(kubeClient, settingsMgr, shardingAlgorithm, enableDynamicClusterDistribution) + clusterSharding, err := sharding.GetClusterSharding(kubeClient, settingsMgr, shardingAlgorithm, enableDynamicClusterDistribution) + errors.CheckError(err) appController, err = controller.NewApplicationController( namespace, settingsMgr, @@ -157,6 +156,7 @@ func NewCommand() *cobra.Command { kubectl, resyncDuration, hardResyncDuration, + time.Duration(appResyncJitter)*time.Second, time.Duration(selfHealTimeoutSeconds)*time.Second, time.Duration(repoErrorGracePeriod)*time.Second, metricsPort, @@ -164,10 +164,11 @@ func NewCommand() *cobra.Command { metricsAplicationLabels, kubectlParallelismLimit, persistResourceHealth, - clusterFilter, + clusterSharding, applicationNamespaces, &workqueueRateLimit, serverSideDiff, + enableDynamicClusterDistribution, ) errors.CheckError(err) cacheutil.CollectMetrics(redisClient, appController.GetMetricsServer()) @@ -194,6 +195,7 @@ func NewCommand() *cobra.Command { clientConfig = cli.AddKubectlFlagsToCmd(&command) command.Flags().Int64Var(&appResyncPeriod, "app-resync", int64(env.ParseDurationFromEnv("ARGOCD_RECONCILIATION_TIMEOUT", defaultAppResyncPeriod*time.Second, 0, math.MaxInt64).Seconds()), "Time period in seconds for application resync.") command.Flags().Int64Var(&appHardResyncPeriod, "app-hard-resync", int64(env.ParseDurationFromEnv("ARGOCD_HARD_RECONCILIATION_TIMEOUT", defaultAppHardResyncPeriod*time.Second, 0, math.MaxInt64).Seconds()), "Time period in seconds for application hard resync.") + command.Flags().Int64Var(&appResyncJitter, "app-resync-jitter", int64(env.ParseDurationFromEnv("ARGOCD_RECONCILIATION_JITTER", 0*time.Second, 0, math.MaxInt64).Seconds()), "Maximum time period in seconds to add as a delay jitter for application resync.") command.Flags().Int64Var(&repoErrorGracePeriod, "repo-error-grace-period-seconds", int64(env.ParseDurationFromEnv("ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS", defaultAppResyncPeriod*time.Second, 0, math.MaxInt64).Seconds()), "Grace period in seconds for ignoring consecutive errors while communicating with repo server.") command.Flags().StringVar(&repoServerAddress, "repo-server", env.StringFromEnv("ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER", common.DefaultRepoServerAddr), "Repo server address.") command.Flags().IntVar(&repoServerTimeoutSeconds, "repo-server-timeout-seconds", env.ParseNumFromEnv("ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_TIMEOUT_SECONDS", 60, 0, math.MaxInt64), "Repo server RPC call timeout seconds.") @@ -218,7 +220,7 @@ func NewCommand() *cobra.Command { command.Flags().StringVar(&shardingAlgorithm, "sharding-method", env.StringFromEnv(common.EnvControllerShardingAlgorithm, common.DefaultShardingAlgorithm), "Enables choice of sharding method. Supported sharding methods are : [legacy, round-robin] ") // global queue rate limit config command.Flags().Int64Var(&workqueueRateLimit.BucketSize, "wq-bucket-size", env.ParseInt64FromEnv("WORKQUEUE_BUCKET_SIZE", 500, 1, math.MaxInt64), "Set Workqueue Rate Limiter Bucket Size, default 500") - command.Flags().Int64Var(&workqueueRateLimit.BucketQPS, "wq-bucket-qps", env.ParseInt64FromEnv("WORKQUEUE_BUCKET_QPS", 50, 1, math.MaxInt64), "Set Workqueue Rate Limiter Bucket QPS, default 50") + command.Flags().Float64Var(&workqueueRateLimit.BucketQPS, "wq-bucket-qps", env.ParseFloat64FromEnv("WORKQUEUE_BUCKET_QPS", math.MaxFloat64, 1, math.MaxFloat64), "Set Workqueue Rate Limiter Bucket QPS, default set to MaxFloat64 which disables the bucket limiter") // individual item rate limit config // when WORKQUEUE_FAILURE_COOLDOWN is 0 per item rate limiting is disabled(default) command.Flags().DurationVar(&workqueueRateLimit.FailureCoolDown, "wq-cooldown-ns", time.Duration(env.ParseInt64FromEnv("WORKQUEUE_FAILURE_COOLDOWN_NS", 0, 0, (24*time.Hour).Nanoseconds())), "Set Workqueue Per Item Rate Limiter Cooldown duration in ns, default 0(per item rate limiter disabled)") @@ -234,58 +236,3 @@ func NewCommand() *cobra.Command { }) return &command } - -func getClusterFilter(kubeClient *kubernetes.Clientset, settingsMgr *settings.SettingsManager, shardingAlgorithm string, enableDynamicClusterDistribution bool) sharding.ClusterFilterFunction { - - var replicas int - shard := env.ParseNumFromEnv(common.EnvControllerShard, -1, -math.MaxInt32, math.MaxInt32) - - applicationControllerName := env.StringFromEnv(common.EnvAppControllerName, common.DefaultApplicationControllerName) - appControllerDeployment, err := kubeClient.AppsV1().Deployments(settingsMgr.GetNamespace()).Get(context.Background(), applicationControllerName, metav1.GetOptions{}) - - // if the application controller deployment was not found, the Get() call returns an empty Deployment object. So, set the variable to nil explicitly - if err != nil && kubeerrors.IsNotFound(err) { - appControllerDeployment = nil - } - - if enableDynamicClusterDistribution && appControllerDeployment != nil && appControllerDeployment.Spec.Replicas != nil { - replicas = int(*appControllerDeployment.Spec.Replicas) - } else { - replicas = env.ParseNumFromEnv(common.EnvControllerReplicas, 0, 0, math.MaxInt32) - } - - var clusterFilter func(cluster *v1alpha1.Cluster) bool - if replicas > 1 { - // check for shard mapping using configmap if application-controller is a deployment - // else use existing logic to infer shard from pod name if application-controller is a statefulset - if enableDynamicClusterDistribution && appControllerDeployment != nil { - - var err error - // retry 3 times if we find a conflict while updating shard mapping configMap. - // If we still see conflicts after the retries, wait for next iteration of heartbeat process. - for i := 0; i <= common.AppControllerHeartbeatUpdateRetryCount; i++ { - shard, err = sharding.GetOrUpdateShardFromConfigMap(kubeClient, settingsMgr, replicas, shard) - if !kubeerrors.IsConflict(err) { - err = fmt.Errorf("unable to get shard due to error updating the sharding config map: %s", err) - break - } - log.Warnf("conflict when getting shard from shard mapping configMap. Retrying (%d/3)", i) - } - errors.CheckError(err) - } else { - if shard < 0 { - var err error - shard, err = sharding.InferShard() - errors.CheckError(err) - } - } - log.Infof("Processing clusters from shard %d", shard) - db := db.NewDB(settingsMgr.GetNamespace(), settingsMgr, kubeClient) - log.Infof("Using filter function: %s", shardingAlgorithm) - distributionFunction := sharding.GetDistributionFunction(db, shardingAlgorithm) - clusterFilter = sharding.GetClusterFilter(db, distributionFunction, shard) - } else { - log.Info("Processing all cluster shards") - } - return clusterFilter -} diff --git a/cmd/argocd-k8s-auth/commands/aws.go b/cmd/argocd-k8s-auth/commands/aws.go index 79a118d2653a3..9b750ac5f92f8 100644 --- a/cmd/argocd-k8s-auth/commands/aws.go +++ b/cmd/argocd-k8s-auth/commands/aws.go @@ -37,13 +37,14 @@ func newAWSCommand() *cobra.Command { var ( clusterName string roleARN string + profile string ) var command = &cobra.Command{ Use: "aws", Run: func(c *cobra.Command, args []string) { ctx := c.Context() - presignedURLString, err := getSignedRequestWithRetry(ctx, time.Minute, 5*time.Second, clusterName, roleARN, getSignedRequest) + presignedURLString, err := getSignedRequestWithRetry(ctx, time.Minute, 5*time.Second, clusterName, roleARN, profile, getSignedRequest) errors.CheckError(err) token := v1Prefix + base64.RawURLEncoding.EncodeToString([]byte(presignedURLString)) // Set token expiration to 1 minute before the presigned URL expires for some cushion @@ -53,16 +54,17 @@ func newAWSCommand() *cobra.Command { } command.Flags().StringVar(&clusterName, "cluster-name", "", "AWS Cluster name") command.Flags().StringVar(&roleARN, "role-arn", "", "AWS Role ARN") + command.Flags().StringVar(&profile, "profile", "", "AWS Profile") return command } -type getSignedRequestFunc func(clusterName, roleARN string) (string, error) +type getSignedRequestFunc func(clusterName, roleARN string, profile string) (string, error) -func getSignedRequestWithRetry(ctx context.Context, timeout, interval time.Duration, clusterName, roleARN string, fn getSignedRequestFunc) (string, error) { +func getSignedRequestWithRetry(ctx context.Context, timeout, interval time.Duration, clusterName, roleARN string, profile string, fn getSignedRequestFunc) (string, error) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() for { - signed, err := fn(clusterName, roleARN) + signed, err := fn(clusterName, roleARN, profile) if err == nil { return signed, nil } @@ -74,8 +76,10 @@ func getSignedRequestWithRetry(ctx context.Context, timeout, interval time.Durat } } -func getSignedRequest(clusterName, roleARN string) (string, error) { - sess, err := session.NewSession() +func getSignedRequest(clusterName, roleARN string, profile string) (string, error) { + sess, err := session.NewSessionWithOptions(session.Options{ + Profile: profile, + }) if err != nil { return "", fmt.Errorf("error creating new AWS session: %s", err) } diff --git a/cmd/argocd-k8s-auth/commands/aws_test.go b/cmd/argocd-k8s-auth/commands/aws_test.go index c22449eba42be..578aae71a2c29 100644 --- a/cmd/argocd-k8s-auth/commands/aws_test.go +++ b/cmd/argocd-k8s-auth/commands/aws_test.go @@ -22,7 +22,7 @@ func TestGetSignedRequestWithRetry(t *testing.T) { } // when - signed, err := getSignedRequestWithRetry(ctx, time.Second, time.Millisecond, "cluster-name", "", mock.getSignedRequestMock) + signed, err := getSignedRequestWithRetry(ctx, time.Second, time.Millisecond, "cluster-name", "", "", mock.getSignedRequestMock) // then assert.NoError(t, err) @@ -41,7 +41,7 @@ func TestGetSignedRequestWithRetry(t *testing.T) { } // when - signed, err := getSignedRequestWithRetry(ctx, time.Second, time.Millisecond, "cluster-name", "", mock.getSignedRequestMock) + signed, err := getSignedRequestWithRetry(ctx, time.Second, time.Millisecond, "cluster-name", "", "", mock.getSignedRequestMock) // then assert.NoError(t, err) @@ -57,7 +57,7 @@ func TestGetSignedRequestWithRetry(t *testing.T) { } // when - signed, err := getSignedRequestWithRetry(ctx, time.Second, time.Millisecond, "cluster-name", "", mock.getSignedRequestMock) + signed, err := getSignedRequestWithRetry(ctx, time.Second, time.Millisecond, "cluster-name", "", "", mock.getSignedRequestMock) // then assert.Error(t, err) @@ -70,7 +70,7 @@ type signedRequestMock struct { returnFunc func(m *signedRequestMock) (string, error) } -func (m *signedRequestMock) getSignedRequestMock(clusterName, roleARN string) (string, error) { +func (m *signedRequestMock) getSignedRequestMock(clusterName, roleARN string, profile string) (string, error) { m.getSignedRequestCalls++ return m.returnFunc(m) } diff --git a/cmd/argocd-server/commands/argocd_server.go b/cmd/argocd-server/commands/argocd_server.go index 72fe765c32c56..27a2db34189b4 100644 --- a/cmd/argocd-server/commands/argocd_server.go +++ b/cmd/argocd-server/commands/argocd_server.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "strings" "time" "github.com/argoproj/pkg/stats" @@ -63,6 +64,7 @@ func NewCommand() *cobra.Command { repoServerAddress string dexServerAddress string disableAuth bool + contentTypes string enableGZip bool tlsConfigCustomizerSrc func() (tls.ConfigCustomizer, error) cacheSrc func() (*servercache.Cache, error) @@ -170,6 +172,11 @@ func NewCommand() *cobra.Command { baseHRef = rootPath } + var contentTypesList []string + if contentTypes != "" { + contentTypesList = strings.Split(contentTypes, ";") + } + argoCDOpts := server.ArgoCDServerOpts{ Insecure: insecure, ListenPort: listenPort, @@ -185,6 +192,7 @@ func NewCommand() *cobra.Command { DexServerAddr: dexServerAddress, DexTLSConfig: dexTlsConfig, DisableAuth: disableAuth, + ContentTypes: contentTypesList, EnableGZip: enableGZip, TLSConfigCustomizer: tlsConfigCustomizer, Cache: cache, @@ -240,6 +248,7 @@ func NewCommand() *cobra.Command { command.Flags().StringVar(&repoServerAddress, "repo-server", env.StringFromEnv("ARGOCD_SERVER_REPO_SERVER", common.DefaultRepoServerAddr), "Repo server address") command.Flags().StringVar(&dexServerAddress, "dex-server", env.StringFromEnv("ARGOCD_SERVER_DEX_SERVER", common.DefaultDexServerAddr), "Dex server address") command.Flags().BoolVar(&disableAuth, "disable-auth", env.ParseBoolFromEnv("ARGOCD_SERVER_DISABLE_AUTH", false), "Disable client authentication") + command.Flags().StringVar(&contentTypes, "api-content-types", env.StringFromEnv("ARGOCD_API_CONTENT_TYPES", "application/json", env.StringFromEnvOpts{AllowEmpty: true}), "Semicolon separated list of allowed content types for non GET api requests. Any content type is allowed if empty.") command.Flags().BoolVar(&enableGZip, "enable-gzip", env.ParseBoolFromEnv("ARGOCD_SERVER_ENABLE_GZIP", true), "Enable GZIP compression") command.AddCommand(cli.NewVersionCmd(cliName)) command.Flags().StringVar(&listenHost, "address", env.StringFromEnv("ARGOCD_SERVER_LISTEN_ADDRESS", common.DefaultAddressAPIServer), "Listen on given address") diff --git a/cmd/argocd/commands/admin/cluster.go b/cmd/argocd/commands/admin/cluster.go index 5d14717a15e7d..2e833a68927f4 100644 --- a/cmd/argocd/commands/admin/cluster.go +++ b/cmd/argocd/commands/admin/cluster.go @@ -25,7 +25,7 @@ import ( "github.com/argoproj/argo-cd/v2/common" "github.com/argoproj/argo-cd/v2/controller/sharding" argocdclient "github.com/argoproj/argo-cd/v2/pkg/apiclient" - argoappv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned" "github.com/argoproj/argo-cd/v2/util/argo" cacheutil "github.com/argoproj/argo-cd/v2/util/cache" @@ -71,14 +71,14 @@ argocd admin cluster namespaces my-cluster `, } type ClusterWithInfo struct { - argoappv1.Cluster + v1alpha1.Cluster // Shard holds controller shard number that handles the cluster Shard int // Namespaces holds list of namespaces managed by Argo CD in the cluster Namespaces []string } -func loadClusters(ctx context.Context, kubeClient *kubernetes.Clientset, appClient *versioned.Clientset, replicas int, namespace string, portForwardRedis bool, cacheSrc func() (*appstatecache.Cache, error), shard int, redisName string, redisHaProxyName string, redisCompressionStr string) ([]ClusterWithInfo, error) { +func loadClusters(ctx context.Context, kubeClient *kubernetes.Clientset, appClient *versioned.Clientset, replicas int, shardingAlgorithm string, namespace string, portForwardRedis bool, cacheSrc func() (*appstatecache.Cache, error), shard int, redisName string, redisHaProxyName string, redisCompressionStr string) ([]ClusterWithInfo, error) { settingsMgr := settings.NewSettingsManager(ctx, kubeClient, namespace) argoDB := db.NewDB(namespace, settingsMgr, kubeClient) @@ -86,6 +86,14 @@ func loadClusters(ctx context.Context, kubeClient *kubernetes.Clientset, appClie if err != nil { return nil, err } + appItems, err := appClient.ArgoprojV1alpha1().Applications(namespace).List(ctx, v1.ListOptions{}) + if err != nil { + return nil, err + } + clusterShardingCache := sharding.NewClusterSharding(argoDB, shard, replicas, shardingAlgorithm) + clusterShardingCache.Init(clustersList, appItems) + clusterShards := clusterShardingCache.GetDistribution() + var cache *appstatecache.Cache if portForwardRedis { overrides := clientcmd.ConfigOverrides{} @@ -109,10 +117,6 @@ func loadClusters(ctx context.Context, kubeClient *kubernetes.Clientset, appClie } } - appItems, err := appClient.ArgoprojV1alpha1().Applications(namespace).List(ctx, v1.ListOptions{}) - if err != nil { - return nil, err - } apps := appItems.Items for i, app := range apps { err := argo.ValidateDestination(ctx, &app.Spec.Destination, argoDB) @@ -122,6 +126,7 @@ func loadClusters(ctx context.Context, kubeClient *kubernetes.Clientset, appClie apps[i] = app } clusters := make([]ClusterWithInfo, len(clustersList.Items)) + batchSize := 10 batchesCount := int(math.Ceil(float64(len(clusters)) / float64(batchSize))) for batchNum := 0; batchNum < batchesCount; batchNum++ { @@ -135,12 +140,10 @@ func loadClusters(ctx context.Context, kubeClient *kubernetes.Clientset, appClie clusterShard := 0 cluster := batch[i] if replicas > 0 { - distributionFunction := sharding.GetDistributionFunction(argoDB, common.DefaultShardingAlgorithm) - distributionFunction(&cluster) + clusterShard = clusterShards[cluster.Server] cluster.Shard = pointer.Int64(int64(clusterShard)) log.Infof("Cluster with uid: %s will be processed by shard %d", cluster.ID, clusterShard) } - if shard != -1 && clusterShard != shard { return nil } @@ -176,6 +179,7 @@ func NewClusterShardsCommand(clientOpts *argocdclient.ClientOptions) *cobra.Comm var ( shard int replicas int + shardingAlgorithm string clientConfig clientcmd.ClientConfig cacheSrc func() (*appstatecache.Cache, error) portForwardRedis bool @@ -183,7 +187,7 @@ func NewClusterShardsCommand(clientOpts *argocdclient.ClientOptions) *cobra.Comm ) var command = cobra.Command{ Use: "shards", - Short: "Print information about each controller shard and portion of Kubernetes resources it is responsible for.", + Short: "Print information about each controller shard and the estimated portion of Kubernetes resources it is responsible for.", Run: func(cmd *cobra.Command, args []string) { ctx := cmd.Context() @@ -203,8 +207,7 @@ func NewClusterShardsCommand(clientOpts *argocdclient.ClientOptions) *cobra.Comm if replicas == 0 { return } - - clusters, err := loadClusters(ctx, kubeClient, appClient, replicas, namespace, portForwardRedis, cacheSrc, shard, clientOpts.RedisName, clientOpts.RedisHaProxyName, redisCompressionStr) + clusters, err := loadClusters(ctx, kubeClient, appClient, replicas, shardingAlgorithm, namespace, portForwardRedis, cacheSrc, shard, clientOpts.RedisName, clientOpts.RedisHaProxyName, redisCompressionStr) errors.CheckError(err) if len(clusters) == 0 { return @@ -216,7 +219,9 @@ func NewClusterShardsCommand(clientOpts *argocdclient.ClientOptions) *cobra.Comm clientConfig = cli.AddKubectlFlagsToCmd(&command) command.Flags().IntVar(&shard, "shard", -1, "Cluster shard filter") command.Flags().IntVar(&replicas, "replicas", 0, "Application controller replicas count. Inferred from number of running controller pods if not specified") + command.Flags().StringVar(&shardingAlgorithm, "sharding-method", common.DefaultShardingAlgorithm, "Sharding method. Defaults: legacy. Supported sharding methods are : [legacy, round-robin] ") command.Flags().BoolVar(&portForwardRedis, "port-forward-redis", true, "Automatically port-forward ha proxy redis from current namespace?") + cacheSrc = appstatecache.AddCacheFlagsToCmd(&command) // parse all added flags so far to get the redis-compression flag that was added by AddCacheFlagsToCmd() above @@ -461,6 +466,7 @@ func NewClusterStatsCommand(clientOpts *argocdclient.ClientOptions) *cobra.Comma var ( shard int replicas int + shardingAlgorithm string clientConfig clientcmd.ClientConfig cacheSrc func() (*appstatecache.Cache, error) portForwardRedis bool @@ -494,7 +500,7 @@ argocd admin cluster stats target-cluster`, replicas, err = getControllerReplicas(ctx, kubeClient, namespace, clientOpts.AppControllerName) errors.CheckError(err) } - clusters, err := loadClusters(ctx, kubeClient, appClient, replicas, namespace, portForwardRedis, cacheSrc, shard, clientOpts.RedisName, clientOpts.RedisHaProxyName, redisCompressionStr) + clusters, err := loadClusters(ctx, kubeClient, appClient, replicas, shardingAlgorithm, namespace, portForwardRedis, cacheSrc, shard, clientOpts.RedisName, clientOpts.RedisHaProxyName, redisCompressionStr) errors.CheckError(err) w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) @@ -508,6 +514,7 @@ argocd admin cluster stats target-cluster`, clientConfig = cli.AddKubectlFlagsToCmd(&command) command.Flags().IntVar(&shard, "shard", -1, "Cluster shard filter") command.Flags().IntVar(&replicas, "replicas", 0, "Application controller replicas count. Inferred from number of running controller pods if not specified") + command.Flags().StringVar(&shardingAlgorithm, "sharding-method", common.DefaultShardingAlgorithm, "Sharding method. Defaults: legacy. Supported sharding methods are : [legacy, round-robin] ") command.Flags().BoolVar(&portForwardRedis, "port-forward-redis", true, "Automatically port-forward ha proxy redis from current namespace?") cacheSrc = appstatecache.AddCacheFlagsToCmd(&command) @@ -610,15 +617,16 @@ func NewGenClusterConfigCommand(pathOpts *clientcmd.PathOptions) *cobra.Command errors.CheckError(err) kubeClientset := fake.NewSimpleClientset() - var awsAuthConf *argoappv1.AWSAuthConfig - var execProviderConf *argoappv1.ExecProviderConfig + var awsAuthConf *v1alpha1.AWSAuthConfig + var execProviderConf *v1alpha1.ExecProviderConfig if clusterOpts.AwsClusterName != "" { - awsAuthConf = &argoappv1.AWSAuthConfig{ + awsAuthConf = &v1alpha1.AWSAuthConfig{ ClusterName: clusterOpts.AwsClusterName, RoleARN: clusterOpts.AwsRoleArn, + Profile: clusterOpts.AwsProfile, } } else if clusterOpts.ExecProviderCommand != "" { - execProviderConf = &argoappv1.ExecProviderConfig{ + execProviderConf = &v1alpha1.ExecProviderConfig{ Command: clusterOpts.ExecProviderCommand, Args: clusterOpts.ExecProviderArgs, Env: clusterOpts.ExecProviderEnv, @@ -642,7 +650,7 @@ func NewGenClusterConfigCommand(pathOpts *clientcmd.PathOptions) *cobra.Command clst := cmdutil.NewCluster(contextName, clusterOpts.Namespaces, clusterOpts.ClusterResources, conf, bearerToken, awsAuthConf, execProviderConf, labelsMap, annotationsMap) if clusterOpts.InClusterEndpoint() { - clst.Server = argoappv1.KubernetesInternalAPIServerAddr + clst.Server = v1alpha1.KubernetesInternalAPIServerAddr } if clusterOpts.ClusterEndpoint == string(cmdutil.KubePublicEndpoint) { // Ignore `kube-public` cluster endpoints, since this command is intended to run without invoking any network connections. diff --git a/cmd/argocd/commands/app.go b/cmd/argocd/commands/app.go index 0a54c517ca696..11762c026b25d 100644 --- a/cmd/argocd/commands/app.go +++ b/cmd/argocd/commands/app.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + k8swatch "k8s.io/apimachinery/pkg/watch" "k8s.io/utils/pointer" "sigs.k8s.io/yaml" @@ -92,6 +93,8 @@ func NewApplicationCommand(clientOpts *argocdclient.ClientOptions) *cobra.Comman command.AddCommand(NewApplicationResourceActionsCommand(clientOpts)) command.AddCommand(NewApplicationListResourcesCommand(clientOpts)) command.AddCommand(NewApplicationLogsCommand(clientOpts)) + command.AddCommand(NewApplicationAddSourceCommand(clientOpts)) + command.AddCommand(NewApplicationRemoveSourceCommand(clientOpts)) return command } @@ -101,6 +104,7 @@ type watchOpts struct { operation bool suspended bool degraded bool + delete bool } // NewApplicationCreateCommand returns a new instance of an `argocd app create` command @@ -301,7 +305,7 @@ func printHeader(acdClient argocdclient.Client, app *argoappv1.Application, ctx fmt.Println() printOperationResult(app.Status.OperationState) } - if showParams { + if !app.Spec.HasMultipleSources() && showParams { printParams(app) } } @@ -314,6 +318,7 @@ func NewApplicationGetCommand(clientOpts *argocdclient.ClientOptions) *cobra.Com output string showParams bool showOperation bool + appNamespace string ) var command = &cobra.Command{ Use: "get APPNAME", @@ -357,7 +362,7 @@ func NewApplicationGetCommand(clientOpts *argocdclient.ClientOptions) *cobra.Com conn, appIf := acdClient.NewApplicationClientOrDie() defer argoio.Close(conn) - appName, appNs := argo.ParseFromQualifiedName(args[0], "") + appName, appNs := argo.ParseFromQualifiedName(args[0], appNamespace) app, err := appIf.Get(ctx, &application.ApplicationQuery{ Name: &appName, @@ -410,6 +415,7 @@ func NewApplicationGetCommand(clientOpts *argocdclient.ClientOptions) *cobra.Com command.Flags().BoolVar(&showParams, "show-params", false, "Show application parameters and overrides") command.Flags().BoolVar(&refresh, "refresh", false, "Refresh application data when retrieving") command.Flags().BoolVar(&hardRefresh, "hard-refresh", false, "Refresh application data as well as target manifests cache") + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Only get application from namespace") return command } @@ -545,16 +551,19 @@ func NewApplicationLogsCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co } func printAppSummaryTable(app *argoappv1.Application, appURL string, windows *argoappv1.SyncWindows) { - source := app.Spec.GetSource() fmt.Printf(printOpFmtStr, "Name:", app.QualifiedName()) fmt.Printf(printOpFmtStr, "Project:", app.Spec.GetProject()) fmt.Printf(printOpFmtStr, "Server:", getServer(app)) fmt.Printf(printOpFmtStr, "Namespace:", app.Spec.Destination.Namespace) fmt.Printf(printOpFmtStr, "URL:", appURL) - fmt.Printf(printOpFmtStr, "Repo:", source.RepoURL) - fmt.Printf(printOpFmtStr, "Target:", source.TargetRevision) - fmt.Printf(printOpFmtStr, "Path:", source.Path) - printAppSourceDetails(&source) + if !app.Spec.HasMultipleSources() { + fmt.Println("Source:") + } else { + fmt.Println("Sources:") + } + for _, source := range app.Spec.GetSources() { + printAppSourceDetails(&source) + } var wds []string var status string var allow, deny, inactiveAllows bool @@ -624,11 +633,19 @@ func printAppSummaryTable(app *argoappv1.Application, appURL string, windows *ar } func printAppSourceDetails(appSrc *argoappv1.ApplicationSource) { + fmt.Printf(printOpFmtStr, "- Repo:", appSrc.RepoURL) + fmt.Printf(printOpFmtStr, " Target:", appSrc.TargetRevision) + if appSrc.Path != "" { + fmt.Printf(printOpFmtStr, " Path:", appSrc.Path) + } + if appSrc.Ref != "" { + fmt.Printf(printOpFmtStr, " Ref:", appSrc.Ref) + } if appSrc.Helm != nil && len(appSrc.Helm.ValueFiles) > 0 { - fmt.Printf(printOpFmtStr, "Helm Values:", strings.Join(appSrc.Helm.ValueFiles, ",")) + fmt.Printf(printOpFmtStr, " Helm Values:", strings.Join(appSrc.Helm.ValueFiles, ",")) } if appSrc.Kustomize != nil && appSrc.Kustomize.NamePrefix != "" { - fmt.Printf(printOpFmtStr, "Name Prefix:", appSrc.Kustomize.NamePrefix) + fmt.Printf(printOpFmtStr, " Name Prefix:", appSrc.Kustomize.NamePrefix) } } @@ -1057,6 +1074,7 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co localRepoRoot string serverSideGenerate bool localIncludes []string + appNamespace string ) shortDesc := "Perform a diff against the target and live state." var command = &cobra.Command{ @@ -1073,7 +1091,7 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co clientset := headless.NewClientOrDie(clientOpts, c) conn, appIf := clientset.NewApplicationClientOrDie() defer argoio.Close(conn) - appName, appNs := argo.ParseFromQualifiedName(args[0], "") + appName, appNs := argo.ParseFromQualifiedName(args[0], appNamespace) app, err := appIf.Get(ctx, &application.ApplicationQuery{ Name: &appName, Refresh: getRefreshType(refresh, hardRefresh), @@ -1116,6 +1134,7 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co defer argoio.Close(conn) cluster, err := clusterIf.Get(ctx, &clusterpkg.ClusterQuery{Name: app.Spec.Destination.Name, Server: app.Spec.Destination.Server}) errors.CheckError(err) + diffOption.local = local diffOption.localRepoRoot = localRepoRoot diffOption.cluster = cluster @@ -1136,6 +1155,7 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co command.Flags().StringVar(&localRepoRoot, "local-repo-root", "/", "Path to the repository root. Used together with --local allows setting the repository root") command.Flags().BoolVar(&serverSideGenerate, "server-side-generate", false, "Used with --local, this will send your manifests to the server for diffing") command.Flags().StringArrayVar(&localIncludes, "local-include", []string{"*.yaml", "*.yml", "*.json"}, "Used with --server-side-generate, specify patterns of filenames to send. Matching is based on filename and not path.") + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Only render the difference in namespace") return command } @@ -1276,6 +1296,8 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra. noPrompt bool propagationPolicy string selector string + wait bool + appNamespace string ) var command = &cobra.Command{ Use: "delete APPNAME", @@ -1299,7 +1321,8 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra. c.HelpFunc()(c, args) os.Exit(1) } - conn, appIf := headless.NewClientOrDie(clientOpts, c).NewApplicationClientOrDie() + acdClient := headless.NewClientOrDie(clientOpts, c) + conn, appIf := acdClient.NewApplicationClientOrDie() defer argoio.Close(conn) var isTerminal bool = isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) var isConfirmAll bool = false @@ -1317,7 +1340,7 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra. } for _, appFullName := range appNames { - appName, appNs := argo.ParseFromQualifiedName(appFullName, "") + appName, appNs := argo.ParseFromQualifiedName(appFullName, appNamespace) appDeleteReq := application.ApplicationDeleteRequest{ Name: &appName, AppNamespace: &appNs, @@ -1346,6 +1369,9 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra. if lowercaseAnswer == "y" { _, err := appIf.Delete(ctx, &appDeleteReq) errors.CheckError(err) + if wait { + checkForDeleteEvent(ctx, acdClient, appFullName) + } fmt.Printf("application '%s' deleted\n", appFullName) } else { fmt.Println("The command to delete '" + appFullName + "' was cancelled.") @@ -1353,6 +1379,10 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra. } else { _, err := appIf.Delete(ctx, &appDeleteReq) errors.CheckError(err) + + if wait { + checkForDeleteEvent(ctx, acdClient, appFullName) + } } } }, @@ -1361,9 +1391,20 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra. command.Flags().StringVarP(&propagationPolicy, "propagation-policy", "p", "foreground", "Specify propagation policy for deletion of application's resources. One of: foreground|background") command.Flags().BoolVarP(&noPrompt, "yes", "y", false, "Turn off prompting to confirm cascaded deletion of application resources") command.Flags().StringVarP(&selector, "selector", "l", "", "Delete all apps with matching label. Supports '=', '==', '!=', in, notin, exists & not exists. Matching apps must satisfy all of the specified label constraints.") + command.Flags().BoolVar(&wait, "wait", false, "Wait until deletion of the application(s) completes") + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Namespace where the application will be deleted from") return command } +func checkForDeleteEvent(ctx context.Context, acdClient argocdclient.Client, appFullName string) { + appEventCh := acdClient.WatchApplicationWithRetry(ctx, appFullName, "") + for appEvent := range appEventCh { + if appEvent.Type == k8swatch.Deleted { + return + } + } +} + // Print simple list of application names func printApplicationNames(apps []argoappv1.Application) { for _, app := range apps { @@ -1575,11 +1616,12 @@ func getWatchOpts(watch watchOpts) watchOpts { // NewApplicationWaitCommand returns a new instance of an `argocd app wait` command func NewApplicationWaitCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { var ( - watch watchOpts - timeout uint - selector string - resources []string - output string + watch watchOpts + timeout uint + selector string + resources []string + output string + appNamespace string ) var command = &cobra.Command{ Use: "wait [APPNAME.. | -l selector]", @@ -1624,10 +1666,14 @@ func NewApplicationWaitCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co list, err := appIf.List(ctx, &application.ApplicationQuery{Selector: pointer.String(selector)}) errors.CheckError(err) for _, i := range list.Items { - appNames = append(appNames, i.Name) + appNames = append(appNames, i.QualifiedName()) } } for _, appName := range appNames { + // Construct QualifiedName + if appNamespace != "" && !strings.Contains(appName, "/") { + appName = appNamespace + "/" + appName + } _, _, err := waitOnApplicationStatus(ctx, acdClient, appName, timeout, watch, selectedResources, output) errors.CheckError(err) } @@ -1637,10 +1683,12 @@ func NewApplicationWaitCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co command.Flags().BoolVar(&watch.health, "health", false, "Wait for health") command.Flags().BoolVar(&watch.suspended, "suspended", false, "Wait for suspended") command.Flags().BoolVar(&watch.degraded, "degraded", false, "Wait for degraded") + command.Flags().BoolVar(&watch.delete, "delete", false, "Wait for delete") command.Flags().StringVarP(&selector, "selector", "l", "", "Wait for apps by label. Supports '=', '==', '!=', in, notin, exists & not exists. Matching apps must satisfy all of the specified label constraints.") command.Flags().StringArrayVar(&resources, "resource", []string{}, fmt.Sprintf("Sync only specific resources as GROUP%[1]sKIND%[1]sNAME or %[2]sGROUP%[1]sKIND%[1]sNAME. Fields may be blank and '*' can be used. This option may be specified repeatedly", resourceFieldDelimiter, resourceExcludeIndicator)) command.Flags().BoolVar(&watch.operation, "operation", false, "Wait for pending operations") command.Flags().UintVar(&timeout, "timeout", defaultCheckTimeoutSeconds, "Time out after this many seconds") + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Only wait for an application in namespace") command.Flags().StringVarP(&output, "output", "o", "wide", "Output format. One of: json|yaml|wide|tree|tree=detailed") return command } @@ -1698,6 +1746,7 @@ func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co diffChangesConfirm bool projects []string output string + appNamespace string ) var command = &cobra.Command{ Use: "sync [APPNAME... | -l selector | --project project-name]", @@ -1742,7 +1791,10 @@ func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co appNames := args if selector != "" || len(projects) > 0 { - list, err := appIf.List(ctx, &application.ApplicationQuery{Selector: pointer.String(selector), Projects: projects}) + list, err := appIf.List(ctx, &application.ApplicationQuery{ + Selector: pointer.String(selector), + AppNamespace: &appNamespace, + Projects: projects}) errors.CheckError(err) // unlike list, we'd want to fail if nothing was found @@ -1763,6 +1815,10 @@ func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co } for _, appQualifiedName := range appNames { + // Construct QualifiedName + if appNamespace != "" && !strings.Contains(appQualifiedName, "/") { + appQualifiedName = appNamespace + "/" + appQualifiedName + } appName, appNs := argo.ParseFromQualifiedName(appQualifiedName, "") if len(selectedLabels) > 0 { @@ -1980,6 +2036,7 @@ func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co command.Flags().BoolVar(&diffChanges, "preview-changes", false, "Preview difference against the target and live state before syncing app and wait for user confirmation") command.Flags().StringArrayVar(&projects, "project", []string{}, "Sync apps that belong to the specified projects. This option may be specified repeatedly.") command.Flags().StringVarP(&output, "output", "o", "wide", "Output format. One of: json|yaml|wide|tree|tree=detailed") + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Only sync an application in namespace") return command } @@ -1995,7 +2052,7 @@ func getAppNamesBySelector(ctx context.Context, appIf application.ApplicationSer return []string{}, fmt.Errorf("no apps match selector %v", selector) } for _, i := range list.Items { - appNames = append(appNames, i.Name) + appNames = append(appNames, i.QualifiedName()) } } return appNames, nil @@ -2131,6 +2188,9 @@ func groupResourceStates(app *argoappv1.Application, selectedResources []*argoap // check if resource health, sync and operation statuses matches watch options func checkResourceStatus(watch watchOpts, healthStatus string, syncStatus string, operationStatus *argoappv1.Operation) bool { + if watch.delete { + return false + } healthCheckPassed := true if watch.suspended && watch.health && watch.degraded { @@ -2283,6 +2343,12 @@ func waitOnApplicationStatus(ctx context.Context, acdClient argocdclient.Client, finalOperationState = app.Status.OperationState operationInProgress := false + + if watch.delete && appEvent.Type == k8swatch.Deleted { + fmt.Printf("Application '%s' deleted\n", app.QualifiedName()) + return nil, nil, nil + } + // consider the operation is in progress if app.Operation != nil { // if it just got requested @@ -2407,7 +2473,8 @@ func printApplicationHistoryTable(revHistory []argoappv1.RevisionHistory) { // NewApplicationHistoryCommand returns a new instance of an `argocd app history` command func NewApplicationHistoryCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { var ( - output string + output string + appNamespace string ) var command = &cobra.Command{ Use: "history APPNAME", @@ -2421,7 +2488,7 @@ func NewApplicationHistoryCommand(clientOpts *argocdclient.ClientOptions) *cobra } conn, appIf := headless.NewClientOrDie(clientOpts, c).NewApplicationClientOrDie() defer argoio.Close(conn) - appName, appNs := argo.ParseFromQualifiedName(args[0], "") + appName, appNs := argo.ParseFromQualifiedName(args[0], appNamespace) app, err := appIf.Get(ctx, &application.ApplicationQuery{ Name: &appName, AppNamespace: &appNs, @@ -2435,6 +2502,7 @@ func NewApplicationHistoryCommand(clientOpts *argocdclient.ClientOptions) *cobra } }, } + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Only show application deployment history in namespace") command.Flags().StringVarP(&output, "output", "o", "wide", "Output format. One of: wide|id") return command } @@ -2459,9 +2527,10 @@ func findRevisionHistory(application *argoappv1.Application, historyId int64) (* // NewApplicationRollbackCommand returns a new instance of an `argocd app rollback` command func NewApplicationRollbackCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { var ( - prune bool - timeout uint - output string + prune bool + timeout uint + output string + appNamespace string ) var command = &cobra.Command{ Use: "rollback APPNAME [ID]", @@ -2472,7 +2541,7 @@ func NewApplicationRollbackCommand(clientOpts *argocdclient.ClientOptions) *cobr c.HelpFunc()(c, args) os.Exit(1) } - appName, appNs := argo.ParseFromQualifiedName(args[0], "") + appName, appNs := argo.ParseFromQualifiedName(args[0], appNamespace) var err error depID := -1 if len(args) > 1 { @@ -2508,6 +2577,7 @@ func NewApplicationRollbackCommand(clientOpts *argocdclient.ClientOptions) *cobr command.Flags().BoolVar(&prune, "prune", false, "Allow deleting unexpected resources") command.Flags().UintVar(&timeout, "timeout", defaultCheckTimeoutSeconds, "Time out after this many seconds") command.Flags().StringVarP(&output, "output", "o", "wide", "Output format. One of: json|yaml|wide|tree|tree=detailed") + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Rollback application in namespace") return command } @@ -2520,7 +2590,11 @@ func printOperationResult(opState *argoappv1.OperationState) { } if opState.SyncResult != nil { fmt.Printf(printOpFmtStr, "Operation:", "Sync") - fmt.Printf(printOpFmtStr, "Sync Revision:", opState.SyncResult.Revision) + if opState.SyncResult.Sources != nil && opState.SyncResult.Revisions != nil { + fmt.Printf(printOpFmtStr, "Sync Revision:", strings.Join(opState.SyncResult.Revisions, ", ")) + } else { + fmt.Printf(printOpFmtStr, "Sync Revision:", opState.SyncResult.Revision) + } } fmt.Printf(printOpFmtStr, "Phase:", opState.Phase) fmt.Printf(printOpFmtStr, "Start:", opState.StartedAt) @@ -2653,6 +2727,7 @@ func NewApplicationTerminateOpCommand(clientOpts *argocdclient.ClientOptions) *c } func NewApplicationEditCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { + var appNamespace string var command = &cobra.Command{ Use: "edit APPNAME", Short: "Edit application", @@ -2663,7 +2738,7 @@ func NewApplicationEditCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co c.HelpFunc()(c, args) os.Exit(1) } - appName, appNs := argo.ParseFromQualifiedName(args[0], "") + appName, appNs := argo.ParseFromQualifiedName(args[0], appNamespace) conn, appIf := headless.NewClientOrDie(clientOpts, c).NewApplicationClientOrDie() defer argoio.Close(conn) app, err := appIf.Get(ctx, &application.ApplicationQuery{ @@ -2703,12 +2778,16 @@ func NewApplicationEditCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co }) }, } + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Only edit application in namespace") return command } func NewApplicationPatchCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { - var patch string - var patchType string + var ( + patch string + patchType string + appNamespace string + ) command := cobra.Command{ Use: "patch APPNAME", @@ -2725,7 +2804,7 @@ func NewApplicationPatchCommand(clientOpts *argocdclient.ClientOptions) *cobra.C c.HelpFunc()(c, args) os.Exit(1) } - appName, appNs := argo.ParseFromQualifiedName(args[0], "") + appName, appNs := argo.ParseFromQualifiedName(args[0], appNamespace) conn, appIf := headless.NewClientOrDie(clientOpts, c).NewApplicationClientOrDie() defer argoio.Close(conn) @@ -2743,8 +2822,129 @@ func NewApplicationPatchCommand(clientOpts *argocdclient.ClientOptions) *cobra.C fmt.Println(string(yamlBytes)) }, } - + command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Only patch application in namespace") command.Flags().StringVar(&patch, "patch", "", "Patch body") command.Flags().StringVar(&patchType, "type", "json", "The type of patch being provided; one of [json merge]") return &command } + +// NewApplicationAddSourceCommand returns a new instance of an `argocd app add-source` command +func NewApplicationAddSourceCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { + var ( + appOpts cmdutil.AppOptions + ) + var command = &cobra.Command{ + Use: "add-source APPNAME", + Short: "Adds a source to the list of sources in the application", + Example: ` # Append a source to the list of sources in the application + argocd app add-source guestbook --repo https://github.com/argoproj/argocd-example-apps.git --path guestbook`, + Run: func(c *cobra.Command, args []string) { + ctx := c.Context() + if len(args) != 1 { + c.HelpFunc()(c, args) + os.Exit(1) + } + + argocdClient := headless.NewClientOrDie(clientOpts, c) + conn, appIf := argocdClient.NewApplicationClientOrDie() + defer argoio.Close(conn) + + appName, appNs := argo.ParseFromQualifiedName(args[0], "") + + app, err := appIf.Get(ctx, &application.ApplicationQuery{ + Name: &appName, + Refresh: getRefreshType(false, false), + AppNamespace: &appNs, + }) + + errors.CheckError(err) + + if c.Flags() == nil { + errors.CheckError(fmt.Errorf("ApplicationSource needs atleast repoUrl, path or chart or ref field. No source to add.")) + } + + if len(app.Spec.Sources) > 0 { + appSource, _ := cmdutil.ConstructSource(&argoappv1.ApplicationSource{}, appOpts, c.Flags()) + + app.Spec.Sources = append(app.Spec.Sources, *appSource) + + _, err = appIf.UpdateSpec(ctx, &application.ApplicationUpdateSpecRequest{ + Name: &app.Name, + Spec: &app.Spec, + Validate: &appOpts.Validate, + AppNamespace: &appNs, + }) + errors.CheckError(err) + + fmt.Printf("Application '%s' updated successfully\n", app.ObjectMeta.Name) + } else { + errors.CheckError(fmt.Errorf("Cannot add source: application %s does not have spec.sources defined", appName)) + } + }, + } + cmdutil.AddAppFlags(command, &appOpts) + return command +} + +// NewApplicationRemoveSourceCommand returns a new instance of an `argocd app remove-source` command +func NewApplicationRemoveSourceCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { + var ( + source_index int + ) + command := &cobra.Command{ + Use: "remove-source APPNAME", + Short: "Remove a source from multiple sources application. Index starts with 0.", + Example: ` # Remove the source at index 1 from application's sources + argocd app remove-source myapplication --source-index 1`, + Run: func(c *cobra.Command, args []string) { + ctx := c.Context() + + if len(args) != 1 { + c.HelpFunc()(c, args) + os.Exit(1) + } + + if source_index < 0 { + errors.CheckError(fmt.Errorf("Index value of source cannot be less than 0")) + } + + argocdClient := headless.NewClientOrDie(clientOpts, c) + conn, appIf := argocdClient.NewApplicationClientOrDie() + defer argoio.Close(conn) + + appName, appNs := argo.ParseFromQualifiedName(args[0], "") + + app, err := appIf.Get(ctx, &application.ApplicationQuery{ + Name: &appName, + Refresh: getRefreshType(false, false), + AppNamespace: &appNs, + }) + errors.CheckError(err) + + if !app.Spec.HasMultipleSources() { + errors.CheckError(fmt.Errorf("Application does not have multiple sources configured")) + } + + if len(app.Spec.GetSources()) == 1 { + errors.CheckError(fmt.Errorf("Cannot remove the only source remaining in the app")) + } + + if len(app.Spec.GetSources()) < source_index { + errors.CheckError(fmt.Errorf("Application does not have source at %d\n", source_index)) + } + + app.Spec.Sources = append(app.Spec.Sources[:source_index], app.Spec.Sources[source_index+1:]...) + + _, err = appIf.UpdateSpec(ctx, &application.ApplicationUpdateSpecRequest{ + Name: &app.Name, + Spec: &app.Spec, + AppNamespace: &appNs, + }) + errors.CheckError(err) + + fmt.Printf("Application '%s' updated successfully\n", app.ObjectMeta.Name) + }, + } + command.Flags().IntVar(&source_index, "source-index", -1, "Index of the source from the list of sources of the app. Index starts from 0.") + return command +} diff --git a/cmd/argocd/commands/app_test.go b/cmd/argocd/commands/app_test.go index 68983560999c8..5217604d26987 100644 --- a/cmd/argocd/commands/app_test.go +++ b/cmd/argocd/commands/app_test.go @@ -1,23 +1,43 @@ package commands import ( + "context" "fmt" + "io" + "net/http" "os" "testing" "time" argocdclient "github.com/argoproj/argo-cd/v2/pkg/apiclient" + accountpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/account" + applicationpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + applicationsetpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/applicationset" + certificatepkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/certificate" + clusterpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/cluster" + gpgkeypkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/gpgkey" + notificationpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/notification" + projectpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/project" + repocredspkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/repocreds" + repositorypkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/repository" + sessionpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/session" + settingspkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/settings" + versionpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/version" "github.com/argoproj/argo-cd/v2/pkg/apis/application" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/argoproj/gitops-engine/pkg/health" "github.com/argoproj/gitops-engine/pkg/utils/kube" + "github.com/coreos/go-oidc/v3/oidc" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" + "golang.org/x/oauth2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + k8swatch "k8s.io/apimachinery/pkg/watch" ) func Test_getInfos(t *testing.T) { @@ -639,11 +659,110 @@ Project: default Server: local Namespace: argocd URL: url -Repo: test -Target: master -Path: /test -Helm Values: path1,path2 -Name Prefix: prefix +Source: +- Repo: test + Target: master + Path: /test + Helm Values: path1,path2 + Name Prefix: prefix +SyncWindow: Sync Denied +Assigned Windows: allow:0 0 * * *:24h,deny:0 0 * * *:24h,allow:0 0 * * *:24h +Sync Policy: Automated (Prune) +Sync Status: OutOfSync from master +Health Status: Progressing (health-message) +` + assert.Equalf(t, expectation, output, "Incorrect print app summary output %q, should be %q", output, expectation) +} + +func TestPrintAppSummaryTable_MultipleSources(t *testing.T) { + output, _ := captureOutput(func() error { + app := &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "argocd", + }, + Spec: v1alpha1.ApplicationSpec{ + SyncPolicy: &v1alpha1.SyncPolicy{ + Automated: &v1alpha1.SyncPolicyAutomated{ + Prune: true, + }, + }, + Project: "default", + Destination: v1alpha1.ApplicationDestination{Server: "local", Namespace: "argocd"}, + Sources: v1alpha1.ApplicationSources{ + { + RepoURL: "test", + TargetRevision: "master", + Path: "/test", + Helm: &v1alpha1.ApplicationSourceHelm{ + ValueFiles: []string{"path1", "path2"}, + }, + Kustomize: &v1alpha1.ApplicationSourceKustomize{NamePrefix: "prefix"}, + }, { + RepoURL: "test2", + TargetRevision: "master2", + Path: "/test2", + }, + }, + }, + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Status: v1alpha1.SyncStatusCodeOutOfSync, + }, + Health: v1alpha1.HealthStatus{ + Status: health.HealthStatusProgressing, + Message: "health-message", + }, + }, + } + + windows := &v1alpha1.SyncWindows{ + { + Kind: "allow", + Schedule: "0 0 * * *", + Duration: "24h", + Applications: []string{ + "*-prod", + }, + ManualSync: true, + }, + { + Kind: "deny", + Schedule: "0 0 * * *", + Duration: "24h", + Namespaces: []string{ + "default", + }, + }, + { + Kind: "allow", + Schedule: "0 0 * * *", + Duration: "24h", + Clusters: []string{ + "in-cluster", + "cluster1", + }, + }, + } + + printAppSummaryTable(app, "url", windows) + return nil + }) + + expectation := `Name: argocd/test +Project: default +Server: local +Namespace: argocd +URL: url +Sources: +- Repo: test + Target: master + Path: /test + Helm Values: path1,path2 + Name Prefix: prefix +- Repo: test2 + Target: master2 + Path: /test2 SyncWindow: Sync Denied Assigned Windows: allow:0 0 * * *:24h,deny:0 0 * * *:24h,allow:0 0 * * *:24h Sync Policy: Automated (Prune) @@ -806,6 +925,14 @@ func TestTargetObjects_invalid(t *testing.T) { assert.Error(t, err) } +func TestCheckForDeleteEvent(t *testing.T) { + + ctx := context.Background() + fakeClient := new(fakeAcdClient) + + checkForDeleteEvent(ctx, fakeClient, "testApp") +} + func TestPrintApplicationNames(t *testing.T) { output, _ := captureOutput(func() error { app := &v1alpha1.Application{ @@ -1599,3 +1726,104 @@ func testApp(name, project string, labels map[string]string, annotations map[str }, } } + +type fakeAcdClient struct{} + +func (c *fakeAcdClient) ClientOptions() argocdclient.ClientOptions { + return argocdclient.ClientOptions{} +} +func (c *fakeAcdClient) HTTPClient() (*http.Client, error) { return nil, nil } +func (c *fakeAcdClient) OIDCConfig(context.Context, *settingspkg.Settings) (*oauth2.Config, *oidc.Provider, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewRepoClient() (io.Closer, repositorypkg.RepositoryServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewRepoClientOrDie() (io.Closer, repositorypkg.RepositoryServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewRepoCredsClient() (io.Closer, repocredspkg.RepoCredsServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewRepoCredsClientOrDie() (io.Closer, repocredspkg.RepoCredsServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewCertClient() (io.Closer, certificatepkg.CertificateServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewCertClientOrDie() (io.Closer, certificatepkg.CertificateServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewClusterClient() (io.Closer, clusterpkg.ClusterServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewClusterClientOrDie() (io.Closer, clusterpkg.ClusterServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewGPGKeyClient() (io.Closer, gpgkeypkg.GPGKeyServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewGPGKeyClientOrDie() (io.Closer, gpgkeypkg.GPGKeyServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewApplicationClient() (io.Closer, applicationpkg.ApplicationServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewApplicationSetClient() (io.Closer, applicationsetpkg.ApplicationSetServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewApplicationClientOrDie() (io.Closer, applicationpkg.ApplicationServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewApplicationSetClientOrDie() (io.Closer, applicationsetpkg.ApplicationSetServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewNotificationClient() (io.Closer, notificationpkg.NotificationServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewNotificationClientOrDie() (io.Closer, notificationpkg.NotificationServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewSessionClient() (io.Closer, sessionpkg.SessionServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewSessionClientOrDie() (io.Closer, sessionpkg.SessionServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewSettingsClient() (io.Closer, settingspkg.SettingsServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewSettingsClientOrDie() (io.Closer, settingspkg.SettingsServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewVersionClient() (io.Closer, versionpkg.VersionServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewVersionClientOrDie() (io.Closer, versionpkg.VersionServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewProjectClient() (io.Closer, projectpkg.ProjectServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewProjectClientOrDie() (io.Closer, projectpkg.ProjectServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) NewAccountClient() (io.Closer, accountpkg.AccountServiceClient, error) { + return nil, nil, nil +} +func (c *fakeAcdClient) NewAccountClientOrDie() (io.Closer, accountpkg.AccountServiceClient) { + return nil, nil +} +func (c *fakeAcdClient) WatchApplicationWithRetry(ctx context.Context, appName string, revision string) chan *v1alpha1.ApplicationWatchEvent { + appEventsCh := make(chan *v1alpha1.ApplicationWatchEvent) + + go func() { + modifiedEvent := new(v1alpha1.ApplicationWatchEvent) + modifiedEvent.Type = k8swatch.Modified + appEventsCh <- modifiedEvent + deletedEvent := new(v1alpha1.ApplicationWatchEvent) + deletedEvent.Type = k8swatch.Deleted + appEventsCh <- deletedEvent + }() + return appEventsCh +} diff --git a/cmd/argocd/commands/applicationset.go b/cmd/argocd/commands/applicationset.go index b38f8837598fb..f5ed6a15b6208 100644 --- a/cmd/argocd/commands/applicationset.go +++ b/cmd/argocd/commands/applicationset.go @@ -350,9 +350,11 @@ func printAppSetSummaryTable(appSet *arogappsetv1.ApplicationSet) { fmt.Printf(printOpFmtStr, "Project:", appSet.Spec.Template.Spec.GetProject()) fmt.Printf(printOpFmtStr, "Server:", getServerForAppSet(appSet)) fmt.Printf(printOpFmtStr, "Namespace:", appSet.Spec.Template.Spec.Destination.Namespace) - fmt.Printf(printOpFmtStr, "Repo:", source.RepoURL) - fmt.Printf(printOpFmtStr, "Target:", source.TargetRevision) - fmt.Printf(printOpFmtStr, "Path:", source.Path) + if !appSet.Spec.Template.Spec.HasMultipleSources() { + fmt.Println("Source:") + } else { + fmt.Println("Sources:") + } printAppSourceDetails(&source) var ( diff --git a/cmd/argocd/commands/applicationset_test.go b/cmd/argocd/commands/applicationset_test.go index 18e5f85feebbc..7740c95a4e63b 100644 --- a/cmd/argocd/commands/applicationset_test.go +++ b/cmd/argocd/commands/applicationset_test.go @@ -180,9 +180,9 @@ func TestPrintAppSetSummaryTable(t *testing.T) { Project: default Server: Namespace: -Repo: -Target: -Path: +Source: +- Repo: + Target: SyncPolicy: `, }, @@ -193,9 +193,9 @@ SyncPolicy: Project: default Server: Namespace: -Repo: -Target: -Path: +Source: +- Repo: + Target: SyncPolicy: Automated `, }, @@ -206,9 +206,9 @@ SyncPolicy: Automated Project: default Server: Namespace: -Repo: -Target: -Path: +Source: +- Repo: + Target: SyncPolicy: Automated `, }, diff --git a/cmd/argocd/commands/cluster.go b/cmd/argocd/commands/cluster.go index 3df4be6632d85..f203b82ae9ac0 100644 --- a/cmd/argocd/commands/cluster.go +++ b/cmd/argocd/commands/cluster.go @@ -111,6 +111,7 @@ func NewClusterAddCommand(clientOpts *argocdclient.ClientOptions, pathOpts *clie awsAuthConf = &argoappv1.AWSAuthConfig{ ClusterName: clusterOpts.AwsClusterName, RoleARN: clusterOpts.AwsRoleArn, + Profile: clusterOpts.AwsProfile, } } else if clusterOpts.ExecProviderCommand != "" { execProviderConf = &argoappv1.ExecProviderConfig{ diff --git a/cmd/argocd/commands/project.go b/cmd/argocd/commands/project.go index 32fb9e779e8ed..be7517b843375 100644 --- a/cmd/argocd/commands/project.go +++ b/cmd/argocd/commands/project.go @@ -78,6 +78,8 @@ func NewProjectCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { command.AddCommand(NewProjectWindowsCommand(clientOpts)) command.AddCommand(NewProjectAddOrphanedIgnoreCommand(clientOpts)) command.AddCommand(NewProjectRemoveOrphanedIgnoreCommand(clientOpts)) + command.AddCommand(NewProjectAddSourceNamespace(clientOpts)) + command.AddCommand(NewProjectRemoveSourceNamespace(clientOpts)) return command } @@ -509,6 +511,88 @@ func NewProjectAddSourceCommand(clientOpts *argocdclient.ClientOptions) *cobra.C return command } +// NewProjectAddSourceNamespace returns a new instance of an `argocd proj add-source-namespace` command +func NewProjectAddSourceNamespace(clientOpts *argocdclient.ClientOptions) *cobra.Command { + var command = &cobra.Command{ + Use: "add-source-namespace PROJECT NAMESPACE", + Short: "Add source namespace to the AppProject", + Example: templates.Examples(` + # Add Kubernetes namespace as source namespace to the AppProject where application resources are allowed to be created in. + argocd proj add-source-namespace PROJECT NAMESPACE + `), + Run: func(c *cobra.Command, args []string) { + ctx := c.Context() + + if len(args) != 2 { + c.HelpFunc()(c, args) + os.Exit(1) + } + projName := args[0] + srcNamespace := args[1] + conn, projIf := headless.NewClientOrDie(clientOpts, c).NewProjectClientOrDie() + defer argoio.Close(conn) + + proj, err := projIf.Get(ctx, &projectpkg.ProjectQuery{Name: projName}) + errors.CheckError(err) + + for _, item := range proj.Spec.SourceNamespaces { + if item == "*" || item == srcNamespace { + fmt.Printf("Source namespace '*' already allowed in project\n") + return + } + } + proj.Spec.SourceNamespaces = append(proj.Spec.SourceNamespaces, srcNamespace) + _, err = projIf.Update(ctx, &projectpkg.ProjectUpdateRequest{Project: proj}) + errors.CheckError(err) + }, + } + return command +} + +// NewProjectRemoveSourceNamespace returns a new instance of an `argocd proj remove-source-namespace` command +func NewProjectRemoveSourceNamespace(clientOpts *argocdclient.ClientOptions) *cobra.Command { + var command = &cobra.Command{ + Use: "remove-source-namespace PROJECT NAMESPACE", + Short: "Removes the source namespace from the AppProject", + Example: templates.Examples(` + # Remove source NAMESPACE in PROJECT + argocd proj remove-source-namespace PROJECT NAMESPACE + `), + Run: func(c *cobra.Command, args []string) { + ctx := c.Context() + + if len(args) != 2 { + c.HelpFunc()(c, args) + os.Exit(1) + } + projName := args[0] + srcNamespace := args[1] + conn, projIf := headless.NewClientOrDie(clientOpts, c).NewProjectClientOrDie() + defer argoio.Close(conn) + + proj, err := projIf.Get(ctx, &projectpkg.ProjectQuery{Name: projName}) + errors.CheckError(err) + + index := -1 + for i, item := range proj.Spec.SourceNamespaces { + if item == srcNamespace && item != "*" { + index = i + break + } + } + if index == -1 { + fmt.Printf("Source namespace '%s' does not exist in project or cannot be removed\n", srcNamespace) + } else { + proj.Spec.SourceNamespaces = append(proj.Spec.SourceNamespaces[:index], proj.Spec.SourceNamespaces[index+1:]...) + _, err = projIf.Update(ctx, &projectpkg.ProjectUpdateRequest{Project: proj}) + errors.CheckError(err) + } + }, + } + + return command +} + func modifyResourcesList(list *[]metav1.GroupKind, add bool, listDesc string, group string, kind string) bool { if add { for _, item := range *list { diff --git a/cmd/util/app.go b/cmd/util/app.go index e08ee80305c48..307b4badd94eb 100644 --- a/cmd/util/app.go +++ b/cmd/util/app.go @@ -68,6 +68,7 @@ type AppOptions struct { kustomizeVersion string kustomizeCommonLabels []string kustomizeCommonAnnotations []string + kustomizeLabelWithoutSelector bool kustomizeForceCommonLabels bool kustomizeForceCommonAnnotations bool kustomizeNamespace string @@ -79,6 +80,7 @@ type AppOptions struct { retryBackoffDuration time.Duration retryBackoffMaxDuration time.Duration retryBackoffFactor int64 + ref string } func AddAppFlags(command *cobra.Command, opts *AppOptions) { @@ -124,6 +126,7 @@ func AddAppFlags(command *cobra.Command, opts *AppOptions) { command.Flags().BoolVar(&opts.Validate, "validate", true, "Validation of repo and cluster") command.Flags().StringArrayVar(&opts.kustomizeCommonLabels, "kustomize-common-label", []string{}, "Set common labels in Kustomize") command.Flags().StringArrayVar(&opts.kustomizeCommonAnnotations, "kustomize-common-annotation", []string{}, "Set common labels in Kustomize") + command.Flags().BoolVar(&opts.kustomizeLabelWithoutSelector, "kustomize-label-without-selector", false, "Do not apply common label to selectors or templates") command.Flags().BoolVar(&opts.kustomizeForceCommonLabels, "kustomize-force-common-label", false, "Force common labels in Kustomize") command.Flags().BoolVar(&opts.kustomizeForceCommonAnnotations, "kustomize-force-common-annotation", false, "Force common annotations in Kustomize") command.Flags().StringVar(&opts.kustomizeNamespace, "kustomize-namespace", "", "Kustomize namespace") @@ -133,6 +136,7 @@ func AddAppFlags(command *cobra.Command, opts *AppOptions) { command.Flags().DurationVar(&opts.retryBackoffDuration, "sync-retry-backoff-duration", argoappv1.DefaultSyncRetryDuration, "Sync retry backoff base duration. Input needs to be a duration (e.g. 2m, 1h)") command.Flags().DurationVar(&opts.retryBackoffMaxDuration, "sync-retry-backoff-max-duration", argoappv1.DefaultSyncRetryMaxDuration, "Max sync retry backoff duration. Input needs to be a duration (e.g. 2m, 1h)") command.Flags().Int64Var(&opts.retryBackoffFactor, "sync-retry-backoff-factor", argoappv1.DefaultSyncRetryFactor, "Factor multiplies the base duration after each failed sync retry") + command.Flags().StringVar(&opts.ref, "ref", "", "Ref is reference to another source within sources field") } func SetAppSpecOptions(flags *pflag.FlagSet, spec *argoappv1.ApplicationSpec, appOpts *AppOptions) int { @@ -140,74 +144,18 @@ func SetAppSpecOptions(flags *pflag.FlagSet, spec *argoappv1.ApplicationSpec, ap if flags == nil { return visited } + source := spec.GetSourcePtr() + if source == nil { + source = &argoappv1.ApplicationSource{} + } + source, visited = ConstructSource(source, *appOpts, flags) flags.Visit(func(f *pflag.Flag) { visited++ - source := spec.GetSourcePtr() - if source == nil { - source = &argoappv1.ApplicationSource{} - } + switch f.Name { - case "repo": - source.RepoURL = appOpts.repoURL - case "path": - source.Path = appOpts.appPath - case "helm-chart": - source.Chart = appOpts.chart - case "revision": - source.TargetRevision = appOpts.revision case "revision-history-limit": i := int64(appOpts.revisionHistoryLimit) spec.RevisionHistoryLimit = &i - case "values": - setHelmOpt(source, helmOpts{valueFiles: appOpts.valuesFiles}) - case "ignore-missing-value-files": - setHelmOpt(source, helmOpts{ignoreMissingValueFiles: appOpts.ignoreMissingValueFiles}) - case "values-literal-file": - var data []byte - - // read uri - parsedURL, err := url.ParseRequestURI(appOpts.values) - if err != nil || !(parsedURL.Scheme == "http" || parsedURL.Scheme == "https") { - data, err = os.ReadFile(appOpts.values) - } else { - data, err = config.ReadRemoteFile(appOpts.values) - } - errors.CheckError(err) - setHelmOpt(source, helmOpts{values: string(data)}) - case "release-name": - setHelmOpt(source, helmOpts{releaseName: appOpts.releaseName}) - case "helm-version": - setHelmOpt(source, helmOpts{version: appOpts.helmVersion}) - case "helm-pass-credentials": - setHelmOpt(source, helmOpts{passCredentials: appOpts.helmPassCredentials}) - case "helm-set": - setHelmOpt(source, helmOpts{helmSets: appOpts.helmSets}) - case "helm-set-string": - setHelmOpt(source, helmOpts{helmSetStrings: appOpts.helmSetStrings}) - case "helm-set-file": - setHelmOpt(source, helmOpts{helmSetFiles: appOpts.helmSetFiles}) - case "helm-skip-crds": - setHelmOpt(source, helmOpts{skipCrds: appOpts.helmSkipCrds}) - case "directory-recurse": - if source.Directory != nil { - source.Directory.Recurse = appOpts.directoryRecurse - } else { - source.Directory = &argoappv1.ApplicationSourceDirectory{Recurse: appOpts.directoryRecurse} - } - case "directory-exclude": - if source.Directory != nil { - source.Directory.Exclude = appOpts.directoryExclude - } else { - source.Directory = &argoappv1.ApplicationSourceDirectory{Exclude: appOpts.directoryExclude} - } - case "directory-include": - if source.Directory != nil { - source.Directory.Include = appOpts.directoryInclude - } else { - source.Directory = &argoappv1.ApplicationSourceDirectory{Include: appOpts.directoryInclude} - } - case "config-management-plugin": - source.Plugin = &argoappv1.ApplicationSourcePlugin{Name: appOpts.configManagementPlugin} case "dest-name": spec.Destination.Name = appOpts.destName case "dest-server": @@ -216,42 +164,6 @@ func SetAppSpecOptions(flags *pflag.FlagSet, spec *argoappv1.ApplicationSpec, ap spec.Destination.Namespace = appOpts.destNamespace case "project": spec.Project = appOpts.project - case "nameprefix": - setKustomizeOpt(source, kustomizeOpts{namePrefix: appOpts.namePrefix}) - case "namesuffix": - setKustomizeOpt(source, kustomizeOpts{nameSuffix: appOpts.nameSuffix}) - case "kustomize-image": - setKustomizeOpt(source, kustomizeOpts{images: appOpts.kustomizeImages}) - case "kustomize-replica": - setKustomizeOpt(source, kustomizeOpts{replicas: appOpts.kustomizeReplicas}) - case "kustomize-version": - setKustomizeOpt(source, kustomizeOpts{version: appOpts.kustomizeVersion}) - case "kustomize-namespace": - setKustomizeOpt(source, kustomizeOpts{namespace: appOpts.kustomizeNamespace}) - case "kustomize-common-label": - parsedLabels, err := label.Parse(appOpts.kustomizeCommonLabels) - errors.CheckError(err) - setKustomizeOpt(source, kustomizeOpts{commonLabels: parsedLabels}) - case "kustomize-common-annotation": - parsedAnnotations, err := label.Parse(appOpts.kustomizeCommonAnnotations) - errors.CheckError(err) - setKustomizeOpt(source, kustomizeOpts{commonAnnotations: parsedAnnotations}) - case "kustomize-force-common-label": - setKustomizeOpt(source, kustomizeOpts{forceCommonLabels: appOpts.kustomizeForceCommonLabels}) - case "kustomize-force-common-annotation": - setKustomizeOpt(source, kustomizeOpts{forceCommonAnnotations: appOpts.kustomizeForceCommonAnnotations}) - case "jsonnet-tla-str": - setJsonnetOpt(source, appOpts.jsonnetTlaStr, false) - case "jsonnet-tla-code": - setJsonnetOpt(source, appOpts.jsonnetTlaCode, true) - case "jsonnet-ext-var-str": - setJsonnetOptExtVar(source, appOpts.jsonnetExtVarStr, false) - case "jsonnet-ext-var-code": - setJsonnetOptExtVar(source, appOpts.jsonnetExtVarCode, true) - case "jsonnet-libs": - setJsonnetOptLibs(source, appOpts.jsonnetLibs) - case "plugin-env": - setPluginOptEnvs(source, appOpts.pluginEnvs) case "sync-policy": switch appOpts.syncPolicy { case "none": @@ -340,6 +252,7 @@ type kustomizeOpts struct { version string commonLabels map[string]string commonAnnotations map[string]string + labelWithoutSelector bool forceCommonLabels bool forceCommonAnnotations bool namespace string @@ -367,6 +280,9 @@ func setKustomizeOpt(src *argoappv1.ApplicationSource, opts kustomizeOpts) { if opts.commonAnnotations != nil { src.Kustomize.CommonAnnotations = opts.commonAnnotations } + if opts.labelWithoutSelector { + src.Kustomize.LabelWithoutSelector = opts.labelWithoutSelector + } if opts.forceCommonLabels { src.Kustomize.ForceCommonLabels = opts.forceCommonLabels } @@ -640,6 +556,7 @@ func constructAppsFromFileUrl(fileURL, appName string, labels, annotations, args if app.Name == "" { return nil, fmt.Errorf("app.Name is empty. --name argument can be used to provide app.Name") } + SetAppSpecOptions(flags, &app.Spec, &appOpts) SetParameterOverrides(app, appOpts.Parameters) mergeLabels(app, labels) @@ -654,9 +571,117 @@ func ConstructApps(fileURL, appName string, labels, annotations, args []string, } else if fileURL != "" { return constructAppsFromFileUrl(fileURL, appName, labels, annotations, args, appOpts, flags) } + return constructAppsBaseOnName(appName, labels, annotations, args, appOpts, flags) } +func ConstructSource(source *argoappv1.ApplicationSource, appOpts AppOptions, flags *pflag.FlagSet) (*argoappv1.ApplicationSource, int) { + visited := 0 + flags.Visit(func(f *pflag.Flag) { + visited++ + switch f.Name { + case "repo": + source.RepoURL = appOpts.repoURL + case "path": + source.Path = appOpts.appPath + case "helm-chart": + source.Chart = appOpts.chart + case "revision": + source.TargetRevision = appOpts.revision + case "values": + setHelmOpt(source, helmOpts{valueFiles: appOpts.valuesFiles}) + case "ignore-missing-value-files": + setHelmOpt(source, helmOpts{ignoreMissingValueFiles: appOpts.ignoreMissingValueFiles}) + case "values-literal-file": + var data []byte + // read uri + parsedURL, err := url.ParseRequestURI(appOpts.values) + if err != nil || !(parsedURL.Scheme == "http" || parsedURL.Scheme == "https") { + data, err = os.ReadFile(appOpts.values) + } else { + data, err = config.ReadRemoteFile(appOpts.values) + } + errors.CheckError(err) + setHelmOpt(source, helmOpts{values: string(data)}) + case "release-name": + setHelmOpt(source, helmOpts{releaseName: appOpts.releaseName}) + case "helm-version": + setHelmOpt(source, helmOpts{version: appOpts.helmVersion}) + case "helm-pass-credentials": + setHelmOpt(source, helmOpts{passCredentials: appOpts.helmPassCredentials}) + case "helm-set": + setHelmOpt(source, helmOpts{helmSets: appOpts.helmSets}) + case "helm-set-string": + setHelmOpt(source, helmOpts{helmSetStrings: appOpts.helmSetStrings}) + case "helm-set-file": + setHelmOpt(source, helmOpts{helmSetFiles: appOpts.helmSetFiles}) + case "helm-skip-crds": + setHelmOpt(source, helmOpts{skipCrds: appOpts.helmSkipCrds}) + case "directory-recurse": + if source.Directory != nil { + source.Directory.Recurse = appOpts.directoryRecurse + } else { + source.Directory = &argoappv1.ApplicationSourceDirectory{Recurse: appOpts.directoryRecurse} + } + case "directory-exclude": + if source.Directory != nil { + source.Directory.Exclude = appOpts.directoryExclude + } else { + source.Directory = &argoappv1.ApplicationSourceDirectory{Exclude: appOpts.directoryExclude} + } + case "directory-include": + if source.Directory != nil { + source.Directory.Include = appOpts.directoryInclude + } else { + source.Directory = &argoappv1.ApplicationSourceDirectory{Include: appOpts.directoryInclude} + } + case "config-management-plugin": + source.Plugin = &argoappv1.ApplicationSourcePlugin{Name: appOpts.configManagementPlugin} + case "nameprefix": + setKustomizeOpt(source, kustomizeOpts{namePrefix: appOpts.namePrefix}) + case "namesuffix": + setKustomizeOpt(source, kustomizeOpts{nameSuffix: appOpts.nameSuffix}) + case "kustomize-image": + setKustomizeOpt(source, kustomizeOpts{images: appOpts.kustomizeImages}) + case "kustomize-replica": + setKustomizeOpt(source, kustomizeOpts{replicas: appOpts.kustomizeReplicas}) + case "kustomize-version": + setKustomizeOpt(source, kustomizeOpts{version: appOpts.kustomizeVersion}) + case "kustomize-namespace": + setKustomizeOpt(source, kustomizeOpts{namespace: appOpts.kustomizeNamespace}) + case "kustomize-common-label": + parsedLabels, err := label.Parse(appOpts.kustomizeCommonLabels) + errors.CheckError(err) + setKustomizeOpt(source, kustomizeOpts{commonLabels: parsedLabels}) + case "kustomize-common-annotation": + parsedAnnotations, err := label.Parse(appOpts.kustomizeCommonAnnotations) + errors.CheckError(err) + setKustomizeOpt(source, kustomizeOpts{commonAnnotations: parsedAnnotations}) + case "kustomize-label-without-selector": + setKustomizeOpt(source, kustomizeOpts{labelWithoutSelector: appOpts.kustomizeLabelWithoutSelector}) + case "kustomize-force-common-label": + setKustomizeOpt(source, kustomizeOpts{forceCommonLabels: appOpts.kustomizeForceCommonLabels}) + case "kustomize-force-common-annotation": + setKustomizeOpt(source, kustomizeOpts{forceCommonAnnotations: appOpts.kustomizeForceCommonAnnotations}) + case "jsonnet-tla-str": + setJsonnetOpt(source, appOpts.jsonnetTlaStr, false) + case "jsonnet-tla-code": + setJsonnetOpt(source, appOpts.jsonnetTlaCode, true) + case "jsonnet-ext-var-str": + setJsonnetOptExtVar(source, appOpts.jsonnetExtVarStr, false) + case "jsonnet-ext-var-code": + setJsonnetOptExtVar(source, appOpts.jsonnetExtVarCode, true) + case "jsonnet-libs": + setJsonnetOptLibs(source, appOpts.jsonnetLibs) + case "plugin-env": + setPluginOptEnvs(source, appOpts.pluginEnvs) + case "ref": + source.Ref = appOpts.ref + } + }) + return source, visited +} + func mergeLabels(app *argoappv1.Application, labels []string) { mapLabels, err := label.Parse(labels) errors.CheckError(err) diff --git a/cmd/util/app_test.go b/cmd/util/app_test.go index 2f49a3cc4c8c4..b5fce9c1e663e 100644 --- a/cmd/util/app_test.go +++ b/cmd/util/app_test.go @@ -123,6 +123,11 @@ func Test_setKustomizeOpt(t *testing.T) { setKustomizeOpt(&src, kustomizeOpts{commonAnnotations: map[string]string{"foo1": "bar1", "foo2": "bar2"}}) assert.Equal(t, &v1alpha1.ApplicationSourceKustomize{CommonAnnotations: map[string]string{"foo1": "bar1", "foo2": "bar2"}}, src.Kustomize) }) + t.Run("Label Without Selector", func(t *testing.T) { + src := v1alpha1.ApplicationSource{} + setKustomizeOpt(&src, kustomizeOpts{commonLabels: map[string]string{"foo1": "bar1", "foo2": "bar2"}, labelWithoutSelector: true}) + assert.Equal(t, &v1alpha1.ApplicationSourceKustomize{CommonLabels: map[string]string{"foo1": "bar1", "foo2": "bar2"}, LabelWithoutSelector: true}, src.Kustomize) + }) } func Test_setJsonnetOpt(t *testing.T) { diff --git a/cmd/util/cluster.go b/cmd/util/cluster.go index 95c071c882b12..dffb52e775a97 100644 --- a/cmd/util/cluster.go +++ b/cmd/util/cluster.go @@ -144,6 +144,7 @@ type ClusterOptions struct { Upsert bool ServiceAccount string AwsRoleArn string + AwsProfile string AwsClusterName string SystemNamespace string Namespaces []string @@ -169,6 +170,7 @@ func AddClusterFlags(command *cobra.Command, opts *ClusterOptions) { command.Flags().BoolVar(&opts.InCluster, "in-cluster", false, "Indicates Argo CD resides inside this cluster and should connect using the internal k8s hostname (kubernetes.default.svc)") command.Flags().StringVar(&opts.AwsClusterName, "aws-cluster-name", "", "AWS Cluster name if set then aws cli eks token command will be used to access cluster") command.Flags().StringVar(&opts.AwsRoleArn, "aws-role-arn", "", "Optional AWS role arn. If set then AWS IAM Authenticator assumes a role to perform cluster operations instead of the default AWS credential provider chain.") + command.Flags().StringVar(&opts.AwsProfile, "aws-profile", "", "Optional AWS profile. If set then AWS IAM Authenticator uses this profile to perform cluster operations instead of the default AWS credential provider chain.") command.Flags().StringArrayVar(&opts.Namespaces, "namespace", nil, "List of namespaces which are allowed to manage") command.Flags().BoolVar(&opts.ClusterResources, "cluster-resources", false, "Indicates if cluster level resources should be managed. The setting is used only if list of managed namespaces is not empty.") command.Flags().StringVar(&opts.Name, "name", "", "Overwrite the cluster name") diff --git a/common/common.go b/common/common.go index c5b9362f7f943..1d04d0e47eb65 100644 --- a/common/common.go +++ b/common/common.go @@ -115,9 +115,9 @@ const ( LegacyShardingAlgorithm = "legacy" // RoundRobinShardingAlgorithm is a flag value that can be opted for Sharding Algorithm it uses an equal distribution accross all shards RoundRobinShardingAlgorithm = "round-robin" - DefaultShardingAlgorithm = LegacyShardingAlgorithm // AppControllerHeartbeatUpdateRetryCount is the retry count for updating the Shard Mapping to the Shard Mapping ConfigMap used by Application Controller AppControllerHeartbeatUpdateRetryCount = 3 + DefaultShardingAlgorithm = LegacyShardingAlgorithm ) // Dex related constants @@ -149,10 +149,14 @@ const ( LabelKeyAppInstance = "app.kubernetes.io/instance" // LabelKeyAppName is the label key to use to uniquely identify the name of the Kubernetes application LabelKeyAppName = "app.kubernetes.io/name" + // LabelKeyAutoLabelClusterInfo if set to true will automatically add extra labels from the cluster info (currently it only adds a k8s version label) + LabelKeyAutoLabelClusterInfo = "argocd.argoproj.io/auto-label-cluster-info" // LabelKeyLegacyApplicationName is the legacy label (v0.10 and below) and is superseded by 'app.kubernetes.io/instance' LabelKeyLegacyApplicationName = "applications.argoproj.io/app-name" // LabelKeySecretType contains the type of argocd secret (currently: 'cluster', 'repository', 'repo-config' or 'repo-creds') LabelKeySecretType = "argocd.argoproj.io/secret-type" + // LabelKeyClusterKubernetesVersion contains the kubernetes version of the cluster secret if it has been enabled + LabelKeyClusterKubernetesVersion = "argocd.argoproj.io/kubernetes-version" // LabelValueSecretTypeCluster indicates a secret type of cluster LabelValueSecretTypeCluster = "cluster" // LabelValueSecretTypeRepository indicates a secret type of repository diff --git a/controller/appcontroller.go b/controller/appcontroller.go index 0ded95de65d15..9d89b6e6b37d6 100644 --- a/controller/appcontroller.go +++ b/controller/appcontroller.go @@ -6,6 +6,7 @@ import ( goerrors "errors" "fmt" "math" + "math/rand" "net/http" "reflect" "runtime/debug" @@ -47,7 +48,6 @@ import ( "github.com/argoproj/argo-cd/v2/controller/sharding" "github.com/argoproj/argo-cd/v2/pkg/apis/application" appv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - argov1alpha "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned" "github.com/argoproj/argo-cd/v2/pkg/client/informers/externalversions/application/v1alpha1" applisters "github.com/argoproj/argo-cd/v2/pkg/client/listers/application/v1alpha1" @@ -113,11 +113,11 @@ type ApplicationController struct { appInformer cache.SharedIndexInformer appLister applisters.ApplicationLister projInformer cache.SharedIndexInformer - deploymentInformer informerv1.DeploymentInformer appStateManager AppStateManager stateCache statecache.LiveStateCache statusRefreshTimeout time.Duration statusHardRefreshTimeout time.Duration + statusRefreshJitter time.Duration selfHealTimeout time.Duration repoClientset apiclient.Clientset db db.ArgoDB @@ -126,9 +126,13 @@ type ApplicationController struct { refreshRequestedAppsMutex *sync.Mutex metricsServer *metrics.MetricsServer kubectlSemaphore *semaphore.Weighted - clusterFilter func(cluster *appv1.Cluster) bool + clusterSharding sharding.ClusterShardingCache projByNameCache sync.Map applicationNamespaces []string + + // dynamicClusterDistributionEnabled if disabled deploymentInformer is never initialized + dynamicClusterDistributionEnabled bool + deploymentInformer informerv1.DeploymentInformer } // NewApplicationController creates new instance of ApplicationController. @@ -142,6 +146,7 @@ func NewApplicationController( kubectl kube.Kubectl, appResyncPeriod time.Duration, appHardResyncPeriod time.Duration, + appResyncJitter time.Duration, selfHealTimeout time.Duration, repoErrorGracePeriod time.Duration, metricsPort int, @@ -149,39 +154,42 @@ func NewApplicationController( metricsApplicationLabels []string, kubectlParallelismLimit int64, persistResourceHealth bool, - clusterFilter func(cluster *appv1.Cluster) bool, + clusterSharding sharding.ClusterShardingCache, applicationNamespaces []string, rateLimiterConfig *ratelimiter.AppControllerRateLimiterConfig, serverSideDiff bool, + dynamicClusterDistributionEnabled bool, ) (*ApplicationController, error) { - log.Infof("appResyncPeriod=%v, appHardResyncPeriod=%v", appResyncPeriod, appHardResyncPeriod) + log.Infof("appResyncPeriod=%v, appHardResyncPeriod=%v, appResyncJitter=%v", appResyncPeriod, appHardResyncPeriod, appResyncJitter) db := db.NewDB(namespace, settingsMgr, kubeClientset) if rateLimiterConfig == nil { rateLimiterConfig = ratelimiter.GetDefaultAppRateLimiterConfig() log.Info("Using default workqueue rate limiter config") } ctrl := ApplicationController{ - cache: argoCache, - namespace: namespace, - kubeClientset: kubeClientset, - kubectl: kubectl, - applicationClientset: applicationClientset, - repoClientset: repoClientset, - appRefreshQueue: workqueue.NewNamedRateLimitingQueue(ratelimiter.NewCustomAppControllerRateLimiter(rateLimiterConfig), "app_reconciliation_queue"), - appOperationQueue: workqueue.NewNamedRateLimitingQueue(ratelimiter.NewCustomAppControllerRateLimiter(rateLimiterConfig), "app_operation_processing_queue"), - projectRefreshQueue: workqueue.NewNamedRateLimitingQueue(ratelimiter.NewCustomAppControllerRateLimiter(rateLimiterConfig), "project_reconciliation_queue"), - appComparisonTypeRefreshQueue: workqueue.NewRateLimitingQueue(ratelimiter.NewCustomAppControllerRateLimiter(rateLimiterConfig)), - db: db, - statusRefreshTimeout: appResyncPeriod, - statusHardRefreshTimeout: appHardResyncPeriod, - refreshRequestedApps: make(map[string]CompareWith), - refreshRequestedAppsMutex: &sync.Mutex{}, - auditLogger: argo.NewAuditLogger(namespace, kubeClientset, common.ApplicationController), - settingsMgr: settingsMgr, - selfHealTimeout: selfHealTimeout, - clusterFilter: clusterFilter, - projByNameCache: sync.Map{}, - applicationNamespaces: applicationNamespaces, + cache: argoCache, + namespace: namespace, + kubeClientset: kubeClientset, + kubectl: kubectl, + applicationClientset: applicationClientset, + repoClientset: repoClientset, + appRefreshQueue: workqueue.NewNamedRateLimitingQueue(ratelimiter.NewCustomAppControllerRateLimiter(rateLimiterConfig), "app_reconciliation_queue"), + appOperationQueue: workqueue.NewNamedRateLimitingQueue(ratelimiter.NewCustomAppControllerRateLimiter(rateLimiterConfig), "app_operation_processing_queue"), + projectRefreshQueue: workqueue.NewNamedRateLimitingQueue(ratelimiter.NewCustomAppControllerRateLimiter(rateLimiterConfig), "project_reconciliation_queue"), + appComparisonTypeRefreshQueue: workqueue.NewRateLimitingQueue(ratelimiter.NewCustomAppControllerRateLimiter(rateLimiterConfig)), + db: db, + statusRefreshTimeout: appResyncPeriod, + statusHardRefreshTimeout: appHardResyncPeriod, + statusRefreshJitter: appResyncJitter, + refreshRequestedApps: make(map[string]CompareWith), + refreshRequestedAppsMutex: &sync.Mutex{}, + auditLogger: argo.NewAuditLogger(namespace, kubeClientset, common.ApplicationController), + settingsMgr: settingsMgr, + selfHealTimeout: selfHealTimeout, + clusterSharding: clusterSharding, + projByNameCache: sync.Map{}, + applicationNamespaces: applicationNamespaces, + dynamicClusterDistributionEnabled: dynamicClusterDistributionEnabled, } if kubectlParallelismLimit > 0 { ctrl.kubectlSemaphore = semaphore.NewWeighted(kubectlParallelismLimit) @@ -224,25 +232,33 @@ func NewApplicationController( } factory := informers.NewSharedInformerFactoryWithOptions(ctrl.kubeClientset, defaultDeploymentInformerResyncDuration, informers.WithNamespace(settingsMgr.GetNamespace())) - deploymentInformer := factory.Apps().V1().Deployments() + + var deploymentInformer informerv1.DeploymentInformer + + // only initialize deployment informer if dynamic distribution is enabled + if dynamicClusterDistributionEnabled { + deploymentInformer = factory.Apps().V1().Deployments() + } readinessHealthCheck := func(r *http.Request) error { - applicationControllerName := env.StringFromEnv(common.EnvAppControllerName, common.DefaultApplicationControllerName) - appControllerDeployment, err := deploymentInformer.Lister().Deployments(settingsMgr.GetNamespace()).Get(applicationControllerName) - if err != nil { - if kubeerrors.IsNotFound(err) { - appControllerDeployment = nil - } else { - return fmt.Errorf("error retrieving Application Controller Deployment: %s", err) - } - } - if appControllerDeployment != nil { - if appControllerDeployment.Spec.Replicas != nil && int(*appControllerDeployment.Spec.Replicas) <= 0 { - return fmt.Errorf("application controller deployment replicas is not set or is less than 0, replicas: %d", appControllerDeployment.Spec.Replicas) + if dynamicClusterDistributionEnabled { + applicationControllerName := env.StringFromEnv(common.EnvAppControllerName, common.DefaultApplicationControllerName) + appControllerDeployment, err := deploymentInformer.Lister().Deployments(settingsMgr.GetNamespace()).Get(applicationControllerName) + if err != nil { + if kubeerrors.IsNotFound(err) { + appControllerDeployment = nil + } else { + return fmt.Errorf("error retrieving Application Controller Deployment: %s", err) + } } - shard := env.ParseNumFromEnv(common.EnvControllerShard, -1, -math.MaxInt32, math.MaxInt32) - if _, err := sharding.GetOrUpdateShardFromConfigMap(kubeClientset.(*kubernetes.Clientset), settingsMgr, int(*appControllerDeployment.Spec.Replicas), shard); err != nil { - return fmt.Errorf("error while updating the heartbeat for to the Shard Mapping ConfigMap: %s", err) + if appControllerDeployment != nil { + if appControllerDeployment.Spec.Replicas != nil && int(*appControllerDeployment.Spec.Replicas) <= 0 { + return fmt.Errorf("application controller deployment replicas is not set or is less than 0, replicas: %d", appControllerDeployment.Spec.Replicas) + } + shard := env.ParseNumFromEnv(common.EnvControllerShard, -1, -math.MaxInt32, math.MaxInt32) + if _, err := sharding.GetOrUpdateShardFromConfigMap(kubeClientset.(*kubernetes.Clientset), settingsMgr, int(*appControllerDeployment.Spec.Replicas), shard); err != nil { + return fmt.Errorf("error while updating the heartbeat for to the Shard Mapping ConfigMap: %s", err) + } } } return nil @@ -260,7 +276,7 @@ func NewApplicationController( return nil, err } } - stateCache := statecache.NewLiveStateCache(db, appInformer, ctrl.settingsMgr, kubectl, ctrl.metricsServer, ctrl.handleObjectUpdated, clusterFilter, argo.NewResourceTracking()) + stateCache := statecache.NewLiveStateCache(db, appInformer, ctrl.settingsMgr, kubectl, ctrl.metricsServer, ctrl.handleObjectUpdated, clusterSharding, argo.NewResourceTracking()) appStateManager := NewAppStateManager(db, applicationClientset, repoClientset, namespace, kubectl, ctrl.settingsMgr, stateCache, projInformer, ctrl.metricsServer, argoCache, ctrl.statusRefreshTimeout, argo.NewResourceTracking(), persistResourceHealth, repoErrorGracePeriod, serverSideDiff) ctrl.appInformer = appInformer ctrl.appLister = appLister @@ -494,13 +510,13 @@ func (ctrl *ApplicationController) getResourceTree(a *appv1.Application, managed if err != nil { return nil, fmt.Errorf("failed to unmarshal live state of managed resources: %w", err) } - var target = &unstructured.Unstructured{} - err = json.Unmarshal([]byte(managedResource.TargetState), &target) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal target state of managed resources: %w", err) - } if live == nil { + var target = &unstructured.Unstructured{} + err = json.Unmarshal([]byte(managedResource.TargetState), &target) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal target state of managed resources: %w", err) + } nodes = append(nodes, appv1.ResourceNode{ ResourceRef: appv1.ResourceRef{ Version: target.GroupVersionKind().Version, @@ -770,7 +786,24 @@ func (ctrl *ApplicationController) Run(ctx context.Context, statusProcessors int go ctrl.appInformer.Run(ctx.Done()) go ctrl.projInformer.Run(ctx.Done()) - go ctrl.deploymentInformer.Informer().Run(ctx.Done()) + + if ctrl.dynamicClusterDistributionEnabled { + // only start deployment informer if dynamic distribution is enabled + go ctrl.deploymentInformer.Informer().Run(ctx.Done()) + } + + clusters, err := ctrl.db.ListClusters(ctx) + if err != nil { + log.Warnf("Cannot init sharding. Error while querying clusters list from database: %v", err) + } else { + appItems, err := ctrl.getAppList(metav1.ListOptions{}) + + if err != nil { + log.Warnf("Cannot init sharding. Error while querying application list from database: %v", err) + } else { + ctrl.clusterSharding.Init(clusters, appItems) + } + } errors.CheckError(ctrl.stateCache.Init()) @@ -1023,7 +1056,7 @@ func (ctrl *ApplicationController) getPermittedAppLiveObjects(app *appv1.Applica return objsMap, nil } -func (ctrl *ApplicationController) isValidDestination(app *appv1.Application) (bool, *argov1alpha.Cluster) { +func (ctrl *ApplicationController) isValidDestination(app *appv1.Application) (bool, *appv1.Cluster) { // Validate the cluster using the Application destination's `name` field, if applicable, // and set the Server field, if needed. if err := argo.ValidateDestination(context.Background(), &app.Spec.Destination, ctrl.db); err != nil { @@ -1636,6 +1669,7 @@ func (ctrl *ApplicationController) needRefreshAppStatus(app *appv1.Application, var reason string compareWith := CompareWithLatest refreshType := appv1.RefreshTypeNormal + softExpired := app.Status.ReconciledAt == nil || app.Status.ReconciledAt.Add(statusRefreshTimeout).Before(time.Now().UTC()) hardExpired := (app.Status.ReconciledAt == nil || app.Status.ReconciledAt.Add(statusHardRefreshTimeout).Before(time.Now().UTC())) && statusHardRefreshTimeout.Seconds() != 0 @@ -1976,15 +2010,11 @@ func (ctrl *ApplicationController) canProcessApp(obj interface{}) bool { } } - if ctrl.clusterFilter != nil { - cluster, err := ctrl.db.GetCluster(context.Background(), app.Spec.Destination.Server) - if err != nil { - return ctrl.clusterFilter(nil) - } - return ctrl.clusterFilter(cluster) + cluster, err := ctrl.db.GetCluster(context.Background(), app.Spec.Destination.Server) + if err != nil { + return ctrl.clusterSharding.IsManagedCluster(nil) } - - return true + return ctrl.clusterSharding.IsManagedCluster(cluster) } func (ctrl *ApplicationController) newApplicationInformerAndLister() (cache.SharedIndexInformer, applisters.ApplicationLister) { @@ -2082,6 +2112,10 @@ func (ctrl *ApplicationController) newApplicationInformerAndLister() (cache.Shar ctrl.appRefreshQueue.AddRateLimited(key) ctrl.appOperationQueue.AddRateLimited(key) } + newApp, newOK := obj.(*appv1.Application) + if err == nil && newOK { + ctrl.clusterSharding.AddApp(newApp) + } }, UpdateFunc: func(old, new interface{}) { if !ctrl.canProcessApp(new) { @@ -2092,15 +2126,27 @@ func (ctrl *ApplicationController) newApplicationInformerAndLister() (cache.Shar if err != nil { return } + var compareWith *CompareWith + var delay *time.Duration + oldApp, oldOK := old.(*appv1.Application) newApp, newOK := new.(*appv1.Application) - if oldOK && newOK && automatedSyncEnabled(oldApp, newApp) { - log.WithField("application", newApp.QualifiedName()).Info("Enabled automated sync") - compareWith = CompareWithLatest.Pointer() + if oldOK && newOK { + if automatedSyncEnabled(oldApp, newApp) { + log.WithField("application", newApp.QualifiedName()).Info("Enabled automated sync") + compareWith = CompareWithLatest.Pointer() + } + if ctrl.statusRefreshJitter != 0 && oldApp.ResourceVersion == newApp.ResourceVersion { + // Handler is refreshing the apps, add a random jitter to spread the load and avoid spikes + jitter := time.Duration(float64(ctrl.statusRefreshJitter) * rand.Float64()) + delay = &jitter + } } - ctrl.requestAppRefresh(newApp.QualifiedName(), compareWith, nil) + + ctrl.requestAppRefresh(newApp.QualifiedName(), compareWith, delay) ctrl.appOperationQueue.AddRateLimited(key) + ctrl.clusterSharding.UpdateApp(newApp) }, DeleteFunc: func(obj interface{}) { if !ctrl.canProcessApp(obj) { @@ -2113,6 +2159,10 @@ func (ctrl *ApplicationController) newApplicationInformerAndLister() (cache.Shar // for deletes, we immediately add to the refresh queue ctrl.appRefreshQueue.Add(key) } + delApp, delOK := obj.(*appv1.Application) + if err == nil && delOK { + ctrl.clusterSharding.DeleteApp(delApp) + } }, }, ) @@ -2136,7 +2186,7 @@ func (ctrl *ApplicationController) projectErrorToCondition(err error, app *appv1 } func (ctrl *ApplicationController) RegisterClusterSecretUpdater(ctx context.Context) { - updater := NewClusterInfoUpdater(ctrl.stateCache, ctrl.db, ctrl.appLister.Applications(""), ctrl.cache, ctrl.clusterFilter, ctrl.getAppProj, ctrl.namespace) + updater := NewClusterInfoUpdater(ctrl.stateCache, ctrl.db, ctrl.appLister.Applications(""), ctrl.cache, ctrl.clusterSharding.IsManagedCluster, ctrl.getAppProj, ctrl.namespace) go updater.Run(ctx) } @@ -2188,4 +2238,26 @@ func (ctrl *ApplicationController) toAppQualifiedName(appName, appNamespace stri return fmt.Sprintf("%s/%s", appNamespace, appName) } -type ClusterFilterFunction func(c *argov1alpha.Cluster, distributionFunction sharding.DistributionFunction) bool +func (ctrl *ApplicationController) getAppList(options metav1.ListOptions) (*appv1.ApplicationList, error) { + watchNamespace := ctrl.namespace + // If we have at least one additional namespace configured, we need to + // watch on them all. + if len(ctrl.applicationNamespaces) > 0 { + watchNamespace = "" + } + + appList, err := ctrl.applicationClientset.ArgoprojV1alpha1().Applications(watchNamespace).List(context.TODO(), options) + if err != nil { + return nil, err + } + newItems := []appv1.Application{} + for _, app := range appList.Items { + if ctrl.isAppNamespaceAllowed(&app) { + newItems = append(newItems, app) + } + } + appList.Items = newItems + return appList, nil +} + +type ClusterFilterFunction func(c *appv1.Cluster, distributionFunction sharding.DistributionFunction) bool diff --git a/controller/appcontroller_test.go b/controller/appcontroller_test.go index bf3d8bb3a2e4c..33a29bc5ca3f8 100644 --- a/controller/appcontroller_test.go +++ b/controller/appcontroller_test.go @@ -17,7 +17,9 @@ import ( "github.com/argoproj/argo-cd/v2/common" statecache "github.com/argoproj/argo-cd/v2/controller/cache" + "github.com/argoproj/argo-cd/v2/controller/sharding" + dbmocks "github.com/argoproj/argo-cd/v2/util/db/mocks" "github.com/argoproj/gitops-engine/pkg/cache/mocks" synccommon "github.com/argoproj/gitops-engine/pkg/sync/common" "github.com/argoproj/gitops-engine/pkg/utils/kube" @@ -142,6 +144,7 @@ func newFakeController(data *fakeData, repoErr error) *ApplicationController { kubectl, time.Minute, time.Hour, + time.Second, time.Minute, time.Second*10, common.DefaultPortArgoCDMetrics, @@ -152,8 +155,14 @@ func newFakeController(data *fakeData, repoErr error) *ApplicationController { nil, data.applicationNamespaces, nil, + + false, false, ) + db := &dbmocks.ArgoDB{} + db.On("GetApplicationControllerReplicas").Return(1) + // Setting a default sharding algorithm for the tests where we cannot set it. + ctrl.clusterSharding = sharding.NewClusterSharding(db, 0, 1, common.DefaultShardingAlgorithm) if err != nil { panic(err) } @@ -686,7 +695,6 @@ func TestFinalizeAppDeletion(t *testing.T) { ctrl := newFakeController(&fakeData{apps: []runtime.Object{app, &defaultProj}, managedLiveObjs: map[kube.ResourceKey]*unstructured.Unstructured{ kube.GetResourceKey(appObj): appObj, }}, nil) - patched := false fakeAppCs := ctrl.applicationClientset.(*appclientset.Clientset) defaultReactor := fakeAppCs.ReactionChain[0] @@ -1809,13 +1817,11 @@ func Test_canProcessApp(t *testing.T) { }) t.Run("with cluster filter, good namespace", func(t *testing.T) { app.Namespace = "good" - ctrl.clusterFilter = func(_ *v1alpha1.Cluster) bool { return true } canProcess := ctrl.canProcessApp(app) assert.True(t, canProcess) }) t.Run("with cluster filter, bad namespace", func(t *testing.T) { app.Namespace = "bad" - ctrl.clusterFilter = func(_ *v1alpha1.Cluster) bool { return true } canProcess := ctrl.canProcessApp(app) assert.False(t, canProcess) }) diff --git a/controller/cache/cache.go b/controller/cache/cache.go index 9eac161714089..d1ae8989cd8e6 100644 --- a/controller/cache/cache.go +++ b/controller/cache/cache.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/tools/cache" "github.com/argoproj/argo-cd/v2/controller/metrics" + "github.com/argoproj/argo-cd/v2/controller/sharding" "github.com/argoproj/argo-cd/v2/pkg/apis/application" appv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/argoproj/argo-cd/v2/util/argo" @@ -168,7 +169,7 @@ func NewLiveStateCache( kubectl kube.Kubectl, metricsServer *metrics.MetricsServer, onObjectUpdated ObjectUpdatedHandler, - clusterFilter func(cluster *appv1.Cluster) bool, + clusterSharding sharding.ClusterShardingCache, resourceTracking argo.ResourceTracking) LiveStateCache { return &liveStateCache{ @@ -179,7 +180,7 @@ func NewLiveStateCache( kubectl: kubectl, settingsMgr: settingsMgr, metricsServer: metricsServer, - clusterFilter: clusterFilter, + clusterSharding: clusterSharding, resourceTracking: resourceTracking, } } @@ -202,7 +203,7 @@ type liveStateCache struct { kubectl kube.Kubectl settingsMgr *settings.SettingsManager metricsServer *metrics.MetricsServer - clusterFilter func(cluster *appv1.Cluster) bool + clusterSharding sharding.ClusterShardingCache resourceTracking argo.ResourceTracking clusters map[string]clustercache.ClusterCache @@ -722,22 +723,24 @@ func (c *liveStateCache) Run(ctx context.Context) error { } func (c *liveStateCache) canHandleCluster(cluster *appv1.Cluster) bool { - if c.clusterFilter == nil { - return true - } - return c.clusterFilter(cluster) + return c.clusterSharding.IsManagedCluster(cluster) } func (c *liveStateCache) handleAddEvent(cluster *appv1.Cluster) { + c.clusterSharding.Add(cluster) if !c.canHandleCluster(cluster) { log.Infof("Ignoring cluster %s", cluster.Server) return } - c.lock.Lock() _, ok := c.clusters[cluster.Server] c.lock.Unlock() if !ok { + log.Debugf("Checking if cache %v / cluster %v has appInformer %v", c, cluster, c.appInformer) + if c.appInformer == nil { + log.Warn("Cannot get a cluster appInformer. Cache may not be started this time") + return + } if c.isClusterHasApps(c.appInformer.GetStore().List(), cluster) { go func() { // warm up cache for cluster with apps @@ -748,6 +751,7 @@ func (c *liveStateCache) handleAddEvent(cluster *appv1.Cluster) { } func (c *liveStateCache) handleModEvent(oldCluster *appv1.Cluster, newCluster *appv1.Cluster) { + c.clusterSharding.Update(oldCluster, newCluster) c.lock.Lock() cluster, ok := c.clusters[newCluster.Server] c.lock.Unlock() @@ -790,6 +794,7 @@ func (c *liveStateCache) handleModEvent(oldCluster *appv1.Cluster, newCluster *a func (c *liveStateCache) handleDeleteEvent(clusterServer string) { c.lock.RLock() + c.clusterSharding.Delete(clusterServer) cluster, ok := c.clusters[clusterServer] c.lock.RUnlock() if ok { diff --git a/controller/cache/cache_test.go b/controller/cache/cache_test.go index c94038a89b881..53a03ca81995e 100644 --- a/controller/cache/cache_test.go +++ b/controller/cache/cache_test.go @@ -21,7 +21,11 @@ import ( "github.com/stretchr/testify/mock" "k8s.io/client-go/kubernetes/fake" + "github.com/argoproj/argo-cd/v2/common" + "github.com/argoproj/argo-cd/v2/controller/metrics" + "github.com/argoproj/argo-cd/v2/controller/sharding" appv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + dbmocks "github.com/argoproj/argo-cd/v2/util/db/mocks" argosettings "github.com/argoproj/argo-cd/v2/util/settings" ) @@ -35,11 +39,13 @@ func TestHandleModEvent_HasChanges(t *testing.T) { clusterCache := &mocks.ClusterCache{} clusterCache.On("Invalidate", mock.Anything, mock.Anything).Return(nil).Once() clusterCache.On("EnsureSynced").Return(nil).Once() - + db := &dbmocks.ArgoDB{} + db.On("GetApplicationControllerReplicas").Return(1) clustersCache := liveStateCache{ clusters: map[string]cache.ClusterCache{ "https://mycluster": clusterCache, }, + clusterSharding: sharding.NewClusterSharding(db, 0, 1, common.DefaultShardingAlgorithm), } clustersCache.handleModEvent(&appv1.Cluster{ @@ -56,14 +62,22 @@ func TestHandleModEvent_ClusterExcluded(t *testing.T) { clusterCache := &mocks.ClusterCache{} clusterCache.On("Invalidate", mock.Anything, mock.Anything).Return(nil).Once() clusterCache.On("EnsureSynced").Return(nil).Once() - + db := &dbmocks.ArgoDB{} + db.On("GetApplicationControllerReplicas").Return(1) clustersCache := liveStateCache{ - clusters: map[string]cache.ClusterCache{ - "https://mycluster": clusterCache, - }, - clusterFilter: func(cluster *appv1.Cluster) bool { - return false + db: nil, + appInformer: nil, + onObjectUpdated: func(managedByApp map[string]bool, ref v1.ObjectReference) { }, + kubectl: nil, + settingsMgr: &argosettings.SettingsManager{}, + metricsServer: &metrics.MetricsServer{}, + // returns a shard that never process any cluster + clusterSharding: sharding.NewClusterSharding(db, 0, 1, common.DefaultShardingAlgorithm), + resourceTracking: nil, + clusters: map[string]cache.ClusterCache{"https://mycluster": clusterCache}, + cacheSettings: cacheSettings{}, + lock: sync.RWMutex{}, } clustersCache.handleModEvent(&appv1.Cluster{ @@ -75,18 +89,20 @@ func TestHandleModEvent_ClusterExcluded(t *testing.T) { Namespaces: []string{"default"}, }) - assert.Len(t, clustersCache.clusters, 0) + assert.Len(t, clustersCache.clusters, 1) } func TestHandleModEvent_NoChanges(t *testing.T) { clusterCache := &mocks.ClusterCache{} clusterCache.On("Invalidate", mock.Anything).Panic("should not invalidate") clusterCache.On("EnsureSynced").Return(nil).Panic("should not re-sync") - + db := &dbmocks.ArgoDB{} + db.On("GetApplicationControllerReplicas").Return(1) clustersCache := liveStateCache{ clusters: map[string]cache.ClusterCache{ "https://mycluster": clusterCache, }, + clusterSharding: sharding.NewClusterSharding(db, 0, 1, common.DefaultShardingAlgorithm), } clustersCache.handleModEvent(&appv1.Cluster{ @@ -99,11 +115,11 @@ func TestHandleModEvent_NoChanges(t *testing.T) { } func TestHandleAddEvent_ClusterExcluded(t *testing.T) { + db := &dbmocks.ArgoDB{} + db.On("GetApplicationControllerReplicas").Return(1) clustersCache := liveStateCache{ - clusters: map[string]cache.ClusterCache{}, - clusterFilter: func(cluster *appv1.Cluster) bool { - return false - }, + clusters: map[string]cache.ClusterCache{}, + clusterSharding: sharding.NewClusterSharding(db, 0, 2, common.DefaultShardingAlgorithm), } clustersCache.handleAddEvent(&appv1.Cluster{ Server: "https://mycluster", @@ -118,6 +134,8 @@ func TestHandleDeleteEvent_CacheDeadlock(t *testing.T) { Server: "https://mycluster", Config: appv1.ClusterConfig{Username: "bar"}, } + db := &dbmocks.ArgoDB{} + db.On("GetApplicationControllerReplicas").Return(1) fakeClient := fake.NewSimpleClientset() settingsMgr := argosettings.NewSettingsManager(context.TODO(), fakeClient, "argocd") liveStateCacheLock := sync.RWMutex{} @@ -126,10 +144,8 @@ func TestHandleDeleteEvent_CacheDeadlock(t *testing.T) { clusters: map[string]cache.ClusterCache{ testCluster.Server: gitopsEngineClusterCache, }, - clusterFilter: func(cluster *appv1.Cluster) bool { - return true - }, - settingsMgr: settingsMgr, + clusterSharding: sharding.NewClusterSharding(db, 0, 1, common.DefaultShardingAlgorithm), + settingsMgr: settingsMgr, // Set the lock here so we can reference it later // nolint We need to overwrite here to have access to the lock lock: liveStateCacheLock, diff --git a/controller/clusterinfoupdater.go b/controller/clusterinfoupdater.go index a2f488534aeb0..d87cdad6be85d 100644 --- a/controller/clusterinfoupdater.go +++ b/controller/clusterinfoupdater.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "github.com/argoproj/argo-cd/v2/common" "time" "github.com/argoproj/argo-cd/v2/util/env" @@ -101,8 +102,11 @@ func (c *clusterInfoUpdater) updateClusters() { } _ = kube.RunAllAsync(len(clustersFiltered), func(i int) error { cluster := clustersFiltered[i] - if err := c.updateClusterInfo(ctx, cluster, infoByServer[cluster.Server]); err != nil { - log.Warnf("Failed to save clusters info: %v", err) + clusterInfo := infoByServer[cluster.Server] + if err := c.updateClusterInfo(ctx, cluster, clusterInfo); err != nil { + log.Warnf("Failed to save cluster info: %v", err) + } else if err := updateClusterLabels(ctx, clusterInfo, cluster, c.db.UpdateCluster); err != nil { + log.Warnf("Failed to update cluster labels: %v", err) } return nil }) @@ -114,6 +118,12 @@ func (c *clusterInfoUpdater) updateClusterInfo(ctx context.Context, cluster appv if err != nil { return fmt.Errorf("error while fetching the apps list: %w", err) } + + updated := c.getUpdatedClusterInfo(ctx, apps, cluster, info, metav1.Now()) + return c.cache.SetClusterInfo(cluster.Server, &updated) +} + +func (c *clusterInfoUpdater) getUpdatedClusterInfo(ctx context.Context, apps []*appv1.Application, cluster appv1.Cluster, info *cache.ClusterInfo, now metav1.Time) appv1.ClusterInfo { var appCount int64 for _, a := range apps { if c.projGetter != nil { @@ -129,7 +139,6 @@ func (c *clusterInfoUpdater) updateClusterInfo(ctx context.Context, cluster appv appCount += 1 } } - now := metav1.Now() clusterInfo := appv1.ClusterInfo{ ConnectionState: appv1.ConnectionState{ModifiedAt: &now}, ApplicationsCount: appCount, @@ -156,5 +165,15 @@ func (c *clusterInfoUpdater) updateClusterInfo(ctx context.Context, cluster appv } } - return c.cache.SetClusterInfo(cluster.Server, &clusterInfo) + return clusterInfo +} + +func updateClusterLabels(ctx context.Context, clusterInfo *cache.ClusterInfo, cluster appv1.Cluster, updateCluster func(context.Context, *appv1.Cluster) (*appv1.Cluster, error)) error { + if clusterInfo != nil && cluster.Labels[common.LabelKeyAutoLabelClusterInfo] == "true" && cluster.Labels[common.LabelKeyClusterKubernetesVersion] != clusterInfo.K8SVersion { + cluster.Labels[common.LabelKeyClusterKubernetesVersion] = clusterInfo.K8SVersion + _, err := updateCluster(ctx, &cluster) + return err + } + + return nil } diff --git a/controller/clusterinfoupdater_test.go b/controller/clusterinfoupdater_test.go index bac0bb56cbe08..d11d4412bf30c 100644 --- a/controller/clusterinfoupdater_test.go +++ b/controller/clusterinfoupdater_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "fmt" "testing" "time" @@ -98,3 +99,92 @@ func TestClusterSecretUpdater(t *testing.T) { assert.Equal(t, test.ExpectedStatus, clusterInfo.ConnectionState.Status) } } + +func TestUpdateClusterLabels(t *testing.T) { + shouldNotBeInvoked := func(ctx context.Context, cluster *v1alpha1.Cluster) (*v1alpha1.Cluster, error) { + shouldNotHappen := errors.New("if an error happens here, something's wrong") + assert.NoError(t, shouldNotHappen) + return nil, shouldNotHappen + } + tests := []struct { + name string + clusterInfo *clustercache.ClusterInfo + cluster v1alpha1.Cluster + updateCluster func(context.Context, *v1alpha1.Cluster) (*v1alpha1.Cluster, error) + wantErr assert.ErrorAssertionFunc + }{ + { + "enableClusterInfoLabels = false", + &clustercache.ClusterInfo{ + Server: "kubernetes.svc.local", + K8SVersion: "1.28", + }, + v1alpha1.Cluster{ + Server: "kubernetes.svc.local", + Labels: nil, + }, + shouldNotBeInvoked, + assert.NoError, + }, + { + "clusterInfo = nil", + nil, + v1alpha1.Cluster{ + Server: "kubernetes.svc.local", + Labels: map[string]string{"argocd.argoproj.io/auto-label-cluster-info": "true"}, + }, + shouldNotBeInvoked, + assert.NoError, + }, + { + "clusterInfo.k8sversion == cluster k8s label", + &clustercache.ClusterInfo{ + Server: "kubernetes.svc.local", + K8SVersion: "1.28", + }, + v1alpha1.Cluster{ + Server: "kubernetes.svc.local", + Labels: map[string]string{"argocd.argoproj.io/kubernetes-version": "1.28", "argocd.argoproj.io/auto-label-cluster-info": "true"}, + }, + shouldNotBeInvoked, + assert.NoError, + }, + { + "clusterInfo.k8sversion != cluster k8s label, no error", + &clustercache.ClusterInfo{ + Server: "kubernetes.svc.local", + K8SVersion: "1.28", + }, + v1alpha1.Cluster{ + Server: "kubernetes.svc.local", + Labels: map[string]string{"argocd.argoproj.io/kubernetes-version": "1.27", "argocd.argoproj.io/auto-label-cluster-info": "true"}, + }, + func(ctx context.Context, cluster *v1alpha1.Cluster) (*v1alpha1.Cluster, error) { + assert.Equal(t, cluster.Labels["argocd.argoproj.io/kubernetes-version"], "1.28") + return nil, nil + }, + assert.NoError, + }, + { + "clusterInfo.k8sversion != cluster k8s label, some error", + &clustercache.ClusterInfo{ + Server: "kubernetes.svc.local", + K8SVersion: "1.28", + }, + v1alpha1.Cluster{ + Server: "kubernetes.svc.local", + Labels: map[string]string{"argocd.argoproj.io/kubernetes-version": "1.27", "argocd.argoproj.io/auto-label-cluster-info": "true"}, + }, + func(ctx context.Context, cluster *v1alpha1.Cluster) (*v1alpha1.Cluster, error) { + assert.Equal(t, cluster.Labels["argocd.argoproj.io/kubernetes-version"], "1.28") + return nil, errors.New("some error happened while saving") + }, + assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.wantErr(t, updateClusterLabels(context.Background(), tt.clusterInfo, tt.cluster, tt.updateCluster), fmt.Sprintf("updateClusterLabels(%v, %v, %v)", context.Background(), tt.clusterInfo, tt.cluster)) + }) + } +} diff --git a/controller/metrics/metrics.go b/controller/metrics/metrics.go index e4ef09552c09d..94405b51eac75 100644 --- a/controller/metrics/metrics.go +++ b/controller/metrics/metrics.go @@ -23,6 +23,8 @@ import ( "github.com/argoproj/argo-cd/v2/util/git" "github.com/argoproj/argo-cd/v2/util/healthz" "github.com/argoproj/argo-cd/v2/util/profile" + + ctrl_metrics "sigs.k8s.io/controller-runtime/pkg/metrics" ) type MetricsServer struct { @@ -160,12 +162,12 @@ func NewMetricsServer(addr string, appLister applister.ApplicationLister, appFil mux := http.NewServeMux() registry := NewAppRegistry(appLister, appFilter, appLabels) - registry.MustRegister(depth, adds, latency, workDuration, unfinished, longestRunningProcessor, retries) + mux.Handle(MetricsPath, promhttp.HandlerFor(prometheus.Gatherers{ // contains app controller specific metrics registry, - // contains process, golang and controller workqueues metrics - prometheus.DefaultGatherer, + // contains workqueue metrics, process and golang metrics + ctrl_metrics.Registry, }, promhttp.HandlerOpts{})) profile.RegisterProfiler(mux) healthz.ServeHealthCheck(mux, healthCheck) diff --git a/controller/metrics/metrics_test.go b/controller/metrics/metrics_test.go index 61a99a46492a2..23628c38347a5 100644 --- a/controller/metrics/metrics_test.go +++ b/controller/metrics/metrics_test.go @@ -2,6 +2,7 @@ package metrics import ( "context" + "fmt" "log" "net/http" "net/http/httptest" @@ -15,12 +16,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/yaml" argoappv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned/fake" appinformer "github.com/argoproj/argo-cd/v2/pkg/client/informers/externalversions" applister "github.com/argoproj/argo-cd/v2/pkg/client/listers/application/v1alpha1" + + "sigs.k8s.io/controller-runtime/pkg/controller" ) const fakeApp = ` @@ -140,6 +144,12 @@ var appFilter = func(obj interface{}) bool { return true } +func init() { + // Create a fake controller so we initialize the internal controller metrics. + // https://github.com/kubernetes-sigs/controller-runtime/blob/4000e996a202917ad7d40f02ed8a2079a9ce25e9/pkg/internal/controller/metrics/metrics.go + _, _ = controller.New("test-controller", nil, controller.Options{}) +} + func newFakeApp(fakeAppYAML string) *argoappv1.Application { var app argoappv1.Application err := yaml.Unmarshal([]byte(fakeAppYAML), &app) @@ -360,7 +370,7 @@ func assertMetricsPrinted(t *testing.T, expectedLines, body string) { if line == "" { continue } - assert.Contains(t, body, line, "expected metrics mismatch") + assert.Contains(t, body, line, fmt.Sprintf("expected metrics mismatch for line: %s", line)) } } @@ -443,3 +453,70 @@ argocd_app_sync_total{dest_server="https://localhost:6443",name="my-app",namespa err = metricsServ.SetExpiration(time.Second) assert.Error(t, err) } + +func TestWorkqueueMetrics(t *testing.T) { + cancel, appLister := newFakeLister() + defer cancel() + metricsServ, err := NewMetricsServer("localhost:8082", appLister, appFilter, noOpHealthCheck, []string{}) + assert.NoError(t, err) + + expectedMetrics := ` +# TYPE workqueue_adds_total counter +workqueue_adds_total{name="test"} + +# TYPE workqueue_depth gauge +workqueue_depth{name="test"} + +# TYPE workqueue_longest_running_processor_seconds gauge +workqueue_longest_running_processor_seconds{name="test"} + +# TYPE workqueue_queue_duration_seconds histogram + +# TYPE workqueue_unfinished_work_seconds gauge +workqueue_unfinished_work_seconds{name="test"} + +# TYPE workqueue_work_duration_seconds histogram +` + workqueue.NewNamed("test") + + req, err := http.NewRequest(http.MethodGet, "/metrics", nil) + assert.NoError(t, err) + rr := httptest.NewRecorder() + metricsServ.Handler.ServeHTTP(rr, req) + assert.Equal(t, rr.Code, http.StatusOK) + body := rr.Body.String() + log.Println(body) + assertMetricsPrinted(t, expectedMetrics, body) +} + +func TestGoMetrics(t *testing.T) { + cancel, appLister := newFakeLister() + defer cancel() + metricsServ, err := NewMetricsServer("localhost:8082", appLister, appFilter, noOpHealthCheck, []string{}) + assert.NoError(t, err) + + expectedMetrics := ` +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds_sum +go_gc_duration_seconds_count +# TYPE go_goroutines gauge +go_goroutines +# TYPE go_info gauge +go_info +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes +# TYPE go_threads gauge +go_threads +` + + req, err := http.NewRequest(http.MethodGet, "/metrics", nil) + assert.NoError(t, err) + rr := httptest.NewRecorder() + metricsServ.Handler.ServeHTTP(rr, req) + assert.Equal(t, rr.Code, http.StatusOK) + body := rr.Body.String() + log.Println(body) + assertMetricsPrinted(t, expectedMetrics, body) +} diff --git a/controller/metrics/workqueue.go b/controller/metrics/workqueue.go deleted file mode 100644 index 2ef10685ee47d..0000000000000 --- a/controller/metrics/workqueue.go +++ /dev/null @@ -1,101 +0,0 @@ -package metrics - -import ( - "github.com/prometheus/client_golang/prometheus" - "k8s.io/client-go/util/workqueue" -) - -const ( - WorkQueueSubsystem = "workqueue" - DepthKey = "depth" - AddsKey = "adds_total" - QueueLatencyKey = "queue_duration_seconds" - WorkDurationKey = "work_duration_seconds" - UnfinishedWorkKey = "unfinished_work_seconds" - LongestRunningProcessorKey = "longest_running_processor_seconds" - RetriesKey = "retries_total" -) - -var ( - depth = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: DepthKey, - Help: "Current depth of workqueue", - }, []string{"name"}) - - adds = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: WorkQueueSubsystem, - Name: AddsKey, - Help: "Total number of adds handled by workqueue", - }, []string{"name"}) - - latency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: WorkQueueSubsystem, - Name: QueueLatencyKey, - Help: "How long in seconds an item stays in workqueue before being requested", - Buckets: []float64{1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 5, 10, 15, 30, 60, 120, 180}, - }, []string{"name"}) - - workDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: WorkQueueSubsystem, - Name: WorkDurationKey, - Help: "How long in seconds processing an item from workqueue takes.", - Buckets: []float64{1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 5, 10, 15, 30, 60, 120, 180}, - }, []string{"name"}) - - unfinished = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: UnfinishedWorkKey, - Help: "How many seconds of work has been done that " + - "is in progress and hasn't been observed by work_duration. Large " + - "values indicate stuck threads. One can deduce the number of stuck " + - "threads by observing the rate at which this increases.", - }, []string{"name"}) - - longestRunningProcessor = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: LongestRunningProcessorKey, - Help: "How many seconds has the longest running " + - "processor for workqueue been running.", - }, []string{"name"}) - - retries = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: WorkQueueSubsystem, - Name: RetriesKey, - Help: "Total number of retries handled by workqueue", - }, []string{"name"}) -) - -func init() { - workqueue.SetProvider(workqueueMetricsProvider{}) -} - -type workqueueMetricsProvider struct{} - -func (workqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { - return depth.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { - return adds.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { - return latency.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { - return workDuration.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { - return unfinished.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { - return longestRunningProcessor.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric { - return retries.WithLabelValues(name) -} diff --git a/controller/sharding/cache.go b/controller/sharding/cache.go new file mode 100644 index 0000000000000..2f3ffcbcb95c6 --- /dev/null +++ b/controller/sharding/cache.go @@ -0,0 +1,264 @@ +package sharding + +import ( + "sync" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/argoproj/argo-cd/v2/util/db" + log "github.com/sirupsen/logrus" +) + +type ClusterShardingCache interface { + Init(clusters *v1alpha1.ClusterList, apps *v1alpha1.ApplicationList) + Add(c *v1alpha1.Cluster) + Delete(clusterServer string) + Update(oldCluster *v1alpha1.Cluster, newCluster *v1alpha1.Cluster) + AddApp(a *v1alpha1.Application) + DeleteApp(a *v1alpha1.Application) + UpdateApp(a *v1alpha1.Application) + IsManagedCluster(c *v1alpha1.Cluster) bool + GetDistribution() map[string]int + GetAppDistribution() map[string]int +} + +type ClusterSharding struct { + Shard int + Replicas int + Shards map[string]int + Clusters map[string]*v1alpha1.Cluster + Apps map[string]*v1alpha1.Application + lock sync.RWMutex + getClusterShard DistributionFunction +} + +func NewClusterSharding(_ db.ArgoDB, shard, replicas int, shardingAlgorithm string) ClusterShardingCache { + log.Debugf("Processing clusters from shard %d: Using filter function: %s", shard, shardingAlgorithm) + clusterSharding := &ClusterSharding{ + Shard: shard, + Replicas: replicas, + Shards: make(map[string]int), + Clusters: make(map[string]*v1alpha1.Cluster), + Apps: make(map[string]*v1alpha1.Application), + } + distributionFunction := NoShardingDistributionFunction() + if replicas > 1 { + log.Debugf("Processing clusters from shard %d: Using filter function: %s", shard, shardingAlgorithm) + distributionFunction = GetDistributionFunction(clusterSharding.getClusterAccessor(), clusterSharding.getAppAccessor(), shardingAlgorithm, replicas) + } else { + log.Info("Processing all cluster shards") + } + clusterSharding.getClusterShard = distributionFunction + return clusterSharding +} + +// IsManagedCluster returns wheter or not the cluster should be processed by a given shard. +func (s *ClusterSharding) IsManagedCluster(c *v1alpha1.Cluster) bool { + s.lock.RLock() + defer s.lock.RUnlock() + if c == nil { // nil cluster (in-cluster) is always managed by current clusterShard + return true + } + clusterShard := 0 + if shard, ok := s.Shards[c.Server]; ok { + clusterShard = shard + } else { + log.Warnf("The cluster %s has no assigned shard.", c.Server) + } + log.Debugf("Checking if cluster %s with clusterShard %d should be processed by shard %d", c.Server, clusterShard, s.Shard) + return clusterShard == s.Shard +} + +func (sharding *ClusterSharding) Init(clusters *v1alpha1.ClusterList, apps *v1alpha1.ApplicationList) { + sharding.lock.Lock() + defer sharding.lock.Unlock() + newClusters := make(map[string]*v1alpha1.Cluster, len(clusters.Items)) + for _, c := range clusters.Items { + cluster := c + newClusters[c.Server] = &cluster + } + sharding.Clusters = newClusters + + newApps := make(map[string]*v1alpha1.Application, len(apps.Items)) + for i := range apps.Items { + app := apps.Items[i] + newApps[app.Name] = &app + } + sharding.Apps = newApps + sharding.updateDistribution() +} + +func (sharding *ClusterSharding) Add(c *v1alpha1.Cluster) { + sharding.lock.Lock() + defer sharding.lock.Unlock() + + old, ok := sharding.Clusters[c.Server] + sharding.Clusters[c.Server] = c + if !ok || hasShardingUpdates(old, c) { + sharding.updateDistribution() + } else { + log.Debugf("Skipping sharding distribution update. Cluster already added") + } +} + +func (sharding *ClusterSharding) Delete(clusterServer string) { + sharding.lock.Lock() + defer sharding.lock.Unlock() + if _, ok := sharding.Clusters[clusterServer]; ok { + delete(sharding.Clusters, clusterServer) + delete(sharding.Shards, clusterServer) + sharding.updateDistribution() + } +} + +func (sharding *ClusterSharding) Update(oldCluster *v1alpha1.Cluster, newCluster *v1alpha1.Cluster) { + sharding.lock.Lock() + defer sharding.lock.Unlock() + + if _, ok := sharding.Clusters[oldCluster.Server]; ok && oldCluster.Server != newCluster.Server { + delete(sharding.Clusters, oldCluster.Server) + delete(sharding.Shards, oldCluster.Server) + } + sharding.Clusters[newCluster.Server] = newCluster + if hasShardingUpdates(oldCluster, newCluster) { + sharding.updateDistribution() + } else { + log.Debugf("Skipping sharding distribution update. No relevant changes") + } +} + +func (sharding *ClusterSharding) GetDistribution() map[string]int { + sharding.lock.RLock() + defer sharding.lock.RUnlock() + shards := sharding.Shards + + distribution := make(map[string]int, len(shards)) + for k, v := range shards { + distribution[k] = v + } + return distribution +} + +func (sharding *ClusterSharding) updateDistribution() { + for k, c := range sharding.Clusters { + shard := 0 + if c.Shard != nil { + requestedShard := int(*c.Shard) + if requestedShard < sharding.Replicas { + shard = requestedShard + } else { + log.Warnf("Specified cluster shard (%d) for cluster: %s is greater than the number of available shard (%d). Using shard 0.", requestedShard, c.Server, sharding.Replicas) + } + } else { + shard = sharding.getClusterShard(c) + } + + existingShard, ok := sharding.Shards[k] + if ok && existingShard != shard { + log.Infof("Cluster %s has changed shard from %d to %d", k, existingShard, shard) + } else if !ok { + log.Infof("Cluster %s has been assigned to shard %d", k, shard) + } else { + log.Debugf("Cluster %s has not changed shard", k) + } + sharding.Shards[k] = shard + } +} + +// hasShardingUpdates returns true if the sharding distribution has explicitly changed +func hasShardingUpdates(old, new *v1alpha1.Cluster) bool { + if old == nil || new == nil { + return false + } + + // returns true if the cluster id has changed because some sharding algorithms depend on it. + if old.ID != new.ID { + return true + } + + if old.Server != new.Server { + return true + } + + // return false if the shard field has not been modified + if old.Shard == nil && new.Shard == nil { + return false + } + return old.Shard == nil || new.Shard == nil || int64(*old.Shard) != int64(*new.Shard) +} + +// A read lock should be acquired before calling getClusterAccessor. +func (d *ClusterSharding) getClusterAccessor() clusterAccessor { + return func() []*v1alpha1.Cluster { + // no need to lock, as this is only called from the updateDistribution function + clusters := make([]*v1alpha1.Cluster, 0, len(d.Clusters)) + for _, c := range d.Clusters { + clusters = append(clusters, c) + } + return clusters + } +} + +// A read lock should be acquired before calling getAppAccessor. +func (d *ClusterSharding) getAppAccessor() appAccessor { + return func() []*v1alpha1.Application { + apps := make([]*v1alpha1.Application, 0, len(d.Apps)) + for _, a := range d.Apps { + apps = append(apps, a) + } + return apps + } +} + +func (sharding *ClusterSharding) AddApp(a *v1alpha1.Application) { + sharding.lock.Lock() + defer sharding.lock.Unlock() + + _, ok := sharding.Apps[a.Name] + sharding.Apps[a.Name] = a + if !ok { + sharding.updateDistribution() + } else { + log.Debugf("Skipping sharding distribution update. App already added") + } +} + +func (sharding *ClusterSharding) DeleteApp(a *v1alpha1.Application) { + sharding.lock.Lock() + defer sharding.lock.Unlock() + if _, ok := sharding.Apps[a.Name]; ok { + delete(sharding.Apps, a.Name) + sharding.updateDistribution() + } +} + +func (sharding *ClusterSharding) UpdateApp(a *v1alpha1.Application) { + sharding.lock.Lock() + defer sharding.lock.Unlock() + + _, ok := sharding.Apps[a.Name] + sharding.Apps[a.Name] = a + if !ok { + sharding.updateDistribution() + } else { + log.Debugf("Skipping sharding distribution update. No relevant changes") + } +} + +// GetAppDistribution should be not be called from a DestributionFunction because +// it could cause a deadlock when updateDistribution is called. +func (sharding *ClusterSharding) GetAppDistribution() map[string]int { + sharding.lock.RLock() + clusters := sharding.Clusters + apps := sharding.Apps + sharding.lock.RUnlock() + + appDistribution := make(map[string]int, len(clusters)) + + for _, a := range apps { + if _, ok := appDistribution[a.Spec.Destination.Server]; !ok { + appDistribution[a.Spec.Destination.Server] = 0 + } + appDistribution[a.Spec.Destination.Server]++ + } + return appDistribution +} diff --git a/controller/sharding/cache_test.go b/controller/sharding/cache_test.go new file mode 100644 index 0000000000000..f7798c31e3608 --- /dev/null +++ b/controller/sharding/cache_test.go @@ -0,0 +1,511 @@ +package sharding + +import ( + "testing" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + dbmocks "github.com/argoproj/argo-cd/v2/util/db/mocks" + "github.com/stretchr/testify/assert" +) + +func setupTestSharding(shard int, replicas int) *ClusterSharding { + shardingAlgorithm := "legacy" // we are using the legacy algorithm as it is deterministic based on the cluster id which is easier to test + db := &dbmocks.ArgoDB{} + return NewClusterSharding(db, shard, replicas, shardingAlgorithm).(*ClusterSharding) +} + +func TestNewClusterSharding(t *testing.T) { + shard := 1 + replicas := 2 + sharding := setupTestSharding(shard, replicas) + + assert.NotNil(t, sharding) + assert.Equal(t, shard, sharding.Shard) + assert.Equal(t, replicas, sharding.Replicas) + assert.NotNil(t, sharding.Shards) + assert.NotNil(t, sharding.Clusters) +} + +func TestClusterSharding_Add(t *testing.T) { + shard := 1 + replicas := 2 + sharding := setupTestSharding(shard, replicas) + + clusterA := &v1alpha1.Cluster{ + ID: "2", + Server: "https://127.0.0.1:6443", + } + + sharding.Add(clusterA) + + clusterB := v1alpha1.Cluster{ + ID: "1", + Server: "https://kubernetes.default.svc", + } + + sharding.Add(&clusterB) + + distribution := sharding.GetDistribution() + + assert.Contains(t, sharding.Clusters, clusterA.Server) + assert.Contains(t, sharding.Clusters, clusterB.Server) + + clusterDistribution, ok := distribution[clusterA.Server] + assert.True(t, ok) + assert.Equal(t, 1, clusterDistribution) + + myClusterDistribution, ok := distribution[clusterB.Server] + assert.True(t, ok) + assert.Equal(t, 0, myClusterDistribution) + + assert.Equal(t, 2, len(distribution)) +} + +func TestClusterSharding_AddRoundRobin_Redistributes(t *testing.T) { + shard := 1 + replicas := 2 + + db := &dbmocks.ArgoDB{} + + sharding := NewClusterSharding(db, shard, replicas, "round-robin").(*ClusterSharding) + + clusterA := &v1alpha1.Cluster{ + ID: "1", + Server: "https://127.0.0.1:6443", + } + sharding.Add(clusterA) + + clusterB := v1alpha1.Cluster{ + ID: "3", + Server: "https://kubernetes.default.svc", + } + sharding.Add(&clusterB) + + distributionBefore := sharding.GetDistribution() + + assert.Contains(t, sharding.Clusters, clusterA.Server) + assert.Contains(t, sharding.Clusters, clusterB.Server) + + clusterDistributionA, ok := distributionBefore[clusterA.Server] + assert.True(t, ok) + assert.Equal(t, 0, clusterDistributionA) + + clusterDistributionB, ok := distributionBefore[clusterB.Server] + assert.True(t, ok) + assert.Equal(t, 1, clusterDistributionB) + + assert.Equal(t, 2, len(distributionBefore)) + + clusterC := v1alpha1.Cluster{ + ID: "2", + Server: "https://1.1.1.1", + } + sharding.Add(&clusterC) + + distributionAfter := sharding.GetDistribution() + + assert.Contains(t, sharding.Clusters, clusterA.Server) + assert.Contains(t, sharding.Clusters, clusterB.Server) + assert.Contains(t, sharding.Clusters, clusterC.Server) + + clusterDistributionA, ok = distributionAfter[clusterA.Server] + assert.True(t, ok) + assert.Equal(t, 0, clusterDistributionA) + + clusterDistributionC, ok := distributionAfter[clusterC.Server] + assert.True(t, ok) + assert.Equal(t, 1, clusterDistributionC) // will be assigned to shard 1 because the .ID is smaller then the "B" cluster + + clusterDistributionB, ok = distributionAfter[clusterB.Server] + assert.True(t, ok) + assert.Equal(t, 0, clusterDistributionB) // will be reassigned to shard 0 because the .ID is bigger then the "C" cluster +} + +func TestClusterSharding_Delete(t *testing.T) { + shard := 1 + replicas := 2 + sharding := setupTestSharding(shard, replicas) + + sharding.Init( + &v1alpha1.ClusterList{ + Items: []v1alpha1.Cluster{ + { + ID: "2", + Server: "https://127.0.0.1:6443", + }, + { + ID: "1", + Server: "https://kubernetes.default.svc", + }, + }, + }, + &v1alpha1.ApplicationList{ + Items: []v1alpha1.Application{ + createApp("app2", "https://127.0.0.1:6443"), + createApp("app1", "https://kubernetes.default.svc"), + }, + }, + ) + + sharding.Delete("https://kubernetes.default.svc") + distribution := sharding.GetDistribution() + assert.Equal(t, 1, len(distribution)) +} + +func TestClusterSharding_Update(t *testing.T) { + shard := 1 + replicas := 2 + sharding := setupTestSharding(shard, replicas) + + sharding.Init( + &v1alpha1.ClusterList{ + Items: []v1alpha1.Cluster{ + { + ID: "2", + Server: "https://127.0.0.1:6443", + }, + { + ID: "1", + Server: "https://kubernetes.default.svc", + }, + }, + }, + &v1alpha1.ApplicationList{ + Items: []v1alpha1.Application{ + createApp("app2", "https://127.0.0.1:6443"), + createApp("app1", "https://kubernetes.default.svc"), + }, + }, + ) + + distributionBefore := sharding.GetDistribution() + assert.Equal(t, 2, len(distributionBefore)) + + distributionA, ok := distributionBefore["https://kubernetes.default.svc"] + assert.True(t, ok) + assert.Equal(t, 0, distributionA) + + sharding.Update(&v1alpha1.Cluster{ + ID: "1", + Server: "https://kubernetes.default.svc", + }, &v1alpha1.Cluster{ + ID: "4", + Server: "https://kubernetes.default.svc", + }) + + distributionAfter := sharding.GetDistribution() + assert.Equal(t, 2, len(distributionAfter)) + + distributionA, ok = distributionAfter["https://kubernetes.default.svc"] + assert.True(t, ok) + assert.Equal(t, 1, distributionA) +} + +func TestClusterSharding_UpdateServerName(t *testing.T) { + shard := 1 + replicas := 2 + sharding := setupTestSharding(shard, replicas) + + sharding.Init( + &v1alpha1.ClusterList{ + Items: []v1alpha1.Cluster{ + { + ID: "2", + Server: "https://127.0.0.1:6443", + }, + { + ID: "1", + Server: "https://kubernetes.default.svc", + }, + }, + }, + &v1alpha1.ApplicationList{ + Items: []v1alpha1.Application{ + createApp("app2", "https://127.0.0.1:6443"), + createApp("app1", "https://kubernetes.default.svc"), + }, + }, + ) + + distributionBefore := sharding.GetDistribution() + assert.Equal(t, 2, len(distributionBefore)) + + distributionA, ok := distributionBefore["https://kubernetes.default.svc"] + assert.True(t, ok) + assert.Equal(t, 0, distributionA) + + sharding.Update(&v1alpha1.Cluster{ + ID: "1", + Server: "https://kubernetes.default.svc", + }, &v1alpha1.Cluster{ + ID: "1", + Server: "https://server2", + }) + + distributionAfter := sharding.GetDistribution() + assert.Equal(t, 2, len(distributionAfter)) + + _, ok = distributionAfter["https://kubernetes.default.svc"] + assert.False(t, ok) // the old server name should not be present anymore + + _, ok = distributionAfter["https://server2"] + assert.True(t, ok) // the new server name should be present +} + +func TestClusterSharding_IsManagedCluster(t *testing.T) { + replicas := 2 + sharding0 := setupTestSharding(0, replicas) + + sharding0.Init( + &v1alpha1.ClusterList{ + Items: []v1alpha1.Cluster{ + { + ID: "1", + Server: "https://kubernetes.default.svc", + }, + { + ID: "2", + Server: "https://127.0.0.1:6443", + }, + }, + }, + &v1alpha1.ApplicationList{ + Items: []v1alpha1.Application{ + createApp("app2", "https://127.0.0.1:6443"), + createApp("app1", "https://kubernetes.default.svc"), + }, + }, + ) + + assert.True(t, sharding0.IsManagedCluster(&v1alpha1.Cluster{ + ID: "1", + Server: "https://kubernetes.default.svc", + })) + + assert.False(t, sharding0.IsManagedCluster(&v1alpha1.Cluster{ + ID: "2", + Server: "https://127.0.0.1:6443", + })) + + sharding1 := setupTestSharding(1, replicas) + + sharding1.Init( + &v1alpha1.ClusterList{ + Items: []v1alpha1.Cluster{ + { + ID: "2", + Server: "https://127.0.0.1:6443", + }, + { + ID: "1", + Server: "https://kubernetes.default.svc", + }, + }, + }, + &v1alpha1.ApplicationList{ + Items: []v1alpha1.Application{ + createApp("app2", "https://127.0.0.1:6443"), + createApp("app1", "https://kubernetes.default.svc"), + }, + }, + ) + + assert.False(t, sharding1.IsManagedCluster(&v1alpha1.Cluster{ + ID: "1", + Server: "https://kubernetes.default.svc", + })) + + assert.True(t, sharding1.IsManagedCluster(&v1alpha1.Cluster{ + ID: "2", + Server: "https://127.0.0.1:6443", + })) + +} + +func TestClusterSharding_ClusterShardOfResourceShouldNotBeChanged(t *testing.T) { + shard := 1 + replicas := 2 + sharding := setupTestSharding(shard, replicas) + + Int64Ptr := func(i int64) *int64 { + return &i + } + + clusterWithNil := &v1alpha1.Cluster{ + ID: "2", + Server: "https://127.0.0.1:6443", + Shard: nil, + } + + clusterWithValue := &v1alpha1.Cluster{ + ID: "1", + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(1), + } + + clusterWithToBigValue := &v1alpha1.Cluster{ + ID: "3", + Server: "https://1.1.1.1", + Shard: Int64Ptr(999), // shard value is explicitly bigger than the number of replicas + } + + sharding.Init( + &v1alpha1.ClusterList{ + Items: []v1alpha1.Cluster{ + *clusterWithNil, + *clusterWithValue, + *clusterWithToBigValue, + }, + }, + &v1alpha1.ApplicationList{ + Items: []v1alpha1.Application{ + createApp("app2", "https://127.0.0.1:6443"), + createApp("app1", "https://kubernetes.default.svc"), + }, + }, + ) + distribution := sharding.GetDistribution() + assert.Equal(t, 3, len(distribution)) + + assert.Nil(t, sharding.Clusters[clusterWithNil.Server].Shard) + + assert.NotNil(t, sharding.Clusters[clusterWithValue.Server].Shard) + assert.Equal(t, int64(1), *sharding.Clusters[clusterWithValue.Server].Shard) + assert.Equal(t, 1, distribution[clusterWithValue.Server]) + + assert.NotNil(t, sharding.Clusters[clusterWithToBigValue.Server].Shard) + assert.Equal(t, int64(999), *sharding.Clusters[clusterWithToBigValue.Server].Shard) + assert.Equal(t, 0, distribution[clusterWithToBigValue.Server]) // will be assigned to shard 0 because the value is bigger than the number of replicas +} + +func TestHasShardingUpdates(t *testing.T) { + Int64Ptr := func(i int64) *int64 { + return &i + } + + testCases := []struct { + name string + old *v1alpha1.Cluster + new *v1alpha1.Cluster + expected bool + }{ + { + name: "No updates", + old: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(1), + }, + new: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(1), + }, + expected: false, + }, + { + name: "Updates", + old: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(1), + }, + new: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(2), + }, + expected: true, + }, + { + name: "Old is nil", + old: nil, + new: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(2), + }, + expected: false, + }, + { + name: "New is nil", + old: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(2), + }, + new: nil, + expected: false, + }, + { + name: "Both are nil", + old: nil, + new: nil, + expected: false, + }, + { + name: "Both shards are nil", + old: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: nil, + }, + new: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: nil, + }, + expected: false, + }, + { + name: "Old shard is nil", + old: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: nil, + }, + new: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(2), + }, + expected: true, + }, + { + name: "New shard is nil", + old: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(2), + }, + new: &v1alpha1.Cluster{ + Server: "https://kubernetes.default.svc", + Shard: nil, + }, + expected: true, + }, + { + name: "Cluster ID has changed", + old: &v1alpha1.Cluster{ + ID: "1", + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(2), + }, + new: &v1alpha1.Cluster{ + ID: "2", + Server: "https://kubernetes.default.svc", + Shard: Int64Ptr(2), + }, + expected: true, + }, + { + name: "Server has changed", + old: &v1alpha1.Cluster{ + ID: "1", + Server: "https://server1", + Shard: Int64Ptr(2), + }, + new: &v1alpha1.Cluster{ + ID: "1", + Server: "https://server2", + Shard: Int64Ptr(2), + }, + expected: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, hasShardingUpdates(tc.old, tc.new)) + }) + } +} diff --git a/controller/sharding/sharding.go b/controller/sharding/sharding.go index 526896531dbca..e4af7010931c6 100644 --- a/controller/sharding/sharding.go +++ b/controller/sharding/sharding.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "hash/fnv" + "math" "os" "sort" "strconv" @@ -20,6 +21,7 @@ import ( "github.com/argoproj/argo-cd/v2/util/db" "github.com/argoproj/argo-cd/v2/util/env" + "github.com/argoproj/argo-cd/v2/util/errors" "github.com/argoproj/argo-cd/v2/util/settings" log "github.com/sirupsen/logrus" kubeerrors "k8s.io/apimachinery/pkg/api/errors" @@ -40,6 +42,8 @@ const ShardControllerMappingKey = "shardControllerMapping" type DistributionFunction func(c *v1alpha1.Cluster) int type ClusterFilterFunction func(c *v1alpha1.Cluster) bool +type clusterAccessor func() []*v1alpha1.Cluster +type appAccessor func() []*v1alpha1.Application // shardApplicationControllerMapping stores the mapping of Shard Number to Application Controller in ConfigMap. // It also stores the heartbeat of last synced time of the application controller. @@ -53,8 +57,7 @@ type shardApplicationControllerMapping struct { // and returns wheter or not the cluster should be processed by a given shard. It calls the distributionFunction // to determine which shard will process the cluster, and if the given shard is equal to the calculated shard // the function will return true. -func GetClusterFilter(db db.ArgoDB, distributionFunction DistributionFunction, shard int) ClusterFilterFunction { - replicas := db.GetApplicationControllerReplicas() +func GetClusterFilter(db db.ArgoDB, distributionFunction DistributionFunction, replicas, shard int) ClusterFilterFunction { return func(c *v1alpha1.Cluster) bool { clusterShard := 0 if c != nil && c.Shard != nil { @@ -73,14 +76,14 @@ func GetClusterFilter(db db.ArgoDB, distributionFunction DistributionFunction, s // GetDistributionFunction returns which DistributionFunction should be used based on the passed algorithm and // the current datas. -func GetDistributionFunction(db db.ArgoDB, shardingAlgorithm string) DistributionFunction { - log.Infof("Using filter function: %s", shardingAlgorithm) - distributionFunction := LegacyDistributionFunction(db) +func GetDistributionFunction(clusters clusterAccessor, apps appAccessor, shardingAlgorithm string, replicasCount int) DistributionFunction { + log.Debugf("Using filter function: %s", shardingAlgorithm) + distributionFunction := LegacyDistributionFunction(replicasCount) switch shardingAlgorithm { case common.RoundRobinShardingAlgorithm: - distributionFunction = RoundRobinDistributionFunction(db) + distributionFunction = RoundRobinDistributionFunction(clusters, replicasCount) case common.LegacyShardingAlgorithm: - distributionFunction = LegacyDistributionFunction(db) + distributionFunction = LegacyDistributionFunction(replicasCount) default: log.Warnf("distribution type %s is not supported, defaulting to %s", shardingAlgorithm, common.DefaultShardingAlgorithm) } @@ -92,15 +95,21 @@ func GetDistributionFunction(db db.ArgoDB, shardingAlgorithm string) Distributio // is lightweight and can be distributed easily, however, it does not ensure an homogenous distribution as // some shards may get assigned more clusters than others. It is the legacy function distribution that is // kept for compatibility reasons -func LegacyDistributionFunction(db db.ArgoDB) DistributionFunction { - replicas := db.GetApplicationControllerReplicas() +func LegacyDistributionFunction(replicas int) DistributionFunction { return func(c *v1alpha1.Cluster) int { if replicas == 0 { + log.Debugf("Replicas count is : %d, returning -1", replicas) return -1 } if c == nil { + log.Debug("In-cluster: returning 0") return 0 } + // if Shard is manually set and the assigned value is lower than the number of replicas, + // then its value is returned otherwise it is the default calculated value + if c.Shard != nil && int(*c.Shard) < replicas { + return int(*c.Shard) + } id := c.ID log.Debugf("Calculating cluster shard for cluster id: %s", id) if id == "" { @@ -121,14 +130,19 @@ func LegacyDistributionFunction(db db.ArgoDB) DistributionFunction { // This function ensures an homogenous distribution: each shards got assigned the same number of // clusters +/-1 , but with the drawback of a reshuffling of clusters accross shards in case of some changes // in the cluster list -func RoundRobinDistributionFunction(db db.ArgoDB) DistributionFunction { - replicas := db.GetApplicationControllerReplicas() + +func RoundRobinDistributionFunction(clusters clusterAccessor, replicas int) DistributionFunction { return func(c *v1alpha1.Cluster) int { if replicas > 0 { if c == nil { // in-cluster does not necessarly have a secret assigned. So we are receiving a nil cluster here. return 0 + } + // if Shard is manually set and the assigned value is lower than the number of replicas, + // then its value is returned otherwise it is the default calculated value + if c.Shard != nil && int(*c.Shard) < replicas { + return int(*c.Shard) } else { - clusterIndexdByClusterIdMap := createClusterIndexByClusterIdMap(db) + clusterIndexdByClusterIdMap := createClusterIndexByClusterIdMap(clusters) clusterIndex, ok := clusterIndexdByClusterIdMap[c.ID] if !ok { log.Warnf("Cluster with id=%s not found in cluster map.", c.ID) @@ -144,6 +158,12 @@ func RoundRobinDistributionFunction(db db.ArgoDB) DistributionFunction { } } +// NoShardingDistributionFunction returns a DistributionFunction that will process all cluster by shard 0 +// the function is created for API compatibility purposes and is not supposed to be activated. +func NoShardingDistributionFunction() DistributionFunction { + return func(c *v1alpha1.Cluster) int { return 0 } +} + // InferShard extracts the shard index based on its hostname. func InferShard() (int, error) { hostname, err := osHostnameFunction() @@ -152,33 +172,29 @@ func InferShard() (int, error) { } parts := strings.Split(hostname, "-") if len(parts) == 0 { - return 0, fmt.Errorf("hostname should ends with shard number separated by '-' but got: %s", hostname) + log.Warnf("hostname should end with shard number separated by '-' but got: %s", hostname) + return 0, nil } shard, err := strconv.Atoi(parts[len(parts)-1]) if err != nil { - return 0, fmt.Errorf("hostname should ends with shard number separated by '-' but got: %s", hostname) + log.Warnf("hostname should end with shard number separated by '-' but got: %s", hostname) + return 0, nil } return int(shard), nil } -func getSortedClustersList(db db.ArgoDB) []v1alpha1.Cluster { - ctx := context.Background() - clustersList, dbErr := db.ListClusters(ctx) - if dbErr != nil { - log.Warnf("Error while querying clusters list from database: %v", dbErr) - return []v1alpha1.Cluster{} - } - clusters := clustersList.Items +func getSortedClustersList(getCluster clusterAccessor) []*v1alpha1.Cluster { + clusters := getCluster() sort.Slice(clusters, func(i, j int) bool { return clusters[i].ID < clusters[j].ID }) return clusters } -func createClusterIndexByClusterIdMap(db db.ArgoDB) map[string]int { - clusters := getSortedClustersList(db) +func createClusterIndexByClusterIdMap(getCluster clusterAccessor) map[string]int { + clusters := getSortedClustersList(getCluster) log.Debugf("ClustersList has %d items", len(clusters)) - clusterById := make(map[string]v1alpha1.Cluster) + clusterById := make(map[string]*v1alpha1.Cluster) clusterIndexedByClusterId := make(map[string]int) for i, cluster := range clusters { log.Debugf("Adding cluster with id=%s and name=%s to cluster's map", cluster.ID, cluster.Name) @@ -193,8 +209,7 @@ func createClusterIndexByClusterIdMap(db db.ArgoDB) map[string]int { // The function takes the shard number from the environment variable (default value -1, if not set) and passes it to this function. // If the shard value passed to this function is -1, that is, the shard was not set as an environment variable, // we default the shard number to 0 for computing the default config map. -func GetOrUpdateShardFromConfigMap(kubeClient *kubernetes.Clientset, settingsMgr *settings.SettingsManager, replicas, shard int) (int, error) { - +func GetOrUpdateShardFromConfigMap(kubeClient kubernetes.Interface, settingsMgr *settings.SettingsManager, replicas, shard int) (int, error) { hostname, err := osHostnameFunction() if err != nil { return -1, err @@ -351,3 +366,59 @@ func getDefaultShardMappingData(replicas int) []shardApplicationControllerMappin } return shardMappingData } + +func GetClusterSharding(kubeClient kubernetes.Interface, settingsMgr *settings.SettingsManager, shardingAlgorithm string, enableDynamicClusterDistribution bool) (ClusterShardingCache, error) { + var replicasCount int + if enableDynamicClusterDistribution { + applicationControllerName := env.StringFromEnv(common.EnvAppControllerName, common.DefaultApplicationControllerName) + appControllerDeployment, err := kubeClient.AppsV1().Deployments(settingsMgr.GetNamespace()).Get(context.Background(), applicationControllerName, metav1.GetOptions{}) + + // if app controller deployment is not found when dynamic cluster distribution is enabled error out + if err != nil { + return nil, fmt.Errorf("(dynamic cluster distribution) failed to get app controller deployment: %v", err) + } + + if appControllerDeployment != nil && appControllerDeployment.Spec.Replicas != nil { + replicasCount = int(*appControllerDeployment.Spec.Replicas) + } else { + return nil, fmt.Errorf("(dynamic cluster distribution) failed to get app controller deployment replica count") + } + + } else { + replicasCount = env.ParseNumFromEnv(common.EnvControllerReplicas, 0, 0, math.MaxInt32) + } + shardNumber := env.ParseNumFromEnv(common.EnvControllerShard, -1, -math.MaxInt32, math.MaxInt32) + if replicasCount > 1 { + // check for shard mapping using configmap if application-controller is a deployment + // else use existing logic to infer shard from pod name if application-controller is a statefulset + if enableDynamicClusterDistribution { + var err error + // retry 3 times if we find a conflict while updating shard mapping configMap. + // If we still see conflicts after the retries, wait for next iteration of heartbeat process. + for i := 0; i <= common.AppControllerHeartbeatUpdateRetryCount; i++ { + shardNumber, err = GetOrUpdateShardFromConfigMap(kubeClient, settingsMgr, replicasCount, shardNumber) + if err != nil && !kubeerrors.IsConflict(err) { + err = fmt.Errorf("unable to get shard due to error updating the sharding config map: %s", err) + break + } + log.Warnf("conflict when getting shard from shard mapping configMap. Retrying (%d/3)", i) + } + errors.CheckError(err) + } else { + if shardNumber < 0 { + var err error + shardNumber, err = InferShard() + errors.CheckError(err) + } + if shardNumber > replicasCount { + log.Warnf("Calculated shard number %d is greated than the number of replicas count. Defaulting to 0", shardNumber) + shardNumber = 0 + } + } + } else { + log.Info("Processing all cluster shards") + shardNumber = 0 + } + db := db.NewDB(settingsMgr.GetNamespace(), settingsMgr, kubeClient) + return NewClusterSharding(db, shardNumber, replicasCount, shardingAlgorithm), nil +} diff --git a/controller/sharding/sharding_test.go b/controller/sharding/sharding_test.go index a8a25e11c4978..1c338aac5f271 100644 --- a/controller/sharding/sharding_test.go +++ b/controller/sharding/sharding_test.go @@ -1,36 +1,45 @@ package sharding import ( + "context" "encoding/json" "errors" "fmt" "os" + "strconv" "testing" "time" "github.com/argoproj/argo-cd/v2/common" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" dbmocks "github.com/argoproj/argo-cd/v2/util/db/mocks" + "github.com/argoproj/argo-cd/v2/util/settings" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/yaml" ) func TestGetShardByID_NotEmptyID(t *testing.T) { db := &dbmocks.ArgoDB{} - db.On("GetApplicationControllerReplicas").Return(1) - assert.Equal(t, 0, LegacyDistributionFunction(db)(&v1alpha1.Cluster{ID: "1"})) - assert.Equal(t, 0, LegacyDistributionFunction(db)(&v1alpha1.Cluster{ID: "2"})) - assert.Equal(t, 0, LegacyDistributionFunction(db)(&v1alpha1.Cluster{ID: "3"})) - assert.Equal(t, 0, LegacyDistributionFunction(db)(&v1alpha1.Cluster{ID: "4"})) + replicasCount := 1 + db.On("GetApplicationControllerReplicas").Return(replicasCount) + assert.Equal(t, 0, LegacyDistributionFunction(replicasCount)(&v1alpha1.Cluster{ID: "1"})) + assert.Equal(t, 0, LegacyDistributionFunction(replicasCount)(&v1alpha1.Cluster{ID: "2"})) + assert.Equal(t, 0, LegacyDistributionFunction(replicasCount)(&v1alpha1.Cluster{ID: "3"})) + assert.Equal(t, 0, LegacyDistributionFunction(replicasCount)(&v1alpha1.Cluster{ID: "4"})) } func TestGetShardByID_EmptyID(t *testing.T) { db := &dbmocks.ArgoDB{} - db.On("GetApplicationControllerReplicas").Return(1) + replicasCount := 1 + db.On("GetApplicationControllerReplicas").Return(replicasCount) distributionFunction := LegacyDistributionFunction - shard := distributionFunction(db)(&v1alpha1.Cluster{}) + shard := distributionFunction(replicasCount)(&v1alpha1.Cluster{}) assert.Equal(t, 0, shard) } @@ -38,7 +47,7 @@ func TestGetShardByID_NoReplicas(t *testing.T) { db := &dbmocks.ArgoDB{} db.On("GetApplicationControllerReplicas").Return(0) distributionFunction := LegacyDistributionFunction - shard := distributionFunction(db)(&v1alpha1.Cluster{}) + shard := distributionFunction(0)(&v1alpha1.Cluster{}) assert.Equal(t, -1, shard) } @@ -46,16 +55,16 @@ func TestGetShardByID_NoReplicasUsingHashDistributionFunction(t *testing.T) { db := &dbmocks.ArgoDB{} db.On("GetApplicationControllerReplicas").Return(0) distributionFunction := LegacyDistributionFunction - shard := distributionFunction(db)(&v1alpha1.Cluster{}) + shard := distributionFunction(0)(&v1alpha1.Cluster{}) assert.Equal(t, -1, shard) } func TestGetShardByID_NoReplicasUsingHashDistributionFunctionWithClusters(t *testing.T) { - db, cluster1, cluster2, cluster3, cluster4, cluster5 := createTestClusters() + clusters, db, cluster1, cluster2, cluster3, cluster4, cluster5 := createTestClusters() // Test with replicas set to 0 db.On("GetApplicationControllerReplicas").Return(0) t.Setenv(common.EnvControllerShardingAlgorithm, common.RoundRobinShardingAlgorithm) - distributionFunction := RoundRobinDistributionFunction(db) + distributionFunction := RoundRobinDistributionFunction(clusters, 0) assert.Equal(t, -1, distributionFunction(nil)) assert.Equal(t, -1, distributionFunction(&cluster1)) assert.Equal(t, -1, distributionFunction(&cluster2)) @@ -65,137 +74,115 @@ func TestGetShardByID_NoReplicasUsingHashDistributionFunctionWithClusters(t *tes } func TestGetClusterFilterDefault(t *testing.T) { - shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) + //shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) + clusterAccessor, _, cluster1, cluster2, cluster3, cluster4, _ := createTestClusters() os.Unsetenv(common.EnvControllerShardingAlgorithm) - db := &dbmocks.ArgoDB{} - db.On("GetApplicationControllerReplicas").Return(2) - filter := GetClusterFilter(db, GetDistributionFunction(db, common.DefaultShardingAlgorithm), shardIndex) - assert.False(t, filter(&v1alpha1.Cluster{ID: "1"})) - assert.True(t, filter(&v1alpha1.Cluster{ID: "2"})) - assert.False(t, filter(&v1alpha1.Cluster{ID: "3"})) - assert.True(t, filter(&v1alpha1.Cluster{ID: "4"})) + replicasCount := 2 + distributionFunction := RoundRobinDistributionFunction(clusterAccessor, replicasCount) + assert.Equal(t, 0, distributionFunction(nil)) + assert.Equal(t, 0, distributionFunction(&cluster1)) + assert.Equal(t, 1, distributionFunction(&cluster2)) + assert.Equal(t, 0, distributionFunction(&cluster3)) + assert.Equal(t, 1, distributionFunction(&cluster4)) } func TestGetClusterFilterLegacy(t *testing.T) { - shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) - db := &dbmocks.ArgoDB{} - db.On("GetApplicationControllerReplicas").Return(2) + //shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) + clusterAccessor, db, cluster1, cluster2, cluster3, cluster4, _ := createTestClusters() + replicasCount := 2 + db.On("GetApplicationControllerReplicas").Return(replicasCount) t.Setenv(common.EnvControllerShardingAlgorithm, common.LegacyShardingAlgorithm) - filter := GetClusterFilter(db, GetDistributionFunction(db, common.LegacyShardingAlgorithm), shardIndex) - assert.False(t, filter(&v1alpha1.Cluster{ID: "1"})) - assert.True(t, filter(&v1alpha1.Cluster{ID: "2"})) - assert.False(t, filter(&v1alpha1.Cluster{ID: "3"})) - assert.True(t, filter(&v1alpha1.Cluster{ID: "4"})) + distributionFunction := RoundRobinDistributionFunction(clusterAccessor, replicasCount) + assert.Equal(t, 0, distributionFunction(nil)) + assert.Equal(t, 0, distributionFunction(&cluster1)) + assert.Equal(t, 1, distributionFunction(&cluster2)) + assert.Equal(t, 0, distributionFunction(&cluster3)) + assert.Equal(t, 1, distributionFunction(&cluster4)) } func TestGetClusterFilterUnknown(t *testing.T) { - shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) - db := &dbmocks.ArgoDB{} - db.On("GetApplicationControllerReplicas").Return(2) + clusterAccessor, db, cluster1, cluster2, cluster3, cluster4, _ := createTestClusters() + appAccessor, _, _, _, _, _ := createTestApps() + // Test with replicas set to 0 + t.Setenv(common.EnvControllerReplicas, "2") + os.Unsetenv(common.EnvControllerShardingAlgorithm) t.Setenv(common.EnvControllerShardingAlgorithm, "unknown") - filter := GetClusterFilter(db, GetDistributionFunction(db, "unknown"), shardIndex) - assert.False(t, filter(&v1alpha1.Cluster{ID: "1"})) - assert.True(t, filter(&v1alpha1.Cluster{ID: "2"})) - assert.False(t, filter(&v1alpha1.Cluster{ID: "3"})) - assert.True(t, filter(&v1alpha1.Cluster{ID: "4"})) + replicasCount := 2 + db.On("GetApplicationControllerReplicas").Return(replicasCount) + distributionFunction := GetDistributionFunction(clusterAccessor, appAccessor, "unknown", replicasCount) + assert.Equal(t, 0, distributionFunction(nil)) + assert.Equal(t, 0, distributionFunction(&cluster1)) + assert.Equal(t, 1, distributionFunction(&cluster2)) + assert.Equal(t, 0, distributionFunction(&cluster3)) + assert.Equal(t, 1, distributionFunction(&cluster4)) } func TestLegacyGetClusterFilterWithFixedShard(t *testing.T) { - shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) - db := &dbmocks.ArgoDB{} - db.On("GetApplicationControllerReplicas").Return(2) - filter := GetClusterFilter(db, GetDistributionFunction(db, common.DefaultShardingAlgorithm), shardIndex) - assert.False(t, filter(nil)) - assert.False(t, filter(&v1alpha1.Cluster{ID: "1"})) - assert.True(t, filter(&v1alpha1.Cluster{ID: "2"})) - assert.False(t, filter(&v1alpha1.Cluster{ID: "3"})) - assert.True(t, filter(&v1alpha1.Cluster{ID: "4"})) + //shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) + t.Setenv(common.EnvControllerReplicas, "5") + clusterAccessor, db, cluster1, cluster2, cluster3, cluster4, _ := createTestClusters() + appAccessor, _, _, _, _, _ := createTestApps() + replicasCount := 5 + db.On("GetApplicationControllerReplicas").Return(replicasCount) + filter := GetDistributionFunction(clusterAccessor, appAccessor, common.DefaultShardingAlgorithm, replicasCount) + assert.Equal(t, 0, filter(nil)) + assert.Equal(t, 4, filter(&cluster1)) + assert.Equal(t, 1, filter(&cluster2)) + assert.Equal(t, 2, filter(&cluster3)) + assert.Equal(t, 2, filter(&cluster4)) var fixedShard int64 = 4 - filter = GetClusterFilter(db, GetDistributionFunction(db, common.DefaultShardingAlgorithm), int(fixedShard)) - assert.False(t, filter(&v1alpha1.Cluster{ID: "4", Shard: &fixedShard})) + cluster5 := &v1alpha1.Cluster{ID: "5", Shard: &fixedShard} + clusterAccessor = getClusterAccessor([]v1alpha1.Cluster{cluster1, cluster2, cluster2, cluster4, *cluster5}) + filter = GetDistributionFunction(clusterAccessor, appAccessor, common.DefaultShardingAlgorithm, replicasCount) + assert.Equal(t, int(fixedShard), filter(cluster5)) fixedShard = 1 - filter = GetClusterFilter(db, GetDistributionFunction(db, common.DefaultShardingAlgorithm), int(fixedShard)) - assert.True(t, filter(&v1alpha1.Cluster{Name: "cluster4", ID: "4", Shard: &fixedShard})) + cluster5.Shard = &fixedShard + clusterAccessor = getClusterAccessor([]v1alpha1.Cluster{cluster1, cluster2, cluster2, cluster4, *cluster5}) + filter = GetDistributionFunction(clusterAccessor, appAccessor, common.DefaultShardingAlgorithm, replicasCount) + assert.Equal(t, int(fixedShard), filter(&v1alpha1.Cluster{ID: "4", Shard: &fixedShard})) } func TestRoundRobinGetClusterFilterWithFixedShard(t *testing.T) { - shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) - db, cluster1, cluster2, cluster3, cluster4, _ := createTestClusters() - db.On("GetApplicationControllerReplicas").Return(2) - filter := GetClusterFilter(db, GetDistributionFunction(db, common.RoundRobinShardingAlgorithm), shardIndex) - assert.False(t, filter(nil)) - assert.False(t, filter(&cluster1)) - assert.True(t, filter(&cluster2)) - assert.False(t, filter(&cluster3)) - assert.True(t, filter(&cluster4)) - - // a cluster with a fixed shard should be processed by the specified exact - // same shard unless the specified shard index is greater than the number of replicas. - var fixedShard int64 = 4 - filter = GetClusterFilter(db, GetDistributionFunction(db, common.RoundRobinShardingAlgorithm), int(fixedShard)) - assert.False(t, filter(&v1alpha1.Cluster{Name: "cluster4", ID: "4", Shard: &fixedShard})) - - fixedShard = 1 - filter = GetClusterFilter(db, GetDistributionFunction(db, common.RoundRobinShardingAlgorithm), int(fixedShard)) - assert.True(t, filter(&v1alpha1.Cluster{Name: "cluster4", ID: "4", Shard: &fixedShard})) -} - -func TestGetClusterFilterLegacyHash(t *testing.T) { - shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) - t.Setenv(common.EnvControllerShardingAlgorithm, "hash") - db, cluster1, cluster2, cluster3, cluster4, _ := createTestClusters() - db.On("GetApplicationControllerReplicas").Return(2) - filter := GetClusterFilter(db, GetDistributionFunction(db, common.LegacyShardingAlgorithm), shardIndex) - assert.False(t, filter(&cluster1)) - assert.True(t, filter(&cluster2)) - assert.False(t, filter(&cluster3)) - assert.True(t, filter(&cluster4)) + //shardIndex := 1 // ensuring that a shard with index 1 will process all the clusters with an "even" id (2,4,6,...) + t.Setenv(common.EnvControllerReplicas, "4") + clusterAccessor, db, cluster1, cluster2, cluster3, cluster4, _ := createTestClusters() + appAccessor, _, _, _, _, _ := createTestApps() + replicasCount := 4 + db.On("GetApplicationControllerReplicas").Return(replicasCount) + + filter := GetDistributionFunction(clusterAccessor, appAccessor, common.RoundRobinShardingAlgorithm, replicasCount) + assert.Equal(t, filter(nil), 0) + assert.Equal(t, filter(&cluster1), 0) + assert.Equal(t, filter(&cluster2), 1) + assert.Equal(t, filter(&cluster3), 2) + assert.Equal(t, filter(&cluster4), 3) // a cluster with a fixed shard should be processed by the specified exact // same shard unless the specified shard index is greater than the number of replicas. - var fixedShard int64 = 4 - filter = GetClusterFilter(db, GetDistributionFunction(db, common.LegacyShardingAlgorithm), int(fixedShard)) - assert.False(t, filter(&v1alpha1.Cluster{Name: "cluster4", ID: "4", Shard: &fixedShard})) + var fixedShard int64 = 1 + cluster5 := v1alpha1.Cluster{Name: "cluster5", ID: "5", Shard: &fixedShard} + clusters := []v1alpha1.Cluster{cluster1, cluster2, cluster3, cluster4, cluster5} + clusterAccessor = getClusterAccessor(clusters) + filter = GetDistributionFunction(clusterAccessor, appAccessor, common.RoundRobinShardingAlgorithm, replicasCount) + assert.Equal(t, int(fixedShard), filter(&cluster5)) fixedShard = 1 - filter = GetClusterFilter(db, GetDistributionFunction(db, common.LegacyShardingAlgorithm), int(fixedShard)) - assert.True(t, filter(&v1alpha1.Cluster{Name: "cluster4", ID: "4", Shard: &fixedShard})) -} - -func TestGetClusterFilterWithEnvControllerShardingAlgorithms(t *testing.T) { - db, cluster1, cluster2, cluster3, cluster4, _ := createTestClusters() - shardIndex := 1 - db.On("GetApplicationControllerReplicas").Return(2) - - t.Run("legacy", func(t *testing.T) { - t.Setenv(common.EnvControllerShardingAlgorithm, common.LegacyShardingAlgorithm) - shardShouldProcessCluster := GetClusterFilter(db, GetDistributionFunction(db, common.LegacyShardingAlgorithm), shardIndex) - assert.False(t, shardShouldProcessCluster(&cluster1)) - assert.True(t, shardShouldProcessCluster(&cluster2)) - assert.False(t, shardShouldProcessCluster(&cluster3)) - assert.True(t, shardShouldProcessCluster(&cluster4)) - assert.False(t, shardShouldProcessCluster(nil)) - }) - - t.Run("roundrobin", func(t *testing.T) { - t.Setenv(common.EnvControllerShardingAlgorithm, common.RoundRobinShardingAlgorithm) - shardShouldProcessCluster := GetClusterFilter(db, GetDistributionFunction(db, common.LegacyShardingAlgorithm), shardIndex) - assert.False(t, shardShouldProcessCluster(&cluster1)) - assert.True(t, shardShouldProcessCluster(&cluster2)) - assert.False(t, shardShouldProcessCluster(&cluster3)) - assert.True(t, shardShouldProcessCluster(&cluster4)) - assert.False(t, shardShouldProcessCluster(nil)) - }) + cluster5 = v1alpha1.Cluster{Name: "cluster5", ID: "5", Shard: &fixedShard} + clusters = []v1alpha1.Cluster{cluster1, cluster2, cluster3, cluster4, cluster5} + clusterAccessor = getClusterAccessor(clusters) + filter = GetDistributionFunction(clusterAccessor, appAccessor, common.RoundRobinShardingAlgorithm, replicasCount) + assert.Equal(t, int(fixedShard), filter(&v1alpha1.Cluster{Name: "cluster4", ID: "4", Shard: &fixedShard})) } func TestGetShardByIndexModuloReplicasCountDistributionFunction2(t *testing.T) { - db, cluster1, cluster2, cluster3, cluster4, cluster5 := createTestClusters() + clusters, db, cluster1, cluster2, cluster3, cluster4, cluster5 := createTestClusters() t.Run("replicas set to 1", func(t *testing.T) { - db.On("GetApplicationControllerReplicas").Return(1).Once() - distributionFunction := RoundRobinDistributionFunction(db) + replicasCount := 1 + db.On("GetApplicationControllerReplicas").Return(replicasCount).Once() + distributionFunction := RoundRobinDistributionFunction(clusters, replicasCount) assert.Equal(t, 0, distributionFunction(nil)) assert.Equal(t, 0, distributionFunction(&cluster1)) assert.Equal(t, 0, distributionFunction(&cluster2)) @@ -205,8 +192,9 @@ func TestGetShardByIndexModuloReplicasCountDistributionFunction2(t *testing.T) { }) t.Run("replicas set to 2", func(t *testing.T) { - db.On("GetApplicationControllerReplicas").Return(2).Once() - distributionFunction := RoundRobinDistributionFunction(db) + replicasCount := 2 + db.On("GetApplicationControllerReplicas").Return(replicasCount).Once() + distributionFunction := RoundRobinDistributionFunction(clusters, replicasCount) assert.Equal(t, 0, distributionFunction(nil)) assert.Equal(t, 0, distributionFunction(&cluster1)) assert.Equal(t, 1, distributionFunction(&cluster2)) @@ -216,8 +204,9 @@ func TestGetShardByIndexModuloReplicasCountDistributionFunction2(t *testing.T) { }) t.Run("replicas set to 3", func(t *testing.T) { - db.On("GetApplicationControllerReplicas").Return(3).Once() - distributionFunction := RoundRobinDistributionFunction(db) + replicasCount := 3 + db.On("GetApplicationControllerReplicas").Return(replicasCount).Once() + distributionFunction := RoundRobinDistributionFunction(clusters, replicasCount) assert.Equal(t, 0, distributionFunction(nil)) assert.Equal(t, 0, distributionFunction(&cluster1)) assert.Equal(t, 1, distributionFunction(&cluster2)) @@ -233,17 +222,19 @@ func TestGetShardByIndexModuloReplicasCountDistributionFunctionWhenClusterNumber // Initial tests where showing that under 1024 clusters, execution time was around 400ms // and for 4096 clusters, execution time was under 9s // The other implementation was giving almost linear time of 400ms up to 10'000 clusters - db := dbmocks.ArgoDB{} - clusterList := &v1alpha1.ClusterList{Items: []v1alpha1.Cluster{}} + clusterPointers := []*v1alpha1.Cluster{} for i := 0; i < 2048; i++ { cluster := createCluster(fmt.Sprintf("cluster-%d", i), fmt.Sprintf("%d", i)) - clusterList.Items = append(clusterList.Items, cluster) + clusterPointers = append(clusterPointers, &cluster) } - db.On("ListClusters", mock.Anything).Return(clusterList, nil) - db.On("GetApplicationControllerReplicas").Return(2) - distributionFunction := RoundRobinDistributionFunction(&db) - for i, c := range clusterList.Items { - assert.Equal(t, i%2, distributionFunction(&c)) + replicasCount := 2 + t.Setenv(common.EnvControllerReplicas, strconv.Itoa(replicasCount)) + _, db, _, _, _, _, _ := createTestClusters() + clusterAccessor := func() []*v1alpha1.Cluster { return clusterPointers } + db.On("GetApplicationControllerReplicas").Return(replicasCount) + distributionFunction := RoundRobinDistributionFunction(clusterAccessor, replicasCount) + for i, c := range clusterPointers { + assert.Equal(t, i%2, distributionFunction(c)) } } @@ -256,12 +247,15 @@ func TestGetShardByIndexModuloReplicasCountDistributionFunctionWhenClusterIsAdde cluster5 := createCluster("cluster5", "5") cluster6 := createCluster("cluster6", "6") + clusters := []v1alpha1.Cluster{cluster1, cluster2, cluster3, cluster4, cluster5} + clusterAccessor := getClusterAccessor(clusters) + clusterList := &v1alpha1.ClusterList{Items: []v1alpha1.Cluster{cluster1, cluster2, cluster3, cluster4, cluster5}} db.On("ListClusters", mock.Anything).Return(clusterList, nil) - // Test with replicas set to 2 - db.On("GetApplicationControllerReplicas").Return(2) - distributionFunction := RoundRobinDistributionFunction(&db) + replicasCount := 2 + db.On("GetApplicationControllerReplicas").Return(replicasCount) + distributionFunction := RoundRobinDistributionFunction(clusterAccessor, replicasCount) assert.Equal(t, 0, distributionFunction(nil)) assert.Equal(t, 0, distributionFunction(&cluster1)) assert.Equal(t, 1, distributionFunction(&cluster2)) @@ -272,17 +266,20 @@ func TestGetShardByIndexModuloReplicasCountDistributionFunctionWhenClusterIsAdde // Now, the database knows cluster6. Shard should be assigned a proper shard clusterList.Items = append(clusterList.Items, cluster6) + distributionFunction = RoundRobinDistributionFunction(getClusterAccessor(clusterList.Items), replicasCount) assert.Equal(t, 1, distributionFunction(&cluster6)) // Now, we remove the last added cluster, it should be unassigned as well clusterList.Items = clusterList.Items[:len(clusterList.Items)-1] + distributionFunction = RoundRobinDistributionFunction(getClusterAccessor(clusterList.Items), replicasCount) assert.Equal(t, -1, distributionFunction(&cluster6)) } func TestGetShardByIndexModuloReplicasCountDistributionFunction(t *testing.T) { - db, cluster1, cluster2, _, _, _ := createTestClusters() - db.On("GetApplicationControllerReplicas").Return(2) - distributionFunction := RoundRobinDistributionFunction(db) + clusters, db, cluster1, cluster2, _, _, _ := createTestClusters() + replicasCount := 2 + db.On("GetApplicationControllerReplicas").Return(replicasCount) + distributionFunction := RoundRobinDistributionFunction(clusters, replicasCount) // Test that the function returns the correct shard for cluster1 and cluster2 expectedShardForCluster1 := 0 @@ -315,14 +312,14 @@ func TestInferShard(t *testing.T) { osHostnameFunction = func() (string, error) { return "exampleshard", nil } _, err = InferShard() - assert.NotNil(t, err) + assert.Nil(t, err) osHostnameFunction = func() (string, error) { return "example-shard", nil } _, err = InferShard() - assert.NotNil(t, err) + assert.Nil(t, err) } -func createTestClusters() (*dbmocks.ArgoDB, v1alpha1.Cluster, v1alpha1.Cluster, v1alpha1.Cluster, v1alpha1.Cluster, v1alpha1.Cluster) { +func createTestClusters() (clusterAccessor, *dbmocks.ArgoDB, v1alpha1.Cluster, v1alpha1.Cluster, v1alpha1.Cluster, v1alpha1.Cluster, v1alpha1.Cluster) { db := dbmocks.ArgoDB{} cluster1 := createCluster("cluster1", "1") cluster2 := createCluster("cluster2", "2") @@ -330,10 +327,27 @@ func createTestClusters() (*dbmocks.ArgoDB, v1alpha1.Cluster, v1alpha1.Cluster, cluster4 := createCluster("cluster4", "4") cluster5 := createCluster("cluster5", "5") + clusters := []v1alpha1.Cluster{cluster1, cluster2, cluster3, cluster4, cluster5} + db.On("ListClusters", mock.Anything).Return(&v1alpha1.ClusterList{Items: []v1alpha1.Cluster{ cluster1, cluster2, cluster3, cluster4, cluster5, }}, nil) - return &db, cluster1, cluster2, cluster3, cluster4, cluster5 + return getClusterAccessor(clusters), &db, cluster1, cluster2, cluster3, cluster4, cluster5 +} + +func getClusterAccessor(clusters []v1alpha1.Cluster) clusterAccessor { + // Convert the array to a slice of pointers + clusterPointers := getClusterPointers(clusters) + clusterAccessor := func() []*v1alpha1.Cluster { return clusterPointers } + return clusterAccessor +} + +func getClusterPointers(clusters []v1alpha1.Cluster) []*v1alpha1.Cluster { + var clusterPointers []*v1alpha1.Cluster + for i := range clusters { + clusterPointers = append(clusterPointers, &clusters[i]) + } + return clusterPointers } func createCluster(name string, id string) v1alpha1.Cluster { @@ -676,3 +690,265 @@ func Test_getOrUpdateShardNumberForController(t *testing.T) { }) } } + +func TestGetClusterSharding(t *testing.T) { + IntPtr := func(i int32) *int32 { + return &i + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.DefaultApplicationControllerName, + Namespace: "argocd", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: IntPtr(1), + }, + } + + deploymentMultiReplicas := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "argocd-application-controller-multi-replicas", + Namespace: "argocd", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: IntPtr(3), + }, + } + + objects := append([]runtime.Object{}, deployment, deploymentMultiReplicas) + kubeclientset := kubefake.NewSimpleClientset(objects...) + + settingsMgr := settings.NewSettingsManager(context.TODO(), kubeclientset, "argocd", settings.WithRepoOrClusterChangedHandler(func() { + })) + + testCases := []struct { + name string + useDynamicSharding bool + envsSetter func(t *testing.T) + cleanup func() + expectedShard int + expectedReplicas int + expectedErr error + }{ + { + name: "Default sharding with statefulset", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvControllerReplicas, "1") + }, + cleanup: func() {}, + useDynamicSharding: false, + expectedShard: 0, + expectedReplicas: 1, + expectedErr: nil, + }, + { + name: "Default sharding with deployment", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvAppControllerName, common.DefaultApplicationControllerName) + }, + cleanup: func() {}, + useDynamicSharding: true, + expectedShard: 0, + expectedReplicas: 1, + expectedErr: nil, + }, + { + name: "Default sharding with deployment and multiple replicas", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvAppControllerName, "argocd-application-controller-multi-replicas") + }, + cleanup: func() {}, + useDynamicSharding: true, + expectedShard: 0, + expectedReplicas: 3, + expectedErr: nil, + }, + { + name: "Statefulset multiple replicas", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvControllerReplicas, "3") + osHostnameFunction = func() (string, error) { return "example-shard-3", nil } + }, + cleanup: func() { + osHostnameFunction = os.Hostname + }, + useDynamicSharding: false, + expectedShard: 3, + expectedReplicas: 3, + expectedErr: nil, + }, + { + name: "Explicit shard with statefulset and 1 replica", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvControllerReplicas, "1") + t.Setenv(common.EnvControllerShard, "3") + }, + cleanup: func() {}, + useDynamicSharding: false, + expectedShard: 0, + expectedReplicas: 1, + expectedErr: nil, + }, + { + name: "Explicit shard with statefulset and 2 replica - and to high shard", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvControllerReplicas, "2") + t.Setenv(common.EnvControllerShard, "3") + }, + cleanup: func() {}, + useDynamicSharding: false, + expectedShard: 0, + expectedReplicas: 2, + expectedErr: nil, + }, + { + name: "Explicit shard with statefulset and 2 replica", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvControllerReplicas, "2") + t.Setenv(common.EnvControllerShard, "1") + }, + cleanup: func() {}, + useDynamicSharding: false, + expectedShard: 1, + expectedReplicas: 2, + expectedErr: nil, + }, + { + name: "Explicit shard with deployment", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvControllerShard, "3") + }, + cleanup: func() {}, + useDynamicSharding: true, + expectedShard: 0, + expectedReplicas: 1, + expectedErr: nil, + }, + { + name: "Explicit shard with deployment and multiple replicas will read from configmap", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvAppControllerName, "argocd-application-controller-multi-replicas") + t.Setenv(common.EnvControllerShard, "3") + }, + cleanup: func() {}, + useDynamicSharding: true, + expectedShard: 0, + expectedReplicas: 3, + expectedErr: nil, + }, + { + name: "Dynamic sharding but missing deployment", + envsSetter: func(t *testing.T) { + t.Setenv(common.EnvAppControllerName, "missing-deployment") + }, + cleanup: func() {}, + useDynamicSharding: true, + expectedShard: 0, + expectedReplicas: 1, + expectedErr: fmt.Errorf("(dynamic cluster distribution) failed to get app controller deployment: deployments.apps \"missing-deployment\" not found"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.envsSetter(t) + defer tc.cleanup() + shardingCache, err := GetClusterSharding(kubeclientset, settingsMgr, "round-robin", tc.useDynamicSharding) + + if shardingCache != nil { + clusterSharding := shardingCache.(*ClusterSharding) + assert.Equal(t, tc.expectedShard, clusterSharding.Shard) + assert.Equal(t, tc.expectedReplicas, clusterSharding.Replicas) + } + + if tc.expectedErr != nil { + if err != nil { + assert.Equal(t, tc.expectedErr.Error(), err.Error()) + } else { + t.Errorf("Expected error %v but got nil", tc.expectedErr) + } + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestAppAwareCache(t *testing.T) { + _, db, cluster1, cluster2, cluster3, cluster4, cluster5 := createTestClusters() + _, app1, app2, app3, app4, app5 := createTestApps() + + clusterSharding := NewClusterSharding(db, 0, 1, "legacy") + + clusterList := &v1alpha1.ClusterList{Items: []v1alpha1.Cluster{cluster1, cluster2, cluster3, cluster4, cluster5}} + appList := &v1alpha1.ApplicationList{Items: []v1alpha1.Application{app1, app2, app3, app4, app5}} + clusterSharding.Init(clusterList, appList) + + appDistribution := clusterSharding.GetAppDistribution() + + assert.Equal(t, 2, appDistribution["cluster1"]) + assert.Equal(t, 2, appDistribution["cluster2"]) + assert.Equal(t, 1, appDistribution["cluster3"]) + + app6 := createApp("app6", "cluster4") + clusterSharding.AddApp(&app6) + + app1Update := createApp("app1", "cluster2") + clusterSharding.UpdateApp(&app1Update) + + clusterSharding.DeleteApp(&app3) + + appDistribution = clusterSharding.GetAppDistribution() + + assert.Equal(t, 1, appDistribution["cluster1"]) + assert.Equal(t, 2, appDistribution["cluster2"]) + assert.Equal(t, 1, appDistribution["cluster3"]) + assert.Equal(t, 1, appDistribution["cluster4"]) +} + +func createTestApps() (appAccessor, v1alpha1.Application, v1alpha1.Application, v1alpha1.Application, v1alpha1.Application, v1alpha1.Application) { + app1 := createApp("app1", "cluster1") + app2 := createApp("app2", "cluster1") + app3 := createApp("app3", "cluster2") + app4 := createApp("app4", "cluster2") + app5 := createApp("app5", "cluster3") + + apps := []v1alpha1.Application{app1, app2, app3, app4, app5} + + return getAppAccessor(apps), app1, app2, app3, app4, app5 +} + +func getAppAccessor(apps []v1alpha1.Application) appAccessor { + // Convert the array to a slice of pointers + appPointers := getAppPointers(apps) + appAccessor := func() []*v1alpha1.Application { return appPointers } + return appAccessor +} + +func getAppPointers(apps []v1alpha1.Application) []*v1alpha1.Application { + var appPointers []*v1alpha1.Application + for i := range apps { + appPointers = append(appPointers, &apps[i]) + } + return appPointers +} + +func createApp(name string, server string) v1alpha1.Application { + var testApp = ` +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ` + name + ` +spec: + destination: + server: ` + server + ` +` + + var app v1alpha1.Application + err := yaml.Unmarshal([]byte(testApp), &app) + if err != nil { + panic(err) + } + return app +} diff --git a/controller/sharding/shuffle_test.go b/controller/sharding/shuffle_test.go index 9e089e31bad0f..1cca783a2afe9 100644 --- a/controller/sharding/shuffle_test.go +++ b/controller/sharding/shuffle_test.go @@ -3,6 +3,7 @@ package sharding import ( "fmt" "math" + "strconv" "testing" "github.com/argoproj/argo-cd/v2/common" @@ -22,9 +23,11 @@ func TestLargeShuffle(t *testing.T) { clusterList.Items = append(clusterList.Items, cluster) } db.On("ListClusters", mock.Anything).Return(clusterList, nil) + clusterAccessor := getClusterAccessor(clusterList.Items) // Test with replicas set to 256 - t.Setenv(common.EnvControllerReplicas, "256") - distributionFunction := RoundRobinDistributionFunction(&db) + replicasCount := 256 + t.Setenv(common.EnvControllerReplicas, strconv.Itoa(replicasCount)) + distributionFunction := RoundRobinDistributionFunction(clusterAccessor, replicasCount) for i, c := range clusterList.Items { assert.Equal(t, i%2567, distributionFunction(&c)) } @@ -44,10 +47,11 @@ func TestShuffle(t *testing.T) { clusterList := &v1alpha1.ClusterList{Items: []v1alpha1.Cluster{cluster1, cluster2, cluster3, cluster4, cluster5, cluster6}} db.On("ListClusters", mock.Anything).Return(clusterList, nil) - + clusterAccessor := getClusterAccessor(clusterList.Items) // Test with replicas set to 3 t.Setenv(common.EnvControllerReplicas, "3") - distributionFunction := RoundRobinDistributionFunction(&db) + replicasCount := 3 + distributionFunction := RoundRobinDistributionFunction(clusterAccessor, replicasCount) assert.Equal(t, 0, distributionFunction(nil)) assert.Equal(t, 0, distributionFunction(&cluster1)) assert.Equal(t, 1, distributionFunction(&cluster2)) diff --git a/controller/state_test.go b/controller/state_test.go index 1a55e25b262d1..d21cda62137de 100644 --- a/controller/state_test.go +++ b/controller/state_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "github.com/argoproj/argo-cd/v2/common" + "github.com/argoproj/argo-cd/v2/controller/testdata" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" argoappv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/argoproj/argo-cd/v2/reposerver/apiclient" @@ -1509,6 +1510,17 @@ func TestUseDiffCache(t *testing.T) { expectedUseCache: true, serverSideDiff: false, }, + { + testName: "will use diff cache with sync policy", + noCache: false, + manifestInfos: manifestInfos("rev1"), + sources: sources(), + app: test.YamlToApplication(testdata.DiffCacheYaml), + manifestRevisions: []string{"rev1"}, + statusRefreshTimeout: time.Hour * 24, + expectedUseCache: true, + serverSideDiff: true, + }, { testName: "will use diff cache for multisource", noCache: false, diff --git a/controller/sync.go b/controller/sync.go index 2d21bf1cb1190..34c12bdb5da3c 100644 --- a/controller/sync.go +++ b/controller/sync.go @@ -103,7 +103,7 @@ func (m *appStateManager) SyncAppState(app *v1alpha1.Application, state *v1alpha if syncOp.SyncOptions.HasOption("FailOnSharedResource=true") && hasSharedResource { state.Phase = common.OperationFailed - state.Message = fmt.Sprintf("Shared resouce found: %s", sharedResourceMessage) + state.Message = fmt.Sprintf("Shared resource found: %s", sharedResourceMessage) return } diff --git a/controller/testdata/data.go b/controller/testdata/data.go index a53c6a8a88b35..028a7caaeac6b 100644 --- a/controller/testdata/data.go +++ b/controller/testdata/data.go @@ -11,4 +11,7 @@ var ( //go:embed target-deployment-new-entries.yaml TargetDeploymentNewEntries string + + //go:embed diff-cache.yaml + DiffCacheYaml string ) diff --git a/controller/testdata/diff-cache.yaml b/controller/testdata/diff-cache.yaml new file mode 100644 index 0000000000000..41a1e8a4bbeb1 --- /dev/null +++ b/controller/testdata/diff-cache.yaml @@ -0,0 +1,498 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + annotations: + argocd-image-updater.argoproj.io/allow-tags: any + argocd-image-updater.argoproj.io/ignore-tags: "" + argocd-image-updater.argoproj.io/image-list-disabled-hack: "" + argocd-image-updater.argoproj.io/update-strategy: semver + argocd-image-updater.argoproj.io/write-back-method: git + argocd-image-updater.argoproj.io/write-back-target: kustomization + argocd-notif-onDeployed.slack-disabled: "" + argocd-notif-onHealthDegraded.slack-disabled: "" + argocd-notif-onSyncFailed.slack-disabled: "" + argocd-notif-onSyncRunning.slack-disabled: "" + argocd-notif-onSyncStatusUnknown.slack-disabled: "" + argocd-notif-onSyncSucceeded.slack-disabled: "" + argocd.argoproj.io/compare-options: ServerSideDiff=true + argocd.argoproj.io/manifest-generate-paths: .;/chart + creationTimestamp: "2024-03-04T21:30:33Z" + finalizers: + - resources-finalizer.argocd.argoproj.io + generation: 263 + labels: + cloud_provider: gcp + cluster_name: gke-alpha-01-europe-west1 + foo: bar + preview: "true" + project: sre + service_class: alpha + stack: gke-v2 + name: velero-test + namespace: argo-cd + ownerReferences: + - apiVersion: argoproj.io/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: ApplicationSet + name: velero + uid: 86cdfba4-8697-47b3-8489-71fab7f4a805 + resourceVersion: "722811357" + uid: 94978696-4fd4-40b3-a1de-38d9df9e9316 +spec: + destination: + name: gke-alpha-01-europe-west1 + namespace: test-lla + project: sre + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + syncPolicy: + retry: + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + limit: 10 + syncOptions: + - CreateNamespace=true + - ApplyOutOfSyncOnly=true + - RespectIgnoreDifferences=false + - ServerSideApply=true + - Validate=true +status: + controllerNamespace: argo-cd + health: + status: Healthy + history: + - deployStartedAt: "2024-03-04T22:00:05Z" + deployedAt: "2024-03-04T22:00:06Z" + id: 14 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-04T22:08:29Z" + deployedAt: "2024-03-04T22:08:30Z" + id: 15 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-04T22:09:16Z" + deployedAt: "2024-03-04T22:09:16Z" + id: 16 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-04T22:11:41Z" + deployedAt: "2024-03-04T22:11:41Z" + id: 17 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-04T22:50:55Z" + deployedAt: "2024-03-04T22:50:55Z" + id: 18 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-04T22:52:56Z" + deployedAt: "2024-03-04T22:52:56Z" + id: 19 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-04T22:56:15Z" + deployedAt: "2024-03-04T22:56:15Z" + id: 20 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-05T07:31:56Z" + deployedAt: "2024-03-05T07:31:57Z" + id: 21 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-05T07:32:44Z" + deployedAt: "2024-03-05T07:32:44Z" + id: 22 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + - deployStartedAt: "2024-03-05T07:33:03Z" + deployedAt: "2024-03-05T07:33:04Z" + id: 23 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + operationState: + finishedAt: "2024-03-05T07:33:04Z" + message: successfully synced (all tasks run) + operation: + initiatedBy: + username: laurent.lavaud@mirakl.com + retry: + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + limit: 10 + sync: + revision: ea8759964626a583667a2bfd08f334ec2070040a + syncOptions: + - ServerSideApply=true + syncStrategy: + hook: {} + phase: Succeeded + startedAt: "2024-03-05T07:33:03Z" + syncResult: + resources: + - group: "" + hookPhase: Running + kind: Service + message: service/test-lla serverside-applied + name: test-lla + namespace: test-lla + status: Synced + syncPhase: Sync + version: v1 + - group: apps + hookPhase: Running + kind: Deployment + message: deployment.apps/test-lla serverside-applied + name: test-lla + namespace: test-lla + status: Synced + syncPhase: Sync + version: v1 + revision: ea8759964626a583667a2bfd08f334ec2070040a + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + reconciledAt: "2024-03-05T07:33:04Z" + resources: + - health: + status: Healthy + kind: Service + name: test-lla + namespace: test-lla + status: Synced + version: v1 + - group: apps + health: + status: Healthy + kind: Deployment + name: test-lla + namespace: test-lla + status: Synced + version: v1 + sourceType: Plugin + summary: + images: + - nginx:latest + sync: + comparedTo: + destination: + name: gke-alpha-01-europe-west1 + namespace: test-lla + source: + path: instances/test + plugin: + env: + - name: RELEASE_NAME + value: test-lla + - name: CHART_REPOSITORY + value: oci://europe-west1-docker.pkg.dev/platform-89be/charts + - name: CHART_NAME + value: velero + - name: PREVIEW + value: "false" + - name: HELM_VALUES + value: | + global: + app: + cluster_name: gke-alpha-01-europe-west1 + service_class: alpha + cloud_provider: gcp + cluster_stack: gke-v2 + - name: HELM_ARGS + value: "" + name: cmp-helm-v2 + repoURL: https://github.com/mirakl/manifests-velero.git + targetRevision: test-lla + revision: rev1 + status: Synced diff --git a/docs/assets/api-management.png b/docs/assets/api-management.png deleted file mode 100644 index ae066f0a6a87d..0000000000000 Binary files a/docs/assets/api-management.png and /dev/null differ diff --git a/docs/assets/groups-claim.png b/docs/assets/groups-claim.png deleted file mode 100644 index d27e03b661f82..0000000000000 Binary files a/docs/assets/groups-claim.png and /dev/null differ diff --git a/docs/assets/groups-scope.png b/docs/assets/groups-scope.png deleted file mode 100644 index 45557b51ead7f..0000000000000 Binary files a/docs/assets/groups-scope.png and /dev/null differ diff --git a/docs/assets/okta-app.png b/docs/assets/okta-app.png new file mode 100644 index 0000000000000..bfc4570826b0a Binary files /dev/null and b/docs/assets/okta-app.png differ diff --git a/docs/assets/okta-auth-policy.png b/docs/assets/okta-auth-policy.png new file mode 100644 index 0000000000000..dbf99a88ed6e3 Binary files /dev/null and b/docs/assets/okta-auth-policy.png differ diff --git a/docs/assets/okta-auth-rule.png b/docs/assets/okta-auth-rule.png new file mode 100644 index 0000000000000..4e85b062f357b Binary files /dev/null and b/docs/assets/okta-auth-rule.png differ diff --git a/docs/assets/okta-create-oidc-app.png b/docs/assets/okta-create-oidc-app.png new file mode 100644 index 0000000000000..cf0b75b0e4a21 Binary files /dev/null and b/docs/assets/okta-create-oidc-app.png differ diff --git a/docs/assets/okta-groups-claim.png b/docs/assets/okta-groups-claim.png new file mode 100644 index 0000000000000..4edb93d42ea91 Binary files /dev/null and b/docs/assets/okta-groups-claim.png differ diff --git a/docs/assets/okta-groups-scope.png b/docs/assets/okta-groups-scope.png new file mode 100644 index 0000000000000..6cd1783c72653 Binary files /dev/null and b/docs/assets/okta-groups-scope.png differ diff --git a/docs/cli_installation.md b/docs/cli_installation.md index 42938bcd751ba..5a314d4ce6be2 100644 --- a/docs/cli_installation.md +++ b/docs/cli_installation.md @@ -37,6 +37,17 @@ sudo install -m 555 argocd-linux-amd64 /usr/local/bin/argocd rm argocd-linux-amd64 ``` +#### Download latest stable version + +You can download the latest stable release by executing below steps: + +```bash +VERSION=$(curl -L -s https://raw.githubusercontent.com/argoproj/argo-cd/stable/VERSION) +curl -sSL -o argocd-linux-amd64 https://github.com/argoproj/argo-cd/releases/download/v$VERSION/argocd-linux-amd64 +sudo install -m 555 argocd-linux-amd64 /usr/local/bin/argocd +rm argocd-linux-amd64 +``` + You should now be able to run `argocd` commands. diff --git a/docs/developer-guide/contributors-quickstart.md b/docs/developer-guide/contributors-quickstart.md index a7646a6cf5f25..68cda35b6d08e 100644 --- a/docs/developer-guide/contributors-quickstart.md +++ b/docs/developer-guide/contributors-quickstart.md @@ -11,7 +11,7 @@ and the [toolchain guide](toolchain-guide.md). -Install version 1.18 or newer (Verify version by running `go version`) +Install Go with a version equal to or greater than the version listed in `go.mod` (verify go version with `go version`). ### Clone the Argo CD repo diff --git a/docs/developer-guide/site.md b/docs/developer-guide/site.md index af32753a323e2..efd6aece9aedb 100644 --- a/docs/developer-guide/site.md +++ b/docs/developer-guide/site.md @@ -7,20 +7,14 @@ The website is built using `mkdocs` and `mkdocs-material`. To test: ```bash +make build-docs make serve-docs ``` - Once running, you can view your locally built documentation at [http://0.0.0.0:8000/](http://0.0.0.0:8000/). -## Deploying - -```bash -make publish-docs -``` - ## Analytics !!! tip Don't forget to disable your ad-blocker when testing. -We collect [Google Analytics](https://analytics.google.com/analytics/web/#/report-home/a105170809w198079555p192782995). \ No newline at end of file +We collect [Google Analytics](https://analytics.google.com/analytics/web/#/report-home/a105170809w198079555p192782995). diff --git a/docs/developer-guide/toolchain-guide.md b/docs/developer-guide/toolchain-guide.md index 42ca7fac87404..335180438dac6 100644 --- a/docs/developer-guide/toolchain-guide.md +++ b/docs/developer-guide/toolchain-guide.md @@ -304,7 +304,7 @@ For installing the tools required to build and test Argo CD on your local system You can change the target location by setting the `BIN` environment before running the installer scripts. For example, you can install the binaries into `~/go/bin` (which should then be the first component in your `PATH` environment, i.e. `export PATH=~/go/bin:$PATH`): ```shell -make BIN=~/go/bin install-tools-local +BIN=~/go/bin make install-tools-local ``` Additionally, you have to install at least the following tools via your OS's package manager (this list might not be always up-to-date): diff --git a/docs/faq.md b/docs/faq.md index 95205ae6bbb4e..83bdf8d7d38b5 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -36,7 +36,7 @@ which might cause health check to return `Progressing` state instead of `Healthy As workaround Argo CD allows providing [health check](operator-manual/health.md) customization which overrides default behavior. -If you are using Traefik for your Ingress, you can update the Traefik config to publish the loadBalancer IP using [publishedservice](https://doc.traefik.io/traefik/providers/kubernetes-ingress/#publishedservice), which will resolve this issue. +If you are using Traefik for your Ingress, you can update the Traefik config to publish the loadBalancer IP using [publishedservice](https://doc.traefik.io/traefik/providers/kubernetes-ingress/#publishedservice), which will resolve this issue. ```yaml providers: @@ -97,7 +97,7 @@ data: ## After deploying my Helm application with Argo CD I cannot see it with `helm ls` and other Helm commands -When deploying a Helm application Argo CD is using Helm +When deploying a Helm application Argo CD is using Helm only as a template mechanism. It runs `helm template` and then deploys the resulting manifests on the cluster instead of doing `helm install`. This means that you cannot use any Helm command to view/verify the application. It is fully managed by Argo CD. @@ -140,15 +140,15 @@ Argo CD automatically sets the `app.kubernetes.io/instance` label and uses it to If the tool does this too, this causes confusion. You can change this label by setting the `application.instanceLabelKey` value in the `argocd-cm`. We recommend that you use `argocd.argoproj.io/instance`. -!!! note +!!! note When you make this change your applications will become out of sync and will need re-syncing. See [#1482](https://github.com/argoproj/argo-cd/issues/1482). ## How often does Argo CD check for changes to my Git or Helm repository ? -The default polling interval is 3 minutes (180 seconds). -You can change the setting by updating the `timeout.reconciliation` value in the [argocd-cm](https://github.com/argoproj/argo-cd/blob/2d6ce088acd4fb29271ffb6f6023dbb27594d59b/docs/operator-manual/argocd-cm.yaml#L279-L282) config map. If there are any Git changes, Argo CD will only update applications with the [auto-sync setting](user-guide/auto_sync.md) enabled. If you set it to `0` then Argo CD will stop polling Git repositories automatically and you can only use alternative methods such as [webhooks](operator-manual/webhook.md) and/or manual syncs for deploying applications. +The default polling interval is 3 minutes (180 seconds) with a configurable jitter. +You can change the setting by updating the `timeout.reconciliation` value and the `timeout.reconciliation.jitter` in the [argocd-cm](https://github.com/argoproj/argo-cd/blob/2d6ce088acd4fb29271ffb6f6023dbb27594d59b/docs/operator-manual/argocd-cm.yaml#L279-L282) config map. If there are any Git changes, Argo CD will only update applications with the [auto-sync setting](user-guide/auto_sync.md) enabled. If you set it to `0` then Argo CD will stop polling Git repositories automatically and you can only use alternative methods such as [webhooks](operator-manual/webhook.md) and/or manual syncs for deploying applications. ## Why Are My Resource Limits `Out Of Sync`? @@ -250,7 +250,7 @@ There are two parts to the message: > map[name:**KEY_BC** value:150] map[name:**KEY_BC** value:500] map[name:**KEY_BD** value:250] map[name:**KEY_BD** value:500] map[name:KEY_BI value:something] - You'll want to identify the keys that are duplicated -- you can focus on the first part, as each duplicated key will appear, once for each of its value with its value in the first list. The second list is really just + You'll want to identify the keys that are duplicated -- you can focus on the first part, as each duplicated key will appear, once for each of its value with its value in the first list. The second list is really just `]` @@ -259,7 +259,7 @@ There are two parts to the message: This includes all of the keys. It's included for debugging purposes -- you don't need to pay much attention to it. It will give you a hint about the precise location in the list for the duplicated keys: > map[name:KEY_AA] map[name:KEY_AB] map[name:KEY_AC] map[name:KEY_AD] map[name:KEY_AE] map[name:KEY_AF] map[name:KEY_AG] map[name:KEY_AH] map[name:KEY_AI] map[name:KEY_AJ] map[name:KEY_AK] map[name:KEY_AL] map[name:KEY_AM] map[name:KEY_AN] map[name:KEY_AO] map[name:KEY_AP] map[name:KEY_AQ] map[name:KEY_AR] map[name:KEY_AS] map[name:KEY_AT] map[name:KEY_AU] map[name:KEY_AV] map[name:KEY_AW] map[name:KEY_AX] map[name:KEY_AY] map[name:KEY_AZ] map[name:KEY_BA] map[name:KEY_BB] map[name:**KEY_BC**] map[name:**KEY_BD**] map[name:KEY_BE] map[name:KEY_BF] map[name:KEY_BG] map[name:KEY_BH] map[name:KEY_BI] map[name:**KEY_BC**] map[name:**KEY_BD**] - + `]` In this case, the duplicated keys have been **emphasized** to help you identify the problematic keys. Many editors have the ability to highlight all instances of a string, using such an editor can help with such problems. diff --git a/docs/operator-manual/application.yaml b/docs/operator-manual/application.yaml index aa2dea5c65b7c..864a293ce6890 100644 --- a/docs/operator-manual/application.yaml +++ b/docs/operator-manual/application.yaml @@ -119,7 +119,7 @@ spec: extVars: - name: foo value: bar - # You can use "code to determine if the value is either string (false, the default) or Jsonnet code (if code is true). + # You can use "code" to determine if the value is either string (false, the default) or Jsonnet code (if code is true). - code: true name: baz value: "true" diff --git a/docs/operator-manual/applicationset/Appset-Any-Namespace.md b/docs/operator-manual/applicationset/Appset-Any-Namespace.md index bf3f8ffecfaf1..4e28bc3a8172d 100644 --- a/docs/operator-manual/applicationset/Appset-Any-Namespace.md +++ b/docs/operator-manual/applicationset/Appset-Any-Namespace.md @@ -72,7 +72,7 @@ data: The allow-list only applies to SCM providers for which the user may configure a custom `api`. Where an SCM or PR generator does not accept a custom API URL, the provider is implicitly allowed. -If you do not intend to allow users to use the SCM or PR generators, you can disable them entirely by setting the environment variable `ARGOCD_APPLICATIONSET_CONTROLLER_ALLOW_SCM_PROVIDERS` to argocd-cmd-params-cm `applicationsetcontroller.allow.scm.providers` to `false`. +If you do not intend to allow users to use the SCM or PR generators, you can disable them entirely by setting the environment variable `ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_SCM_PROVIDERS` to argocd-cmd-params-cm `applicationsetcontroller.enable.scm.providers` to `false`. ### Overview diff --git a/docs/operator-manual/applicationset/Generators-Cluster.md b/docs/operator-manual/applicationset/Generators-Cluster.md index ca1a49aad295b..aa18983fe3d54 100644 --- a/docs/operator-manual/applicationset/Generators-Cluster.md +++ b/docs/operator-manual/applicationset/Generators-Cluster.md @@ -136,6 +136,29 @@ However, if you do wish to target both local and non-local clusters, while also These steps might seem counterintuitive, but the act of changing one of the default values for the local cluster causes the Argo CD Web UI to create a new secret for this cluster. In the Argo CD namespace, you should now see a Secret resource named `cluster-(cluster suffix)` with label `argocd.argoproj.io/secret-type": "cluster"`. You may also create a local [cluster secret declaratively](../../declarative-setup/#clusters), or with the CLI using `argocd cluster add "(context name)" --in-cluster`, rather than through the Web UI. +### Fetch clusters based on their K8s version + +There is also the possibility to fetch clusters based upon their Kubernetes version. To do this, the label `argocd.argoproj.io/auto-label-cluster-info` needs to be set to `true` on the cluster secret. +Once that has been set, the controller will dynamically label the cluster secret with the Kubernetes version it is running on. To retrieve that value, you need to use the +`argocd.argoproj.io/kubernetes-version`, as the example below demonstrates: + +```yaml +spec: + goTemplate: true + generators: + - clusters: + selector: + matchLabels: + argocd.argoproj.io/kubernetes-version: 1.28 + # matchExpressions are also supported. + #matchExpressions: + # - key: argocd.argoproj.io/kubernetes-version + # operator: In + # values: + # - "1.27" + # - "1.28" +``` + ### Pass additional key-value pairs via `values` field You may pass additional, arbitrary string key-value pairs via the `values` field of the cluster generator. Values added via the `values` field are added as `values.(field)` diff --git a/docs/operator-manual/applicationset/GoTemplate.md b/docs/operator-manual/applicationset/GoTemplate.md index 4a2b6cf55140b..1d62eeea9f93a 100644 --- a/docs/operator-manual/applicationset/GoTemplate.md +++ b/docs/operator-manual/applicationset/GoTemplate.md @@ -12,7 +12,7 @@ An additional `normalize` function makes any string parameter usable as a valid with hyphens and truncating at 253 characters. This is useful when making parameters safe for things like Application names. -Another function has `slugify` function has been added which, by default, sanitizes and smart truncate (means doesn't cut a word into 2). This function accepts a couple of arguments: +Another `slugify` function has been added which, by default, sanitizes and smart truncates (it doesn't cut a word into 2). This function accepts a couple of arguments: - The first argument (if provided) is an integer specifying the maximum length of the slug. - The second argument (if provided) is a boolean indicating whether smart truncation is enabled. - The last argument (if provided) is the input name that needs to be slugified. @@ -206,6 +206,8 @@ ApplicationSet controller provides: 1. contains no more than 253 characters 2. contains only lowercase alphanumeric characters, '-' or '.' 3. starts and ends with an alphanumeric character + +- `slugify`: sanitizes like `normalize` and smart truncates (it doesn't cut a word into 2) like described in the [introduction](#introduction) section. - `toYaml` / `fromYaml` / `fromYamlArray` helm like functions diff --git a/docs/operator-manual/applicationset/Template.md b/docs/operator-manual/applicationset/Template.md index 573e297bff2e2..ba8c196c32fa5 100644 --- a/docs/operator-manual/applicationset/Template.md +++ b/docs/operator-manual/applicationset/Template.md @@ -85,7 +85,7 @@ spec: spec: project: "default" source: - revision: HEAD + targetRevision: HEAD repoURL: https://github.com/argoproj/argo-cd.git # New path value is generated here: path: 'applicationset/examples/template-override/{{cluster}}-override' @@ -111,16 +111,15 @@ In this example, the ApplicationSet controller will generate an `Application` re ## Template Patch -Templating is only available on string type. However, some uses cases may require to apply templating on other types. +Templating is only available on string type. However, some use cases may require applying templating on other types. Example: -- Set the automated sync policy -- Switch prune boolean to true -- Add multiple helm value files - -Argo CD has a `templatePatch` feature to allow advanced templating. It supports both json and yaml. +- Conditionally set the automated sync policy. +- Conditionally switch prune boolean to `true`. +- Add multiple helm value files from a list. +The `templatePatch` feature enables advanced templating, with support for `json` and `yaml`. ```yaml apiVersion: argoproj.io/v1alpha1 @@ -174,3 +173,6 @@ spec: The `spec.project` field is not supported in `templatePatch`. If you need to change the project, you can use the `spec.project` field in the `template` field. + +!!! important + When writing a `templatePatch`, you're crafting a patch. So, if the patch includes an empty `spec: # nothing in here`, it will effectively clear out existing fields. See [#17040](https://github.com/argoproj/argo-cd/issues/17040) for an example of this behavior. diff --git a/docs/operator-manual/argocd-cm.yaml b/docs/operator-manual/argocd-cm.yaml index 5e4ed095be56d..a291a57a4c9dd 100644 --- a/docs/operator-manual/argocd-cm.yaml +++ b/docs/operator-manual/argocd-cm.yaml @@ -308,14 +308,22 @@ data: # have either a permanent banner or a regular closeable banner, and NOT both. eg. A user can't dismiss a # notification message (closeable) banner, to then immediately see a permanent banner. # ui.bannerpermanent: "true" - # An option to specify the position of the banner, either the top or bottom of the page. The default is at the top. - # Uncomment to make the banner appear at the bottom of the page. Any value other than "bottom" will make the banner appear at the top. + # An option to specify the position of the banner, either the top or bottom of the page, or both. The valid values + # are: "top", "bottom" and "both". The default (if the option is not provided), is "top". If "both" is specified, then + # the content appears both at the top and the bottom of the page. Uncomment the following line to make the banner appear + # at the bottom of the page. Change the value as needed. # ui.bannerposition: "bottom" # Application reconciliation timeout is the max amount of time required to discover if a new manifests version got # published to the repository. Reconciliation by timeout is disabled if timeout is set to 0. Three minutes by default. # > Note: argocd-repo-server deployment must be manually restarted after changing the setting. timeout.reconciliation: 180s + # With a large number of applications, the periodic refresh for each application can cause a spike in the refresh queue + # and can cause a spike in the repo-server component. To avoid this, you can set a jitter to the sync timeout, which will + # spread out the refreshes and give time to the repo-server to catch up. The jitter is the maximum duration that can be + # added to the sync timeout. So, if the sync timeout is 3 minutes and the jitter is 1 minute, then the actual timeout will + # be between 3 and 4 minutes. Disabled when the value is 0, defaults to 0. + timeout.reconciliation.jitter: 0 # cluster.inClusterEnabled indicates whether to allow in-cluster server address. This is enabled by default. cluster.inClusterEnabled: "true" @@ -405,4 +413,4 @@ data: # Mandatory if multiple services are specified. cluster: name: some-cluster - server: https://some-cluster + server: https://some-cluster \ No newline at end of file diff --git a/docs/operator-manual/argocd-cmd-params-cm.yaml b/docs/operator-manual/argocd-cmd-params-cm.yaml index dac955a9662de..3cb79d85f3150 100644 --- a/docs/operator-manual/argocd-cmd-params-cm.yaml +++ b/docs/operator-manual/argocd-cmd-params-cm.yaml @@ -90,6 +90,9 @@ data: server.k8sclient.retry.max: "0" # The initial backoff delay on the first retry attempt in ms. Subsequent retries will double this backoff time up to a maximum threshold server.k8sclient.retry.base.backoff: "100" + # Semicolon-separated list of content types allowed on non-GET requests. Set an empty string to allow all. Be aware + # that allowing content types besides application/json may make your API more vulnerable to CSRF attacks. + server.api.content.types: "application/json" # Set the logging format. One of: text|json (default "text") server.log.format: "text" diff --git a/docs/operator-manual/declarative-setup.md b/docs/operator-manual/declarative-setup.md index c1f5ba2b2d3bd..aec0877b21d02 100644 --- a/docs/operator-manual/declarative-setup.md +++ b/docs/operator-manual/declarative-setup.md @@ -549,6 +549,7 @@ bearerToken: string awsAuthConfig: clusterName: string roleARN: string + profile: string # Configure external command to supply client credentials # See https://godoc.org/k8s.io/client-go/tools/clientcmd/api#ExecConfig execProviderConfig: @@ -731,6 +732,140 @@ data: "rolearn": ":role/" "username": "" ``` + +#### Alternative EKS Authentication Methods +In some scenarios it may not be possible to use IRSA, such as when the Argo CD cluster is running on a different cloud +provider's platform. In this case, there are two options: +1. Use `execProviderConfig` to call the AWS authentication mechanism which enables the injection of environment variables to supply credentials +2. Leverage the new AWS profile option available in Argo CD release 2.10 + +Both of these options will require the steps involving IAM and the `aws-auth` config map (defined above) to provide the +principal with access to the cluster. + +##### Using execProviderConfig with Environment Variables +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: mycluster-secret + labels: + argocd.argoproj.io/secret-type: cluster +type: Opaque +stringData: + name: mycluster + server: https://mycluster.example.com + namespaces: "my,managed,namespaces" + clusterResources: "true" + config: | + { + "execProviderConfig": { + "command": "argocd-k8s-auth", + "args": ["aws", "--cluster-name", "my-eks-cluster"], + "apiVersion": "client.authentication.k8s.io/v1beta1", + "env": { + "AWS_REGION": "xx-east-1", + "AWS_ACCESS_KEY_ID": "{{ .aws_key_id }}", + "AWS_SECRET_ACCESS_KEY": "{{ .aws_key_secret }}", + "AWS_SESSION_TOKEN": "{{ .aws_token }}" + } + }, + "tlsClientConfig": { + "insecure": false, + "caData": "{{ .cluster_cert }}" + } + } +``` + +This example assumes that the role being attached to the credentials that have been supplied, if this is not the case +the role can be appended to the `args` section like so: + +```yaml +... + "args": ["aws", "--cluster-name", "my-eks-cluster", "--roleARN", "arn:aws:iam:::role/"], +... +``` +This construct can be used in conjunction with something like the External Secrets Operator to avoid storing the keys in +plain text and additionally helps to provide a foundation for key rotation. + +##### Using An AWS Profile For Authentication +The option to use profiles, added in release 2.10, provides a method for supplying credentials while still using the +standard Argo CD EKS cluster declaration with an additional command flag that points to an AWS credentials file: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: mycluster-secret + labels: + argocd.argoproj.io/secret-type: cluster +type: Opaque +stringData: + name: "mycluster.com" + server: "https://mycluster.com" + config: | + { + "awsAuthConfig": { + "clusterName": "my-eks-cluster-name", + "roleARN": "arn:aws:iam:::role/", + "profile": "/mount/path/to/my-profile-file" + }, + "tlsClientConfig": { + "insecure": false, + "caData": "" + } + } +``` +This will instruct ArgoCD to read the file at the provided path and use the credentials defined within to authenticate to +AWS. The profile must be mounted in order for this to work. For example, the following values can be defined in a Helm +based ArgoCD deployment: + +```yaml +controller: + extraVolumes: + - name: my-profile-volume + secret: + secretName: my-aws-profile + items: + - key: my-profile-file + path: my-profile-file + extraVolumeMounts: + - name: my-profile-mount + mountPath: /mount/path/to + readOnly: true + +server: + extraVolumes: + - name: my-profile-volume + secret: + secretName: my-aws-profile + items: + - key: my-profile-file + path: my-profile-file + extraVolumeMounts: + - name: my-profile-mount + mountPath: /mount/path/to + readOnly: true +``` + +Where the secret is defined as follows: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-aws-profile +type: Opaque +stringData: + my-profile-file: | + [default] + region = + aws_access_key_id = + aws_secret_access_key = + aws_session_token = +``` + +> ⚠️ Secret mounts are updated on an interval, not real time. If rotation is a requirement ensure the token lifetime outlives the mount update interval and the rotation process doesn't immediately invalidate the existing token + + ### GKE GKE cluster secret example using argocd-k8s-auth and [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity): diff --git a/docs/operator-manual/dynamic-cluster-distribution.md b/docs/operator-manual/dynamic-cluster-distribution.md index a32258c3f2f0a..9d5d2104a1795 100644 --- a/docs/operator-manual/dynamic-cluster-distribution.md +++ b/docs/operator-manual/dynamic-cluster-distribution.md @@ -17,16 +17,10 @@ which does not require a restart of the application controller pods. ## Enabling Dynamic Distribution of Clusters -This feature is disabled by default while it is in alpha. To enable it, you must set the environment `ARGOCD_ENABLE_DYNAMIC_CLUSTER_DISTRIBUTION` to true when running the Application Controller. - -In order to utilize the feature, the manifests `manifests/ha/base/controller-deployment/` can be applied as a Kustomize -overlay. This overlay sets the StatefulSet replicas to `0` and deploys the application controller as a Deployment. The -dynamic distribution code automatically kicks in when the controller is deployed as a Deployment. +This feature is disabled by default while it is in alpha. In order to utilize the feature, the manifests `manifests/ha/base/controller-deployment/` can be applied as a Kustomize overlay. This overlay sets the StatefulSet replicas to `0` and deploys the application controller as a Deployment. Also, you must set the environment `ARGOCD_ENABLE_DYNAMIC_CLUSTER_DISTRIBUTION` to true when running the Application Controller as a deployment. !!! important - The use of a Deployment instead of a StatefulSet is an implementation detail which may change in future versions of - this feature. Therefore, the directory name of the Kustomize overlay may change as well. Monitor the release notes - to avoid issues. + The use of a Deployment instead of a StatefulSet is an implementation detail which may change in future versions of this feature. Therefore, the directory name of the Kustomize overlay may change as well. Monitor the release notes to avoid issues. Note the introduction of new environment variable `ARGOCD_CONTROLLER_HEARTBEAT_TIME`. The environment variable is explained in [working of Dynamic Distribution Heartbeat Process](#working-of-dynamic-distribution) diff --git a/docs/operator-manual/high_availability.md b/docs/operator-manual/high_availability.md index b2051ad1a152c..a532200216d9b 100644 --- a/docs/operator-manual/high_availability.md +++ b/docs/operator-manual/high_availability.md @@ -57,7 +57,7 @@ performance. For performance reasons the controller monitors and caches only the preferred version into a version of the resource stored in Git. If `kubectl convert` fails because the conversion is not supported then the controller falls back to Kubernetes API query which slows down reconciliation. In this case, we advise to use the preferred resource version in Git. -* The controller polls Git every 3m by default. You can change this duration using the `timeout.reconciliation` setting in the `argocd-cm` ConfigMap. The value of `timeout.reconciliation` is a duration string e.g `60s`, `1m`, `1h` or `1d`. +* The controller polls Git every 3m by default. You can change this duration using the `timeout.reconciliation` and `timeout.reconciliation.jitter` setting in the `argocd-cm` ConfigMap. The value of the fields is a duration string e.g `60s`, `1m`, `1h` or `1d`. * If the controller is managing too many clusters and uses too much memory then you can shard clusters across multiple controller replicas. To enable sharding, increase the number of replicas in `argocd-application-controller` `StatefulSet` @@ -244,30 +244,41 @@ spec: # ... ``` +### Application Sync Timeout & Jitter + +Argo CD has a timeout for application syncs. It will trigger a refresh for each application periodically when the timeout expires. +With a large number of applications, this will cause a spike in the refresh queue and can cause a spike to the repo-server component. To avoid this, you can set a jitter to the sync timeout which will spread out the refreshes and give time to the repo-server to catch up. + +The jitter is the maximum duration that can be added to the sync timeout, so if the sync timeout is 5 minutes and the jitter is 1 minute, then the actual timeout will be between 5 and 6 minutes. + +To configure the jitter you can set the following environment variables: + +* `ARGOCD_RECONCILIATION_JITTER` - The jitter to apply to the sync timeout. Disabled when value is 0. Defaults to 0. + ## Rate Limiting Application Reconciliations -To prevent high controller resource usage or sync loops caused either due to misbehaving apps or other environment specific factors, +To prevent high controller resource usage or sync loops caused either due to misbehaving apps or other environment specific factors, we can configure rate limits on the workqueues used by the application controller. There are two types of rate limits that can be configured: * Global rate limits * Per item rate limits -The final rate limiter uses a combination of both and calculates the final backoff as `max(globalBackoff, perItemBackoff)`. +The final rate limiter uses a combination of both and calculates the final backoff as `max(globalBackoff, perItemBackoff)`. ### Global rate limits - This is enabled by default, it is a simple bucket based rate limiter that limits the number of items that can be queued per second. -This is useful to prevent a large number of apps from being queued at the same time. - + This is disabled by default, it is a simple bucket based rate limiter that limits the number of items that can be queued per second. +This is useful to prevent a large number of apps from being queued at the same time. + To configure the bucket limiter you can set the following environment variables: * `WORKQUEUE_BUCKET_SIZE` - The number of items that can be queued in a single burst. Defaults to 500. - * `WORKQUEUE_BUCKET_QPS` - The number of items that can be queued per second. Defaults to 50. + * `WORKQUEUE_BUCKET_QPS` - The number of items that can be queued per second. Defaults to MaxFloat64, which disables the limiter. -### Per item rate limits +### Per item rate limits - This by default returns a fixed base delay/backoff value but can be configured to return exponential values, read further to understand it's working. -Per item rate limiter limits the number of times a particular item can be queued. This is based on exponential backoff where the backoff time for an item keeps increasing exponentially + This by default returns a fixed base delay/backoff value but can be configured to return exponential values. +Per item rate limiter limits the number of times a particular item can be queued. This is based on exponential backoff where the backoff time for an item keeps increasing exponentially if it is queued multiple times in a short period, but the backoff is reset automatically if a configured `cool down` period has elapsed since the last time the item was queued. To configure the per item limiter you can set the following environment variables: @@ -277,16 +288,16 @@ To configure the per item limiter you can set the following environment variable * `WORKQUEUE_MAX_DELAY_NS` : The max delay in nanoseconds, this is the max backoff limit. Defaults to 3 * 10^9 (=3s) * `WORKQUEUE_BACKOFF_FACTOR` : The backoff factor, this is the factor by which the backoff is increased for each retry. Defaults to 1.5 -The formula used to calculate the backoff time for an item, where `numRequeue` is the number of times the item has been queued +The formula used to calculate the backoff time for an item, where `numRequeue` is the number of times the item has been queued and `lastRequeueTime` is the time at which the item was last queued: - When `WORKQUEUE_FAILURE_COOLDOWN_NS` != 0 : ``` -backoff = time.Since(lastRequeueTime) >= WORKQUEUE_FAILURE_COOLDOWN_NS ? - WORKQUEUE_BASE_DELAY_NS : +backoff = time.Since(lastRequeueTime) >= WORKQUEUE_FAILURE_COOLDOWN_NS ? + WORKQUEUE_BASE_DELAY_NS : min( - WORKQUEUE_MAX_DELAY_NS, + WORKQUEUE_MAX_DELAY_NS, WORKQUEUE_BASE_DELAY_NS * WORKQUEUE_BACKOFF_FACTOR ^ (numRequeue) ) ``` diff --git a/docs/operator-manual/ingress.md b/docs/operator-manual/ingress.md index 5ea947345d507..aad2208c21873 100644 --- a/docs/operator-manual/ingress.md +++ b/docs/operator-manual/ingress.md @@ -166,6 +166,43 @@ The argocd-server Service needs to be annotated with `projectcontour.io/upstream The API server should then be run with TLS disabled. Edit the `argocd-server` deployment to add the `--insecure` flag to the argocd-server command, or simply set `server.insecure: "true"` in the `argocd-cmd-params-cm` ConfigMap [as described here](server-commands/additional-configuration-method.md). +Contour httpproxy CRD: + +Using a contour httpproxy CRD allows you to use the same hostname for the GRPC and REST api. + +```yaml +apiVersion: projectcontour.io/v1 +kind: HTTPProxy +metadata: + name: argocd-server + namespace: argocd +spec: + ingressClassName: contour + virtualhost: + fqdn: path.to.argocd.io + tls: + secretName: wildcard-tls + routes: + - conditions: + - prefix: / + - header: + name: Content-Type + contains: application/grpc + services: + - name: argocd-server + port: 80 + protocol: h2c # allows for unencrypted http2 connections + timeoutPolicy: + response: 1h + idle: 600s + idleConnection: 600s + - conditions: + - prefix: / + services: + - name: argocd-server + port: 80 +``` + ## [kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx) ### Option 1: SSL-Passthrough diff --git a/docs/operator-manual/metrics.md b/docs/operator-manual/metrics.md index cfd2a8a8093ac..a3ddbfe9904d3 100644 --- a/docs/operator-manual/metrics.md +++ b/docs/operator-manual/metrics.md @@ -70,6 +70,8 @@ Scraped at the `argocd-server-metrics:8083/metrics` endpoint. | `argocd_redis_request_total` | counter | Number of Kubernetes requests executed during application reconciliation. | | `grpc_server_handled_total` | counter | Total number of RPCs completed on the server, regardless of success or failure. | | `grpc_server_msg_sent_total` | counter | Total number of gRPC stream messages sent by the server. | +| `argocd_proxy_extension_request_total` | counter | Number of requests sent to the configured proxy extensions. | +| `argocd_proxy_extension_request_duration_seconds` | histogram | Request duration in seconds between the Argo CD API server and the proxy extension backend. | ## Repo Server Metrics Metrics about the Repo Server. @@ -79,6 +81,7 @@ Scraped at the `argocd-repo-server:8084/metrics` endpoint. |--------|:----:|-------------| | `argocd_git_request_duration_seconds` | histogram | Git requests duration seconds. | | `argocd_git_request_total` | counter | Number of git requests performed by repo server | +| `argocd_git_fetch_fail_total` | counter | Number of git fetch requests failures by repo server | | `argocd_redis_request_duration_seconds` | histogram | Redis requests duration seconds. | | `argocd_redis_request_total` | counter | Number of Kubernetes requests executed during application reconciliation. | | `argocd_repo_pending_request_total` | gauge | Number of pending requests requiring repository lock | @@ -168,6 +171,8 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: argocd-redis-haproxy-metrics + labels: + release: prometheus-operator spec: selector: matchLabels: @@ -176,7 +181,7 @@ spec: - port: http-exporter-port ``` -For notifications controller, you need to additionally add following: +For notifications controller, you need to additionally add following: ```yaml apiVersion: monitoring.coreos.com/v1 diff --git a/docs/operator-manual/notifications/catalog.md b/docs/operator-manual/notifications/catalog.md index 8f413ac7eb5b3..add7084304b98 100644 --- a/docs/operator-manual/notifications/catalog.md +++ b/docs/operator-manual/notifications/catalog.md @@ -62,8 +62,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -90,8 +89,7 @@ teams: "value": "{{.app.status.sync.revision}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -145,8 +143,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -169,8 +166,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -224,8 +220,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -252,8 +247,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -307,8 +301,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -335,8 +328,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -394,8 +386,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -418,8 +409,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -472,8 +462,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -500,8 +489,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" diff --git a/docs/operator-manual/notifications/functions.md b/docs/operator-manual/notifications/functions.md index 3d614e4e53a55..c50d122024b76 100644 --- a/docs/operator-manual/notifications/functions.md +++ b/docs/operator-manual/notifications/functions.md @@ -48,6 +48,16 @@ Transforms given GIT URL into HTTPs format. Returns repository URL full name `(/)`. Currently supports only Github, GitLab and Bitbucket. +
+**`repo.QueryEscape(s string) string`** + +QueryEscape escapes the string, so it can be safely placed inside a URL + +Example: +``` +/projects/{{ call .repo.QueryEscape (call .repo.FullNameByRepoURL .app.status.RepoURL) }}/merge_requests +``` +
**`repo.GetCommitMetadata(sha string) CommitMetadata`** diff --git a/docs/operator-manual/notifications/services/alertmanager.md b/docs/operator-manual/notifications/services/alertmanager.md index e0f9d7e4e7889..033a76a29ea65 100755 --- a/docs/operator-manual/notifications/services/alertmanager.md +++ b/docs/operator-manual/notifications/services/alertmanager.md @@ -43,7 +43,7 @@ You should turn off "send_resolved" or you will receive unnecessary recovery not apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.alertmanager: | targets: @@ -58,7 +58,7 @@ If your alertmanager has changed the default api, you can customize "apiPath". apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.alertmanager: | targets: @@ -89,7 +89,7 @@ stringData: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.alertmanager: | targets: @@ -110,7 +110,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.alertmanager: | targets: diff --git a/docs/operator-manual/notifications/services/awssqs.md b/docs/operator-manual/notifications/services/awssqs.md index 6b744f4744b93..5331533826348 100755 --- a/docs/operator-manual/notifications/services/awssqs.md +++ b/docs/operator-manual/notifications/services/awssqs.md @@ -1,8 +1,8 @@ -# AWS SQS +# AWS SQS ## Parameters -This notification service is capable of sending simple messages to AWS SQS queue. +This notification service is capable of sending simple messages to AWS SQS queue. * `queue` - name of the queue you are intending to send messages to. Can be overridden with target destination annotation. * `region` - region of the sqs queue can be provided via env variable AWS_DEFAULT_REGION @@ -30,7 +30,7 @@ metadata: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.awssqs: | region: "us-east-2" @@ -63,7 +63,7 @@ stringData: ### Minimal configuration using AWS Env variables -Ensure following list of environment variables are injected via OIDC, or other method. And assuming SQS is local to the account. +Ensure the following list of environment variables are injected via OIDC, or another method. And assuming SQS is local to the account. You may skip usage of secret for sensitive data and omit other parameters. (Setting parameters via ConfigMap takes precedent.) Variables: @@ -89,7 +89,7 @@ metadata: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.awssqs: | queue: "myqueue" @@ -104,3 +104,16 @@ data: - oncePer: obj.metadata.annotations["generation"] ``` + +## FIFO SQS Queues + +FIFO queues require a [MessageGroupId](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html#SQS-SendMessage-request-MessageGroupId) to be sent along with every message, every message with a matching MessageGroupId will be processed one by one in order. + +To send to a FIFO SQS Queue you must include a `messageGroupId` in the template such as in the example below: + +```yaml +template.deployment-ready: | + message: | + Deployment {{.obj.metadata.name}} is ready! + messageGroupId: {{.obj.metadata.name}}-deployment +``` diff --git a/docs/operator-manual/notifications/services/email.md b/docs/operator-manual/notifications/services/email.md index b81ab6cde8b4c..7fd3f0e22379c 100755 --- a/docs/operator-manual/notifications/services/email.md +++ b/docs/operator-manual/notifications/services/email.md @@ -20,7 +20,7 @@ The following snippet contains sample Gmail service configuration: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.email.gmail: | username: $email-username @@ -36,7 +36,7 @@ Without authentication: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.email.example: | host: smtp.example.com @@ -52,7 +52,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: template.app-sync-succeeded: | email: diff --git a/docs/operator-manual/notifications/services/github.md b/docs/operator-manual/notifications/services/github.md index be76ab150d1a1..1fa1a985d2682 100755 --- a/docs/operator-manual/notifications/services/github.md +++ b/docs/operator-manual/notifications/services/github.md @@ -24,7 +24,7 @@ in `argocd-notifications-cm` ConfigMap apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.github: | appID: @@ -76,6 +76,7 @@ template.app-deployed: | logURL: "{{.context.argocdUrl}}/applications/{{.app.metadata.name}}?operation=true" requiredContexts: [] autoMerge: true + transientEnvironment: false pullRequestComment: content: | Application {{.app.metadata.name}} is now running new version of deployments manifests. diff --git a/docs/operator-manual/notifications/services/googlechat.md b/docs/operator-manual/notifications/services/googlechat.md index 885ce685a4511..821c23023e863 100755 --- a/docs/operator-manual/notifications/services/googlechat.md +++ b/docs/operator-manual/notifications/services/googlechat.md @@ -19,7 +19,7 @@ The Google Chat notification service send message notifications to a google chat apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.googlechat: | webhooks: diff --git a/docs/operator-manual/notifications/services/grafana.md b/docs/operator-manual/notifications/services/grafana.md index a36672d0fa423..1f3e77701f044 100755 --- a/docs/operator-manual/notifications/services/grafana.md +++ b/docs/operator-manual/notifications/services/grafana.md @@ -21,7 +21,7 @@ Available parameters : apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.grafana: | apiUrl: https://grafana.example.com/api diff --git a/docs/operator-manual/notifications/services/mattermost.md b/docs/operator-manual/notifications/services/mattermost.md index 98e0d0fd7b82f..d1f187e955b9c 100755 --- a/docs/operator-manual/notifications/services/mattermost.md +++ b/docs/operator-manual/notifications/services/mattermost.md @@ -19,7 +19,7 @@ in `argocd-notifications-cm` ConfigMap apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.mattermost: | apiURL: diff --git a/docs/operator-manual/notifications/services/newrelic.md b/docs/operator-manual/notifications/services/newrelic.md index d98288a846422..b0c7e340c9b28 100755 --- a/docs/operator-manual/notifications/services/newrelic.md +++ b/docs/operator-manual/notifications/services/newrelic.md @@ -14,7 +14,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.newrelic: | apiURL: diff --git a/docs/operator-manual/notifications/services/opsgenie.md b/docs/operator-manual/notifications/services/opsgenie.md index 665d0081e7c73..e92ee99756ab8 100755 --- a/docs/operator-manual/notifications/services/opsgenie.md +++ b/docs/operator-manual/notifications/services/opsgenie.md @@ -12,14 +12,15 @@ To be able to send notifications with argocd-notifications you have to create an 8. Give your integration a name, copy the "API key" and safe it somewhere for later 9. Make sure the checkboxes for "Create and Update Access" and "enable" are selected, disable the other checkboxes to remove unnecessary permissions 10. Click "Safe Integration" at the bottom -11. Check your browser for the correct server apiURL. If it is "app.opsgenie.com" then use the us/international api url `api.opsgenie.com` in the next step, otherwise use `api.eu.opsgenie.com` (european api). -12. You are finished with configuring opsgenie. Now you need to configure argocd-notifications. Use the apiUrl, the team name and the apiKey to configure the opsgenie integration in the `argocd-notifications-secret` secret. +11. Check your browser for the correct server apiURL. If it is "app.opsgenie.com" then use the US/international api url `api.opsgenie.com` in the next step, otherwise use `api.eu.opsgenie.com` (European API). +12. You are finished with configuring Opsgenie. Now you need to configure argocd-notifications. Use the apiUrl, the team name and the apiKey to configure the Opsgenie integration in the `argocd-notifications-secret` secret. + ```yaml apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.opsgenie: | apiUrl: diff --git a/docs/operator-manual/notifications/services/pagerduty.md b/docs/operator-manual/notifications/services/pagerduty.md index 0e1ab965332e1..c6e1e41dac81d 100755 --- a/docs/operator-manual/notifications/services/pagerduty.md +++ b/docs/operator-manual/notifications/services/pagerduty.md @@ -1,17 +1,17 @@ -# Pagerduty +# PagerDuty ## Parameters -The Pagerduty notification service is used to create pagerduty incidents and requires specifying the following settings: +The PagerDuty notification service is used to create PagerDuty incidents and requires specifying the following settings: -* `pagerdutyToken` - the pagerduty auth token +* `pagerdutyToken` - the PagerDuty auth token * `from` - email address of a valid user associated with the account making the request. * `serviceID` - The ID of the resource. ## Example -The following snippet contains sample Pagerduty service configuration: +The following snippet contains sample PagerDuty service configuration: ```yaml apiVersion: v1 @@ -26,7 +26,7 @@ stringData: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.pagerduty: | token: $pagerdutyToken @@ -35,13 +35,13 @@ data: ## Template -[Notification templates](../templates.md) support specifying subject for pagerduty notifications: +[Notification templates](../templates.md) support specifying subject for PagerDuty notifications: ```yaml apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: template.rollout-aborted: | message: Rollout {{.rollout.metadata.name}} is aborted. @@ -62,5 +62,5 @@ apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: - notifications.argoproj.io/subscribe.on-rollout-aborted.pagerduty: "" + notifications.argoproj.io/subscribe.on-rollout-aborted.pagerduty: "" ``` diff --git a/docs/operator-manual/notifications/services/pagerduty_v2.md b/docs/operator-manual/notifications/services/pagerduty_v2.md index 21e8d942e4e93..549cdc937b150 100755 --- a/docs/operator-manual/notifications/services/pagerduty_v2.md +++ b/docs/operator-manual/notifications/services/pagerduty_v2.md @@ -28,7 +28,7 @@ stringData: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.pagerdutyv2: | serviceKeys: @@ -43,7 +43,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: template.rollout-aborted: | message: Rollout {{.rollout.metadata.name}} is aborted. @@ -74,5 +74,5 @@ apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: - notifications.argoproj.io/subscribe.on-rollout-aborted.pagerdutyv2: "" + notifications.argoproj.io/subscribe.on-rollout-aborted.pagerdutyv2: "" ``` diff --git a/docs/operator-manual/notifications/services/pushover.md b/docs/operator-manual/notifications/services/pushover.md index 37cb20b277dcc..a09b3660f9233 100755 --- a/docs/operator-manual/notifications/services/pushover.md +++ b/docs/operator-manual/notifications/services/pushover.md @@ -1,13 +1,13 @@ # Pushover 1. Create an app at [pushover.net](https://pushover.net/apps/build). -2. Store the API key in `` Secret and define the secret name in `` ConfigMap: +2. Store the API key in `` Secret and define the secret name in `argocd-notifications-cm` ConfigMap: ```yaml apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.pushover: | token: $pushover-token diff --git a/docs/operator-manual/notifications/services/rocketchat.md b/docs/operator-manual/notifications/services/rocketchat.md index f1157050139d0..20aaa405c80d0 100755 --- a/docs/operator-manual/notifications/services/rocketchat.md +++ b/docs/operator-manual/notifications/services/rocketchat.md @@ -43,7 +43,7 @@ stringData: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.rocketchat: | email: $rocketchat-email diff --git a/docs/operator-manual/notifications/services/slack.md b/docs/operator-manual/notifications/services/slack.md index 0f3fdf1739210..41bdddd7617c4 100755 --- a/docs/operator-manual/notifications/services/slack.md +++ b/docs/operator-manual/notifications/services/slack.md @@ -15,6 +15,7 @@ The Slack notification service configuration includes following settings: | `signingSecret` | False | `string` | | `8f742231b10e8888abcd99yyyzzz85a5` | | `token` | **True** | `string` | The app's OAuth access token. | `xoxb-1234567890-1234567890123-5n38u5ed63fgzqlvuyxvxcx6` | | `username` | False | `string` | The app username. | `argocd` | +| `disableUnfurl` | False | `bool` | Disable slack unfurling links in messages | `true` | ## Configuration @@ -48,7 +49,7 @@ The Slack notification service configuration includes following settings: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.slack: | token: $slack-token diff --git a/docs/operator-manual/notifications/services/teams.md b/docs/operator-manual/notifications/services/teams.md index 8b8c6b819c795..0e44456d4de19 100755 --- a/docs/operator-manual/notifications/services/teams.md +++ b/docs/operator-manual/notifications/services/teams.md @@ -18,7 +18,7 @@ The Teams notification service send message notifications using Teams bot and re apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.teams: | recipientUrls: diff --git a/docs/operator-manual/notifications/services/telegram.md b/docs/operator-manual/notifications/services/telegram.md index 953c2a9fca0bf..8612a09d1ca84 100755 --- a/docs/operator-manual/notifications/services/telegram.md +++ b/docs/operator-manual/notifications/services/telegram.md @@ -2,13 +2,13 @@ 1. Get an API token using [@Botfather](https://t.me/Botfather). 2. Store token in `` Secret and configure telegram integration -in `` ConfigMap: +in `argocd-notifications-cm` ConfigMap: ```yaml apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.telegram: | token: $telegram-token diff --git a/docs/operator-manual/notifications/services/webex.md b/docs/operator-manual/notifications/services/webex.md index 440ed1ddc738f..eba4c5e11b8dc 100755 --- a/docs/operator-manual/notifications/services/webex.md +++ b/docs/operator-manual/notifications/services/webex.md @@ -24,7 +24,7 @@ The Webex Teams notification service configuration includes following settings: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.webex: | token: $webex-token diff --git a/docs/operator-manual/notifications/services/webhook.md b/docs/operator-manual/notifications/services/webhook.md index 965098402236f..4b8ca38a685ad 100755 --- a/docs/operator-manual/notifications/services/webhook.md +++ b/docs/operator-manual/notifications/services/webhook.md @@ -31,7 +31,7 @@ Use the following steps to configure webhook: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.webhook.: | url: https:/// @@ -50,7 +50,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: template.github-commit-status: | webhook: @@ -82,7 +82,7 @@ metadata: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.webhook.github: | url: https://api.github.com @@ -97,7 +97,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.webhook.github: | url: https://api.github.com @@ -128,7 +128,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.webhook.jenkins: | url: http:///job//build?token= @@ -145,7 +145,7 @@ type: Opaque apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.webhook.form: | url: https://form.example.com @@ -166,7 +166,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: + name: argocd-notifications-cm data: service.webhook.slack_webhook: | url: https://hooks.slack.com/services/xxxxx diff --git a/docs/operator-manual/notifications/triggers.md b/docs/operator-manual/notifications/triggers.md index 02d0228c40997..49a6244777959 100644 --- a/docs/operator-manual/notifications/triggers.md +++ b/docs/operator-manual/notifications/triggers.md @@ -71,7 +71,7 @@ When one repo is used to sync multiple applications, the `oncePer: app.status.sy ### oncePer -The `oncePer` filed is supported like as follows. +The `oncePer` field is supported like as follows. ```yaml apiVersion: argoproj.io/v1alpha1 diff --git a/docs/operator-manual/notifications/troubleshooting.md b/docs/operator-manual/notifications/troubleshooting.md index 6e144bb0c9985..616cd4b024e82 100644 --- a/docs/operator-manual/notifications/troubleshooting.md +++ b/docs/operator-manual/notifications/troubleshooting.md @@ -16,7 +16,7 @@ Additionally, you can specify `:empty` to use empty secret with no notification ```bash argocd admin notifications trigger get \ - --config-map ./argocd admin notifications-cm.yaml --secret :empty + --config-map ./argocd-notifications-cm.yaml --secret :empty ``` * Trigger notification using in-cluster config map and secret: diff --git a/docs/operator-manual/server-commands/argocd-application-controller.md b/docs/operator-manual/server-commands/argocd-application-controller.md index 1d71fc6494445..61c0c32119895 100644 --- a/docs/operator-manual/server-commands/argocd-application-controller.md +++ b/docs/operator-manual/server-commands/argocd-application-controller.md @@ -17,6 +17,7 @@ argocd-application-controller [flags] ``` --app-hard-resync int Time period in seconds for application hard resync. --app-resync int Time period in seconds for application resync. (default 180) + --app-resync-jitter int Maximum time period in seconds to add as a delay jitter for application resync. --app-state-cache-expiration duration Cache expiration for app state (default 1h0m0s) --application-namespaces strings List of additional namespaces that applications are allowed to be reconciled from --as string Username to impersonate for the operation @@ -76,7 +77,7 @@ argocd-application-controller [flags] --username string Username for basic authentication to the API server --wq-backoff-factor float Set Workqueue Per Item Rate Limiter Backoff Factor, default is 1.5 (default 1.5) --wq-basedelay-ns duration Set Workqueue Per Item Rate Limiter Base Delay duration in nanoseconds, default 1000000 (1ms) (default 1ms) - --wq-bucket-qps int Set Workqueue Rate Limiter Bucket QPS, default 50 (default 50) + --wq-bucket-qps float Set Workqueue Rate Limiter Bucket QPS, default set to MaxFloat64 which disables the bucket limiter (default 1.7976931348623157e+308) --wq-bucket-size int Set Workqueue Rate Limiter Bucket Size, default 500 (default 500) --wq-cooldown-ns duration Set Workqueue Per Item Rate Limiter Cooldown duration in ns, default 0(per item rate limiter disabled) --wq-maxdelay-ns duration Set Workqueue Per Item Rate Limiter Max Delay duration in nanoseconds, default 1000000000 (1s) (default 1s) diff --git a/docs/operator-manual/server-commands/argocd-server.md b/docs/operator-manual/server-commands/argocd-server.md index 1da27d735e1cd..a72cc041299ad 100644 --- a/docs/operator-manual/server-commands/argocd-server.md +++ b/docs/operator-manual/server-commands/argocd-server.md @@ -26,6 +26,7 @@ argocd-server [flags] ``` --address string Listen on given address (default "0.0.0.0") + --api-content-types string Semicolon separated list of allowed content types for non GET api requests. Any content type is allowed if empty. (default "application/json") --app-state-cache-expiration duration Cache expiration for app state (default 1h0m0s) --application-namespaces strings List of additional namespaces where application resources can be managed in --as string Username to impersonate for the operation diff --git a/docs/operator-manual/upgrading/2.9-2.10.md b/docs/operator-manual/upgrading/2.9-2.10.md index 4cd7c379bdc81..adb37c4babf4b 100644 --- a/docs/operator-manual/upgrading/2.9-2.10.md +++ b/docs/operator-manual/upgrading/2.9-2.10.md @@ -10,3 +10,7 @@ removed. To avoid unexpected behavior, follow the [client-side to server-side resource upgrade guide](https://kubernetes.io/docs/reference/using-api/server-side-apply/#upgrading-from-client-side-apply-to-server-side-apply) before enabling `managedNamespaceMetadata` on an existing namespace. + +## Upgraded Helm Version + +Note that bundled Helm version has been upgraded from 3.13.2 to 3.14.2. diff --git a/docs/operator-manual/user-management/index.md b/docs/operator-manual/user-management/index.md index 496dd17a83e9f..c002b77ada5ed 100644 --- a/docs/operator-manual/user-management/index.md +++ b/docs/operator-manual/user-management/index.md @@ -500,7 +500,7 @@ data: #### Alternative -If you want to store sensitive data in **another** Kubernetes `Secret`, instead of `argocd-secret`. ArgoCD knows to check the keys under `data` in your Kubernetes `Secret` for a corresponding key whenever a value in a configmap starts with `$`, then your Kubernetes `Secret` name and `:` (colon). +If you want to store sensitive data in **another** Kubernetes `Secret`, instead of `argocd-secret`. ArgoCD knows to check the keys under `data` in your Kubernetes `Secret` for a corresponding key whenever a value in a configmap or secret starts with `$`, then your Kubernetes `Secret` name and `:` (colon). Syntax: `$:` diff --git a/docs/operator-manual/user-management/microsoft.md b/docs/operator-manual/user-management/microsoft.md index 33a6b3e945940..486d647fde3d0 100644 --- a/docs/operator-manual/user-management/microsoft.md +++ b/docs/operator-manual/user-management/microsoft.md @@ -1,13 +1,16 @@ # Microsoft -* [Azure AD SAML Enterprise App Auth using Dex](#azure-ad-saml-enterprise-app-auth-using-dex) -* [Azure AD App Registration Auth using OIDC](#azure-ad-app-registration-auth-using-oidc) -* [Azure AD App Registration Auth using Dex](#azure-ad-app-registration-auth-using-dex) +!!! note "" + Entra ID was formerly known as Azure AD. -## Azure AD SAML Enterprise App Auth using Dex -### Configure a new Azure AD Enterprise App +* [Entra ID SAML Enterprise App Auth using Dex](#entra-id-saml-enterprise-app-auth-using-dex) +* [Entra ID App Registration Auth using OIDC](#entra-id-app-registration-auth-using-oidc) +* [Entra ID App Registration Auth using Dex](#entra-id-app-registration-auth-using-dex) -1. From the `Azure Active Directory` > `Enterprise applications` menu, choose `+ New application` +## Entra ID SAML Enterprise App Auth using Dex +### Configure a new Entra ID Enterprise App + +1. From the `Microsoft Entra ID` > `Enterprise applications` menu, choose `+ New application` 2. Select `Non-gallery application` 3. Enter a `Name` for the application (e.g. `Argo CD`), then choose `Add` 4. Once the application is created, open it from the `Enterprise applications` menu. @@ -31,9 +34,9 @@ - *Keep a copy of the encoded output to be used in the next section.* 9. From the `Single sign-on` menu, copy the `Login URL` parameter, to be used in the next section. -### Configure Argo to use the new Azure AD Enterprise App +### Configure Argo to use the new Entra ID Enterprise App -1. Edit `argocd-cm` and add the following `dex.config` to the data section, replacing the `caData`, `my-argo-cd-url` and `my-login-url` your values from the Azure AD App: +1. Edit `argocd-cm` and add the following `dex.config` to the data section, replacing the `caData`, `my-argo-cd-url` and `my-login-url` your values from the Entra ID App: data: url: https://my-argo-cd-url @@ -56,7 +59,7 @@ groupsAttr: Group 2. Edit `argocd-rbac-cm` to configure permissions, similar to example below. - - Use Azure AD `Group IDs` for assigning roles. + - Use Entra ID `Group IDs` for assigning roles. - See [RBAC Configurations](../rbac.md) for more detailed scenarios. # example policy @@ -70,11 +73,11 @@ p, role:org-admin, repositories, delete, *, allow g, "84ce98d1-e359-4f3b-85af-985b458de3c6", role:org-admin # (azure group assigned to role) -## Azure AD App Registration Auth using OIDC -### Configure a new Azure AD App registration -#### Add a new Azure AD App registration +## Entra ID App Registration Auth using OIDC +### Configure a new Entra ID App registration +#### Add a new Entra ID App registration -1. From the `Azure Active Directory` > `App registrations` menu, choose `+ New registration` +1. From the `Microsoft Entra ID` > `App registrations` menu, choose `+ New registration` 2. Enter a `Name` for the application (e.g. `Argo CD`). 3. Specify who can use the application (e.g. `Accounts in this organizational directory only`). 4. Enter Redirect URI (optional) as follows (replacing `my-argo-cd-url` with your Argo URL), then choose `Add`. @@ -92,29 +95,29 @@ - **Redirect URI:** `http://localhost:8085/auth/callback` ![Azure App registration's Authentication](../../assets/azure-app-registration-authentication.png "Azure App registration's Authentication") -#### Add credentials a new Azure AD App registration +#### Add credentials a new Entra ID App registration 1. From the `Certificates & secrets` menu, choose `+ New client secret` 2. Enter a `Name` for the secret (e.g. `ArgoCD-SSO`). - Make sure to copy and save generated value. This is a value for the `client_secret`. ![Azure App registration's Secret](../../assets/azure-app-registration-secret.png "Azure App registration's Secret") -#### Setup permissions for Azure AD Application +#### Setup permissions for Entra ID Application 1. From the `API permissions` menu, choose `+ Add a permission` 2. Find `User.Read` permission (under `Microsoft Graph`) and grant it to the created application: - ![Azure AD API permissions](../../assets/azure-api-permissions.png "Azure AD API permissions") + ![Entra ID API permissions](../../assets/azure-api-permissions.png "Entra ID API permissions") 3. From the `Token Configuration` menu, choose `+ Add groups claim` - ![Azure AD token configuration](../../assets/azure-token-configuration.png "Azure AD token configuration") + ![Entra ID token configuration](../../assets/azure-token-configuration.png "Entra ID token configuration") -### Associate an Azure AD group to your Azure AD App registration +### Associate an Entra ID group to your Entra ID App registration -1. From the `Azure Active Directory` > `Enterprise applications` menu, search the App that you created (e.g. `Argo CD`). - - An Enterprise application with the same name of the Azure AD App registration is created when you add a new Azure AD App registration. +1. From the `Microsoft Entra ID` > `Enterprise applications` menu, search the App that you created (e.g. `Argo CD`). + - An Enterprise application with the same name of the Entra ID App registration is created when you add a new Entra ID App registration. 2. From the `Users and groups` menu of the app, add any users or groups requiring access to the service. ![Azure Enterprise SAML Users](../../assets/azure-enterprise-users.png "Azure Enterprise SAML Users") -### Configure Argo to use the new Azure AD App registration +### Configure Argo to use the new Entra ID App registration 1. Edit `argocd-cm` and configure the `data.oidc.config` and `data.url` section: @@ -173,7 +176,7 @@ Refer to [operator-manual/argocd-rbac-cm.yaml](https://github.com/argoproj/argo-cd/blob/master/docs/operator-manual/argocd-rbac-cm.yaml) for all of the available variables. -## Azure AD App Registration Auth using Dex +## Entra ID App Registration Auth using Dex Configure a new AD App Registration, as above. Then, add the `dex.config` to `argocd-cm`: @@ -200,9 +203,9 @@ data: 1. Open a new browser tab and enter your ArgoCD URI: https://`` ![Azure SSO Web Log In](../../assets/azure-sso-web-log-in-via-azure.png "Azure SSO Web Log In") -3. Click `LOGIN VIA AZURE` button to log in with your Azure Active Directory account. You’ll see the ArgoCD applications screen. +3. Click `LOGIN VIA AZURE` button to log in with your Microsoft Entra ID account. You’ll see the ArgoCD applications screen. ![Azure SSO Web Application](../../assets/azure-sso-web-application.png "Azure SSO Web Application") -4. Navigate to User Info and verify Group ID. Groups will have your group’s Object ID that you added in the `Setup permissions for Azure AD Application` step. +4. Navigate to User Info and verify Group ID. Groups will have your group’s Object ID that you added in the `Setup permissions for Entra ID Application` step. ![Azure SSO Web User Info](../../assets/azure-sso-web-user-info.png "Azure SSO Web User Info") ### Log in to ArgoCD using CLI diff --git a/docs/operator-manual/user-management/okta.md b/docs/operator-manual/user-management/okta.md index 09d7099d19954..308254759de6e 100644 --- a/docs/operator-manual/user-management/okta.md +++ b/docs/operator-manual/user-management/okta.md @@ -118,34 +118,81 @@ data: ## OIDC (without Dex) -!!! warning "Do you want groups for RBAC later?" - If you want `groups` scope returned from Okta you need to unfortunately contact support to enable [API Access Management with Okta](https://developer.okta.com/docs/concepts/api-access-management/) or [_just use SAML above!_](#saml-with-dex) +!!! warning "Okta groups for RBAC" + If you want `groups` scope returned from Okta, you will need to enable [API Access Management with Okta](https://developer.okta.com/docs/concepts/api-access-management/). This addon is free, and automatically enabled, on Okta developer edition. However, it's an optional add-on for production environments, with an additional associated cost. - Next you may need the API Access Management feature, which the support team can enable for your OktaPreview domain for testing, to enable "custom scopes" and a separate endpoint to use instead of the "public" `/oauth2/v1/authorize` API Access Management endpoint. This might be a paid feature if you want OIDC unfortunately. The free alternative I found was SAML. + You may alternately add a "groups" scope and claim to the default authorization server, and then filter the claim in the Okta application configuration. It's not clear if this requires the Authorization Server add-on. + + If this is not an option for you, use the [SAML (with Dex)](#saml-with-dex) option above instead. + +!!! note + These instructions and screenshots are of Okta version 2023.05.2 E. You can find the current version in the Okta website footer. + +First, create the OIDC integration: + +1. On the `Okta Admin` page, navigate to the Okta Applications at `Applications > Applications.` +1. Choose `Create App Integration`, and choose `OIDC`, and then `Web Application` in the resulting dialogues. + ![Okta OIDC app dialogue](../../assets/okta-create-oidc-app.png) +1. Update the following: + 1. `App Integration name` and `Logo` - set these to suit your needs; they'll be displayed in the Okta catalogue. + 1. `Sign-in redirect URLs`: Add `https://argocd.example.com/auth/callback`; replacing `argocd.example.com` with your ArgoCD web interface URL. Also add `http://localhost:8085/auth/callback` if you would like to be able to login with the CLI. + 1. `Sign-out redirect URIs`: Add `https://argocd.example.com`; substituting the correct domain name as above. + 1. Either assign groups, or choose to skip this step for now. + 1. Leave the rest of the options as-is, and save the integration. + ![Okta app settings](../../assets/okta-app.png) +1. Copy the `Client ID` and the `Client Secret` from the newly created app; you will need these later. + +Next, create a custom Authorization server: 1. On the `Okta Admin` page, navigate to the Okta API Management at `Security > API`. - ![Okta API Management](../../assets/api-management.png) -1. Choose your `default` authorization server. -1. Click `Scopes > Add Scope` - 1. Add a scope called `groups`. - ![Groups Scope](../../assets/groups-scope.png) -1. Click `Claims > Add Claim.` - 1. Add a claim called `groups` - 1. Choose the matching options you need, one example is: - * e.g. to match groups starting with `argocd-` you'd return an `ID Token` using your scope name from step 3 (e.g. `groups`) where the groups name `matches` the `regex` `argocd-.*` - ![Groups Claim](../../assets/groups-claim.png) -1. Edit the `argocd-cm` and configure the `data.oidc.config` section: +1. Click `Add Authorization Server`, and assign it a name and a description. The `Audience` should match your ArgoCD URL - `https://argocd.example.com` +1. Click `Scopes > Add Scope`: + 1. Add a scope called `groups`. Leave the rest of the options as default. + ![Groups Scope](../../assets/okta-groups-scope.png) +1. Click `Claims > Add Claim`: + 1. Add a claim called `groups`. + 1. Adjust the `Include in token type` to `ID Token`, `Always`. + 1. Adjust the `Value type` to `Groups`. + 1. Add a filter that will match the Okta groups you want passed on to ArgoCD; for example `Regex: argocd-.*`. + 1. Set `Include in` to `groups` (the scope you created above). + ![Groups Claim](../../assets/okta-groups-claim.png) +1. Click on `Access Policies` > `Add Policy.` This policy will restrict how this authorization server is used. + 1. Add a name and description. + 1. Assign the policy to the client (application integration) you created above. The field should auto-complete as you type. + 1. Create the policy. + ![Auth Policy](../../assets/okta-auth-policy.png) +1. Add a rule to the policy: + 1. Add a name; `default` is a reasonable name for this rule. + 1. Fine-tune the settings to suit your organization's security posture. Some ideas: + 1. uncheck all the grant types except the Authorization Code. + 1. Adjust the token lifetime to govern how long a session can last. + 1. Restrict refresh token lifetime, or completely disable it. + ![Default rule](../../assets/okta-auth-rule.png) +1. Finally, click `Back to Authorization Servers`, and copy the `Issuer URI`. You will need this later. + +If you haven't yet created Okta groups, and assigned them to the application integration, you should do that now: + +1. Go to `Directory > Groups` +1. For each group you wish to add: + 1. Click `Add Group`, and choose a meaningful name. It should match the regex or pattern you added to your custom `group` claim. + 1. Click on the group (refresh the page if the new group didn't show up in the list). + 1. Assign Okta users to the group. + 1. Click on `Applications` and assign the OIDC application integration you created to this group. + 1. Repeat as needed. + +Finally, configure ArgoCD itself. Edit the `argocd-cm` configmap: ```yaml +url: https://argocd.example.com oidc.config: | name: Okta - issuer: https://yourorganization.oktapreview.com - clientID: 0oaltaqg3oAIf2NOa0h3 - clientSecret: ZXF_CfUc-rtwNfzFecGquzdeJ_MxM4sGc8pDT2Tg6t + # this is the authorization server URI + issuer: https://example.okta.com/oauth2/aus9abcdefgABCDEFGd7 + clientID: 0oa9abcdefgh123AB5d7 + clientSecret: ABCDEFG1234567890abcdefg requestedScopes: ["openid", "profile", "email", "groups"] requestedIDTokenClaims: {"groups": {"essential": true}} ``` - - +You may want to store the `clientSecret` in a Kubernetes secret; see [how to deal with SSO secrets](./index.md/#sensitive-data-and-sso-client-secrets ) for more details. diff --git a/docs/operator-manual/webhook.md b/docs/operator-manual/webhook.md index eb15c4cb02369..a0e6c8deba1b2 100644 --- a/docs/operator-manual/webhook.md +++ b/docs/operator-manual/webhook.md @@ -97,3 +97,13 @@ stringData: ``` After saving, the changes should take effect automatically. + +### Alternative + +If you want to store webhook data in **another** Kubernetes `Secret`, instead of `argocd-secret`. ArgoCD knows to check the keys under `data` in your Kubernetes `Secret` starts with `$`, then your Kubernetes `Secret` name and `:` (colon). + +Syntax: `$:` + +> NOTE: Secret must have label `app.kubernetes.io/part-of: argocd` + +For more information refer to the corresponding section in the [User Management Documentation](user-management/index.md#alternative). diff --git a/docs/proposals/decouple-application-sync-user-using-impersonation.md b/docs/proposals/decouple-application-sync-user-using-impersonation.md new file mode 100644 index 0000000000000..e7e459a7059c0 --- /dev/null +++ b/docs/proposals/decouple-application-sync-user-using-impersonation.md @@ -0,0 +1,592 @@ +--- +title: Decouple Control plane and Application Sync privileges +authors: + - "@anandf" +sponsors: + - Red Hat +reviewers: + - "@blakepettersson" + - "@crenshaw-dev" + - "@jannfis" +approvers: + - "@alexmt" + - "@crenshaw-dev" + - "@jannfis" + +creation-date: 2023-06-23 +last-updated: 2024-02-06 +--- + +# Decouple Application Sync using Impersonation + +Application syncs in Argo CD have the same privileges as the Argo CD control plane. As a consequence, in a multi-tenant setup, the Argo CD control plane privileges needs to match the tenant that needs the highest privileges. As an example, if an Argo CD instance has 10 Applications and only one of them requires admin privileges, then the Argo CD control plane must have admin privileges in order to be able to sync that one Application. Argo CD provides a multi-tenancy model to restrict what each Application can do using `AppProjects`, even though the control plane has higher privileges, however that creates a large attack surface since if Argo CD is compromised, attackers would have cluster-admin access to the cluster. + +The goal of this proposal is to perform the Application sync as a different user using impersonation and use the service account provided in the cluster config purely for control plane operations. + +### What is Impersonation + +Impersonation is a feature in Kubernetes and enabled in the `kubectl` CLI client, using which, a user can act as another user through impersonation headers. For example, an admin could use this feature to debug an authorization policy by temporarily impersonating another user and seeing if a request was denied. + +Impersonation requests first authenticate as the requesting user, then switch to the impersonated user info. + +``` +kubectl --as ... +kubectl --as --as-group ... +``` + +## Open Questions [optional] + +- Should the restrictions imposed as part of the `AppProjects` be honored if the impersonation feature is enabled ? +>Yes, other restrictions implemented by `AppProject` related to whitelisting/blacklisting resources must continue to be honoured. +- Can an Application refer to a service account with elevated privileges like say `cluster-admin`, `admin`, and service accounts used for running the ArgoCD controllers itself ? +>Yes, this is possible as long as the ArgoCD admin user explicitly allows it through the `AppProject` configuration. +- Among the destinations configured in the `AppProject`, if there are multiple matches for a given destination, which destination option should be used ? +>If there are more than one matching destination, either with a glob pattern match or an exact match, then we use the first valid match to determine the service account to be used for the sync operation. +- Can the kubernetes audit trail events capture the impersonation. +>Yes, kubernetes audit trail events capture both the actual user and the impersonating user details and hence its possible to track who executed the commands and as which user permissions using the audit trails. +- Would the Sync hooks be using the impersonation service account. +>Yes, if the impersonation feature is enabled and customers use Sync hooks, then impersonation service account would be used for executing the hook jobs as well. +- If application resources have hardcoded namespaces in the git repository, would different service accounts be used for each resource during the sync operation ? +>The service account to be used for impersonation is determined on a per Application level rather than on per resource level. The value specified in `Application.spec.destination.namespace` would be used to determine the service account to be used for the sync operation of all resources present in the `Application`. + +## Summary + +In a multi team/multi tenant environment, an application team is typically granted access to a namespace to self-manage their Applications in a declarative way. Current implementation of ArgoCD requires the ArgoCD Administrator to create an `AppProject` with access settings configured to replicate the RBAC resources that are configured for each team. This approach requires duplication of effort and also requires syncing the access between both to maintain the security posture. It would be desirable for users to use the existing RBAC rules without having to revert to Argo CD API to create and manage these Applications. One namespace per team, or even one namespace per application is what we are looking to address as part of this proposal. + +## Motivation + +This proposal would allow ArgoCD administrators to manage the cluster permissions using kubernetes native RBAC implementation rather than using complex configurations in `AppProjects` to restrict access to individual applications. By decoupling the privileges required for application sync from the privileges required for ArgoCD control plane operations, the security requirement of providing least privileges can be achieved there by improving the security posture of ArgoCD. For implementing multi team/tenant use cases, this decoupling would be greatly beneficial. + +### Assumptions + +- Namespaces are pre-populated with one or more `ServiceAccounts` that define the permissions for each `AppProject`. +- Many users prefer to control access to kubernetes resources through kubernetes RBAC constructs instead of Argo specific constructs. +- Each tenant is generally given access to a specific namespace along with a service account, role or cluster role and role binding to control access to that namespace. +- `Applications` created by a tenant manage namespaced resources. +- An `AppProject` can either be mapped to a single tenant or multiple related tenants and the respective destinations that needs to be managed via the `AppProject`, needs to be configured. + + +### Goals +- Applications may only impersonate ServiceAccounts that live in the same namespace as the destination namespace configured in the application.If the service account is created in a different namespace, then the user can provide the service account name in the format `:` . ServiceAccount to be used for syncing each application is determined by the target destination configured in the `AppProject` associated with the `Application`. +- If impersonation feature is enabled, and no service account name is provided in the associated `AppProject`, then the default service account of the destination namespace of the `Application` should be used. +- Access restrictions implemented through properties in AppProject (if done) must have the existing behavior. From a security standpoint, any restrictions that were available before switching to a service account based approach should continue to exist even when the impersonation feature is enabled. + +### Non-Goals + +None + +## Proposal + +As part of this proposal, it would be possible for an ArgoCD Admin to specify a service account name in `AppProjects` CR for a single or a group of destinations. A destination is uniquely identified by a target cluster and a namespace combined. + +When applications gets synced, based on its destination (target cluster and namespace combination), the `defaultServiceAccount` configured in the `AppProject` will be selected and used for impersonation when executing the kubectl commands for the sync operation. + +We would be introducing a new element `destinationServiceAccounts` in `AppProject.spec`. This element is used for the sole purpose of specifying the impersonation configuration. The `defaultServiceAccount` configured for the `AppProject` would be used for the sync operation for a particular destination cluster and namespace. If impersonation feature is enabled and no specific service account is provided in the `AppProject` CR, then the `default` service account in the destination namespace would be used for impersonation. + +``` +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: my-project + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: Example Project + # Allow manifests to deploy from any Git repos + sourceRepos: + - '*' + destinations: + - * + destinationServiceAccounts: + - server: https://kubernetes.default.svc + namespace: guestbook + defaultServiceAccount: guestbook-deployer + - server: https://kubernetes.default.svc + namespace: guestbook-dev + defaultServiceAccount: guestbook-dev-deployer + - server: https://kubernetes.default.svc + namespace: guestbook-stage + defaultServiceAccount: guestbook-stage-deployer +``` + +### Structure of DestinationServiceAccount: +|Parameter| Type | Required/Optional| Description| +| ------ | ------ | ------- | -------- | +| server | string | Required | Server specifies the URL of the target cluster's Kubernetes control plane API. Glob patterns are supported. | +| namespace | string | Required | Namespace specifies the target namespace for the application's resources. Glob patterns are supported. | +| defaultServiceAccount | string | Required| DefaultServiceAccount specifies the service account to be impersonated when performing the `Application` sync operation.| + +**Note:** Only server URL for the target cluster is supported and target cluster name is not supported. + +### Future enhancements + +In a future release, we plan to support overriding of service accounts at the application level. In that case, we would be adding an element called `allowedServiceAccounts` to `AppProject.spec.destinationServiceAccounts[*]` + +### Use cases + +#### Use case 1: + +As a user, I would like to use kubernetes security constructs to restrict user access for application sync +So that, I can provide granular permissions based on the principle of least privilege required for syncing an application. + +#### Use case 2: + +As a user, I would like to configure a common service account for all applications associated to an AppProject +So that, I can use a generic convention of naming service accounts and avoid associating the service account per application. + +### Design considerations + +- Extending the `destinations` field under `AppProjects` was an option that was considered. But since the intent of it was to restrict the destinations that an associated `Application` can use, it was not used. Also the destination fields allowed negation operator (`!`) which would complicate the service account matching logic. The decision to create a new struct under `AppProject.Spec` for specifying the service account for each destination was considered a better alternative. + +- The field name `defaultServiceAccount` was chosen instead of `serviceAccount` as we wanted to support overriding of the service account at an `Application` at a later point in time and wanted to reserve the name `serviceAccount` for future extension. + +- Not supporting all impersonation options at the moment to keep the initial design to a minimum. Based on the need and feedback, support to impersonate users or groups can be added in future. + +### Implementation Details/Notes/Constraints + +#### Component : GitOps Engine + +- Fix GitOps Engine code to honor Impersonate configuration set in the Application sync context for all kubectl commands that are being executed. + +#### Component: ArgoCD API + +- Create a new struct type `DestinationServiceAccount` having fields `namespace`, `server` and `defaultServiceAccount` +- Create a new field `DestinationServiceAccounts` under a `AppProject.Spec` that takes in a list of `DestinationServiceAccount` objects. +- Add Documentation for newly introduced struct and its fields for `DestinationServiceAccount` and `DestinationServiceAccounts` under `AppProject.Spec` + +#### Component: ArgoCD Application Controller + +- Provide a configuration in `argocd-cm` which can be modified to enable the Impersonation feature. Set `applicationcontroller.enable.impersonation: true` in the Argo CD ConfigMap. Default value of `applicationcontroller.enable.impersonation` would be `false` and user has to explicitly override it to use this feature. +- Provide an option to override the Impersonation feature using environment variables. +Set `ARGOCD_APPLICATION_CONTROLLER_ENABLE_IMPERSONATION=true` in the Application controller environment variables. Default value of the environment variable must be `false` and user has to explicitly set it to `true` to use this feature. +- Provide an option to enable this feature using a command line flag `--enable-impersonation`. This new argument option needs to be added to the Application controller args. +- Fix Application Controller `sync.go` to set the Impersonate configuration from the AppProject CR to the `SyncContext` Object (rawConfig and restConfig field, need to understand which config is used for the actual sync and if both configs need to be impersonated.) + +#### Component: ArgoCD UI + +- Provide option to create `DestinationServiceAccount` with fields `namespace`, `server` and `defaultServiceAccount`. +- Provide option to add multiple `DestinationServiceAccounts` to an `AppProject` created/updated via the web console. +- Update the User Guide documentation on how to use these newly added fields from the web console. + +#### Component: ArgoCD CLI + +- Provide option to create `DestinationServiceAccount` with fields `namespace`, `server` and `defaultServiceAccount`. +- Provide option to add multiple `DestinationServiceAccounts` to an `AppProject` created/updated via the web console. +- Update the User Guide and other documentation where the CLI option usages are explained. + +#### Component: Documentation + +- Add note that this is a Beta feature in the documentation. +- Add a separate section for this feature under user-guide section. +- Update the ArgoCD CLI command reference documentation. +- Update the ArgoCD UI command reference documentation. + +### Detailed examples + +#### Example 1: Service account for application sync specified at the AppProject level for all namespaces + +In this specific scenario, service account name `generic-deployer` will get used for the application sync as the namespace `guestbook` matches the glob pattern `*`. + +- Install ArgoCD in the `argocd` namespace. +``` +kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-cd/master/manifests/install.yaml -n argocd +``` + +- Enable the impersonation feature in ArgoCD. +``` +kubectl set env statefulset/argocd-application-controller ARGOCD_APPLICATION_CONTROLLER_ENABLE_IMPERSONATION=true +``` + +- Create a namespace called `guestbook` and a service account called `guestbook-deployer`. +``` +kubectl create namespace guestbook +kubectl create serviceaccount guestbook-deployer +``` + +- Create Role and RoleBindings and configure RBAC access for creating `Service` and `Deployment` objects in namespace `guestbook` for service account `guestbook-deployer`. +``` +kubectl create role guestbook-deployer-role --verb get,list,update,delete --resource pods,deployment,service +kubectl create rolebinding guestbook-deployer-rb --serviceaccount guestbook-deployer --role guestbook-deployer-role +``` + +- Create the `Application` in the `argocd` namespace and the required `AppProject` as below +``` +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: guestbook + namespace: argocd +spec: + project: my-project + source: + repoURL: https://github.com/argoproj/argocd-example-apps.git + targetRevision: HEAD + path: guestbook + destination: + server: https://kubernetes.default.svc + namespace: guestbook +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: my-project + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: Example Project + # Allow manifests to deploy from any Git repos + sourceRepos: + - '*' + destinations: + - namespace: * + server: https://kubernetes.default.svc + destinationServiceAccounts: + - namespace: * + server: https://kubernetes.default.svc + defaultServiceAccount: generic-deployer +``` + +#### Example 2: Service account for application sync specified at the AppProject level for specific namespaces + +In this specific scenario, service account name `guestbook-deployer` will get used for the application sync as the namespace `guestbook` matches the target namespace `guestbook`. + +- Install ArgoCD in the `argocd` namespace. +``` +kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-cd/master/manifests/install.yaml -n argocd +``` + +- Enable the impersonation feature in ArgoCD. +``` +kubectl set env statefulset/argocd-application-controller ARGOCD_APPLICATION_CONTROLLER_ENABLE_IMPERSONATION=true +``` + +- Create a namespace called `guestbook` and a service account called `guestbook-deployer`. +``` +kubectl create namespace guestbook +kubectl create serviceaccount guestbook-deployer +``` +- Create Role and RoleBindings and configure RBAC access for creating `Service` and `Deployment` objects in namespace `guestbook` for service account `guestbook-deployer`. +``` +kubectl create role guestbook-deployer-role --verb get,list,update,delete --resource pods,deployment,service +kubectl create rolebinding guestbook-deployer-rb --serviceaccount guestbook-deployer --role guestbook-deployer-role +``` + +In this specific scenario, service account name `guestbook-deployer` will get used as it matches to the specific namespace `guestbook`. +``` +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: guestbook + namespace: argocd +spec: + project: my-project + source: + repoURL: https://github.com/argoproj/argocd-example-apps.git + targetRevision: HEAD + path: guestbook + destination: + server: https://kubernetes.default.svc + namespace: guestbook +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: my-project + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: Example Project + # Allow manifests to deploy from any Git repos + sourceRepos: + - '*' + destinations: + - namespace: guestbook + server: https://kubernetes.default.svc + - namespace: guestbook-ui + server: https://kubernetes.default.svc + destinationServiceAccounts: + - namespace: guestbook + server: https://kubernetes.default.svc + defaultServiceAccount: guestbook-deployer + - namespace: guestbook-ui + server: https://kubernetes.default.svc + defaultServiceAccount: guestbook-ui-deployer +``` + +#### Example 3: Remote destination with cluster-admin access and using different service account for the sync operation + +**Note**: In this example, we are relying on the default service account `argocd-manager` with `cluster-admin` privileges which gets created when adding a remote cluster destination using the ArgoCD CLI. + +- Install ArgoCD in the `argocd` namespace. +``` +kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-cd/master/manifests/install.yaml -n argocd +``` + +- Enable the impersonation feature in ArgoCD. +``` +kubectl set env statefulset/argocd-application-controller ARGOCD_APPLICATION_CONTROLLER_ENABLE_IMPERSONATION=true +``` + +- Add the remote cluster as a destination to argocd +``` +argocd cluster add remote-cluster --name remote-cluster +``` +**Note:** The above command would create a service account named `argocd-manager` in `kube-system` namespace and `ClusterRole` named `argocd-manager-role` with full cluster admin access and a `ClusterRoleBinding` named `argocd-manager-role-binding` mapping the `argocd-manager-role` to the service account `remote-cluster` + +- In the remote cluster, create a namespace called `guestbook` and a service account called `guestbook-deployer`. +``` +kubectl ctx remote-cluster +kubectl create namespace guestbook +kubectl create serviceaccount guestbook-deployer +``` + +- In the remote cluster, create `Role` and `RoleBindings` and configure RBAC access for creating `Service` and `Deployment` objects in namespace `guestbook` for service account `guestbook-deployer`. + +``` +kubectl ctx remote-cluster +kubectl create role guestbook-deployer-role --verb get,list,update,delete --resource pods,deployment,service +kubectl create rolebinding guestbook-deployer-rb --serviceaccount guestbook-deployer --role guestbook-deployer-role +``` + +- Create the `Application` and `AppProject` for the `guestbook` application. +``` +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: guestbook + namespace: argocd +spec: + project: my-project + source: + repoURL: https://github.com/argoproj/argocd-example-apps.git + targetRevision: HEAD + path: guestbook + destination: + server: https://kubernetes.default.svc + namespace: guestbook +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: my-project + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: Example Project + # Allow manifests to deploy from any Git repos + sourceRepos: + - '*' + destinations: + - namespace: guestbook + server: https://kubernetes.default.svc + serviceAccountName: guestbook-deployer + destinationServiceAccounts: + - namespace: guestbook + server: https://kubernetes.default.svc + defaultServiceAccount: guestbook-deployer +``` + +#### Example 4: Remote destination with a custom service account for the sync operation + +**Note**: In this example, we are relying on a non default service account `guestbook` created in the target cluster and namespace for the sync operation. This use case is for handling scenarios where the remote cluster is managed by a different administrator and providing a service account with `cluster-admin` level access is not feasible. + +- Install ArgoCD in the `argocd` namespace. +``` +kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-cd/master/manifests/install.yaml -n argocd +``` + +- Enable the impersonation feature in ArgoCD. +``` +kubectl set env statefulset/argocd-application-controller ARGOCD_APPLICATION_CONTROLLER_ENABLE_IMPERSONATION=true +``` + +- In the remote cluster, create a service account called `argocd-admin` +``` +kubectl ctx remote-cluster +kubectl create serviceaccount argocd-admin +kubectl create clusterrole argocd-admin-role --verb=impersonate --resource="users,groups,serviceaccounts" +kubectl create clusterrole argocd-admin-role-access-review --verb=create --resource="selfsubjectaccessreviews" +kubectl create clusterrolebinding argocd-admin-role-binding --serviceaccount argocd-admin --clusterrole argocd-admin-role +kubectl create clusterrolebinding argocd-admin-access-review-role-binding --serviceaccount argocd-admin --clusterrole argocd-admin-role +``` + +- In the remote cluster, create a namespace called `guestbook` and a service account called `guestbook-deployer`. +``` +kubectl ctx remote-cluster +kubectl create namespace guestbook +kubectl create serviceaccount guestbook-deployer +``` + +- In the remote cluster, create `Role` and `RoleBindings` and configure RBAC access for creating `Service` and `Deployment` objects in namespace `guestbook` for service account `guestbook-deployer`. +``` +kubectl create role guestbook-deployer-role --verb get,list,update,delete --resource pods,deployment,service +kubectl create rolebinding guestbook-deployer-rb --serviceaccount guestbook-deployer --role guestbook-deployer-role +``` + +In this specific scenario, service account name `guestbook-deployer` will get used as it matches to the specific namespace `guestbook`. +``` +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: guestbook + namespace: argocd +spec: + project: my-project + source: + repoURL: https://github.com/argoproj/argocd-example-apps.git + targetRevision: HEAD + path: guestbook + destination: + server: https://kubernetes.default.svc + namespace: guestbook +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: my-project + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: Example Project + # Allow manifests to deploy from any Git repos + sourceRepos: + - '*' + destinations: + - namespace: guestbook + server: https://kubernetes.default.svc + - namespace: guestbook-ui + server: https://kubernetes.default.svc + destinationServiceAccounts: + - namespace: guestbook + server: https://kubernetes.default.svc + defaultServiceAccount: guestbook-deployer + - namespace: guestbook-ui + server: https://kubernetes.default.svc + defaultServiceAccount: guestbook-ui-deployer +``` + +### Special cases + +#### Specifying service account in a different namespace + +By default, the service account would be looked up in the Application's destination namespace configured through `Application.Spec.Destination.Namespace` field. If the service account is in a different namespace, then users can provide the namespace of the service account explicitly in the format : +eg: +``` + ... + destinationServiceAccounts: + - server: https://kubernetes.default.svc + namespace: * + defaultServiceAccount: mynamespace:guestbook-deployer + ... +``` + +#### Multiple matches of destinations + +If there are multiple matches for a given destination, the first valid match in the list of `destinationServiceAccounts` would be used. + +eg: +Lets assume that the `AppProject` has the below `destinationServiceAccounts` configured. +``` + ... + destinationServiceAccounts: + - server: https://kubernetes.default.svc + namespace: guestbook-prod + defaultServiceAccount: guestbook-prod-deployer + - server: https://kubernetes.default.svc + namespace: guestbook-* + defaultServiceAccount: guestbook-generic-deployer + - server: https://kubernetes.default.svc + namespace: * + defaultServiceAccount: generic-deployer + ... +``` +- If the application destination namespace is `myns`, then the service account `generic-deployer` would be used as the first valid match is the glob pattern `*` and there are no other valid matches in the list. +- If the application destination namespace is `guestbook-dev` or `guestbook-stage`, then both glob patterns `*` and `guestbook-*` are valid matches, however `guestbook-*` pattern appears first and hence, the service account `guestbook-generic-deployer` would be used for the impersonation. +- If the application destination namespace is `guestbook-prod`, then there are three candidates, however the first valid match in the list is the one with service account `guestbook-prod-deployer` and that would be used for the impersonation. + +#### Application resources referring to multiple namespaces +If application resources have hardcoded namespaces in the git repository, would different service accounts be used for each resource during the sync operation ? + +The service account to be used for impersonation is determined on a per Application level rather than on per resource level. The value specified in `Application.spec.destination.namespace` would be used to determine the service account to be used for the sync operation of all resources present in the `Application`. + +### Security Considerations + +* How does this proposal impact the security aspects of Argo CD workloads ? +* Are there any unresolved follow-ups that need to be done to make the enhancement more robust ? + +### Risks and Mitigations + +#### Privilege Escalation + +There could be an issue of privilege escalation, if we allow users to impersonate without restrictions. This is mitigated by only allowing admin users to configure service account used for the sync operation at the `AppProject` level. + +Instead of allowing users to impersonate all possible users, administrators can restrict the users a particular service account can impersonate using the `resourceNames` field in the RBAC spec. + + +### Upgrade / Downgrade Strategy + +If applicable, how will the component be upgraded and downgraded? Make sure this is in the test +plan. + +Consider the following in developing an upgrade/downgrade strategy for this enhancement: + +- What changes (in invocations, configurations, API use, etc.) is an existing cluster required to + make on upgrade in order to keep previous behavior? +- What changes (in invocations, configurations, API use, etc.) is an existing cluster required to + make on upgrade in order to make use of the enhancement? + +- This feature would be implemented on an `opt-in` based on a feature flag and disabled by default. +- The new struct being added to `AppProject.Spec` would be introduced as an optional field and would be enabled only if the feature is enabled explicitly by a feature flag. If new property is used in the CR, but the feature flag is not enabled, then a warning message would be displayed during reconciliation of such CRs. + + +## Drawbacks + +- When using this feature, there is an overhead in creating namespaces, service accounts and the required RBAC policies and mapping the service accounts with the corresponding `AppProject` configuration. + +## Alternatives + +### Option 1 +Allow all options available in the `ImpersonationConfig` available to the user through the `AppProject` CRs. + +``` +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: my-project + namespace: argocd +spec: + description: Example Project + # Allow manifests to deploy from any Git repos + sourceRepos: + - '*' + destinations: + - namespace: * + server: https://kubernetes.default.svc + namespace: guestbook + impersonate: + user: system:serviceaccount:dev_ns:admin + uid: 1234 + groups: + - admin + - view + - edit +``` + +### Related issue + +https://github.com/argoproj/argo-cd/issues/7689 + + +### Related links + +https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation + +### Prior art + +https://github.com/argoproj/argo-cd/pull/3377 +https://github.com/argoproj/argo-cd/pull/7651 \ No newline at end of file diff --git a/docs/proposals/native-oci-support.md b/docs/proposals/native-oci-support.md index 64918fde8904e..7ec0053729c2e 100644 --- a/docs/proposals/native-oci-support.md +++ b/docs/proposals/native-oci-support.md @@ -126,10 +126,10 @@ Consider the following in developing an upgrade/downgrade strategy for this enha ## Drawbacks -* Sourcing content from an OCI registry may be perceived to be against GitOps principles as content is not sourced from a Git repository. This concern could be mitigated by attaching additional details related to the content (such as original Git source [URL, revision]). Though it should be noted that the GitOps principles only require a source of truth to be visioned and immutable which OCI registires support. +* Sourcing content from an OCI registry may be perceived to be against GitOps principles as content is not sourced from a Git repository. This concern could be mitigated by attaching additional details related to the content (such as original Git source [URL, revision]). Though it should be noted that the GitOps principles only require a source of truth to be visioned and immutable which OCI registries support. ## Alternatives ### Config Management Plugin -Content stored within OCI artifacts could be sourced using a Config Management Plugin which would not require changes to the core capabilities provided by Argo CD. However, this would be hacky and not represent itself within the Argo CD UI. \ No newline at end of file +Content stored within OCI artifacts could be sourced using a Config Management Plugin which would not require changes to the core capabilities provided by Argo CD. However, this would be hacky and not represent itself within the Argo CD UI. diff --git a/docs/requirements.txt b/docs/requirements.txt index 5ffcd4ff0221b..d350ac4870ee2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,6 @@ mkdocs==1.3.0 +# Strict mode has been disabled in latest versions of mkdocs-material. +# Thus pointing to the older version of mkdocs-material. mkdocs-material==7.1.8 markdown_include==0.6.0 pygments==2.15.0 diff --git a/docs/user-guide/commands/argocd_admin_app_generate-spec.md b/docs/user-guide/commands/argocd_admin_app_generate-spec.md index 78213de5c170c..af171470f4343 100644 --- a/docs/user-guide/commands/argocd_admin_app_generate-spec.md +++ b/docs/user-guide/commands/argocd_admin_app_generate-spec.md @@ -67,6 +67,7 @@ argocd admin app generate-spec APPNAME [flags] --kustomize-force-common-annotation Force common annotations in Kustomize --kustomize-force-common-label Force common labels in Kustomize --kustomize-image stringArray Kustomize images (e.g. --kustomize-image node:8.15.0 --kustomize-image mysql=mariadb,alpine@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d) + --kustomize-label-without-selector Do not apply common label to selectors or templates --kustomize-namespace string Kustomize namespace --kustomize-replica stringArray Kustomize replicas (e.g. --kustomize-replica my-development=2 --kustomize-replica my-statefulset=4) --kustomize-version string Kustomize version @@ -79,6 +80,7 @@ argocd admin app generate-spec APPNAME [flags] --path string Path in repository to the app directory, ignored if a file is set --plugin-env stringArray Additional plugin envs --project string Application project name + --ref string Ref is reference to another source within sources field --release-name string Helm release-name --repo string Repository URL, ignored if a file is set --revision string The tracking source branch, tag, commit or Helm chart version the application will sync to diff --git a/docs/user-guide/commands/argocd_admin_cluster.md b/docs/user-guide/commands/argocd_admin_cluster.md index bad60a0dd32bf..544c0de08959c 100644 --- a/docs/user-guide/commands/argocd_admin_cluster.md +++ b/docs/user-guide/commands/argocd_admin_cluster.md @@ -62,6 +62,6 @@ argocd admin cluster namespaces my-cluster * [argocd admin cluster generate-spec](argocd_admin_cluster_generate-spec.md) - Generate declarative config for a cluster * [argocd admin cluster kubeconfig](argocd_admin_cluster_kubeconfig.md) - Generates kubeconfig for the specified cluster * [argocd admin cluster namespaces](argocd_admin_cluster_namespaces.md) - Print information namespaces which Argo CD manages in each cluster. -* [argocd admin cluster shards](argocd_admin_cluster_shards.md) - Print information about each controller shard and portion of Kubernetes resources it is responsible for. +* [argocd admin cluster shards](argocd_admin_cluster_shards.md) - Print information about each controller shard and the estimated portion of Kubernetes resources it is responsible for. * [argocd admin cluster stats](argocd_admin_cluster_stats.md) - Prints information cluster statistics and inferred shard number diff --git a/docs/user-guide/commands/argocd_admin_cluster_generate-spec.md b/docs/user-guide/commands/argocd_admin_cluster_generate-spec.md index cc24418b023f8..79f88233fab32 100644 --- a/docs/user-guide/commands/argocd_admin_cluster_generate-spec.md +++ b/docs/user-guide/commands/argocd_admin_cluster_generate-spec.md @@ -13,6 +13,7 @@ argocd admin cluster generate-spec CONTEXT [flags] ``` --annotation stringArray Set metadata annotations (e.g. --annotation key=value) --aws-cluster-name string AWS Cluster name if set then aws cli eks token command will be used to access cluster + --aws-profile string Optional AWS profile. If set then AWS IAM Authenticator uses this profile to perform cluster operations instead of the default AWS credential provider chain. --aws-role-arn string Optional AWS role arn. If set then AWS IAM Authenticator assumes a role to perform cluster operations instead of the default AWS credential provider chain. --bearer-token string Authentication token that should be used to access K8S API server --cluster-endpoint string Cluster endpoint to use. Can be one of the following: 'kubeconfig', 'kube-public', or 'internal'. diff --git a/docs/user-guide/commands/argocd_admin_cluster_shards.md b/docs/user-guide/commands/argocd_admin_cluster_shards.md index 6648b91b2199e..48f6138d47b4a 100644 --- a/docs/user-guide/commands/argocd_admin_cluster_shards.md +++ b/docs/user-guide/commands/argocd_admin_cluster_shards.md @@ -2,7 +2,7 @@ ## argocd admin cluster shards -Print information about each controller shard and portion of Kubernetes resources it is responsible for. +Print information about each controller shard and the estimated portion of Kubernetes resources it is responsible for. ``` argocd admin cluster shards [flags] @@ -43,6 +43,7 @@ argocd admin cluster shards [flags] --sentinelmaster string Redis sentinel master group name. (default "master") --server string The address and port of the Kubernetes API server --shard int Cluster shard filter (default -1) + --sharding-method string Sharding method. Defaults: legacy. Supported sharding methods are : [legacy, round-robin] (default "legacy") --tls-server-name string If provided, this name will be used to validate server certificate. If this is not provided, hostname used to contact the server is used. --token string Bearer token for authentication to the API server --user string The name of the kubeconfig user to use diff --git a/docs/user-guide/commands/argocd_admin_cluster_stats.md b/docs/user-guide/commands/argocd_admin_cluster_stats.md index 960fd12caaef1..c5297ce7e35ed 100644 --- a/docs/user-guide/commands/argocd_admin_cluster_stats.md +++ b/docs/user-guide/commands/argocd_admin_cluster_stats.md @@ -57,6 +57,7 @@ argocd admin cluster stats target-cluster --sentinelmaster string Redis sentinel master group name. (default "master") --server string The address and port of the Kubernetes API server --shard int Cluster shard filter (default -1) + --sharding-method string Sharding method. Defaults: legacy. Supported sharding methods are : [legacy, round-robin] (default "legacy") --tls-server-name string If provided, this name will be used to validate server certificate. If this is not provided, hostname used to contact the server is used. --token string Bearer token for authentication to the API server --user string The name of the kubeconfig user to use diff --git a/docs/user-guide/commands/argocd_app.md b/docs/user-guide/commands/argocd_app.md index 543fcd96035ec..ff8fe0d4a01b6 100644 --- a/docs/user-guide/commands/argocd_app.md +++ b/docs/user-guide/commands/argocd_app.md @@ -78,6 +78,7 @@ argocd app [flags] * [argocd](argocd.md) - argocd controls a Argo CD server * [argocd app actions](argocd_app_actions.md) - Manage Resource actions +* [argocd app add-source](argocd_app_add-source.md) - Adds a source to the list of sources in the application * [argocd app create](argocd_app_create.md) - Create an application * [argocd app delete](argocd_app_delete.md) - Delete an application * [argocd app delete-resource](argocd_app_delete-resource.md) - Delete resource in an application @@ -90,6 +91,7 @@ argocd app [flags] * [argocd app manifests](argocd_app_manifests.md) - Print manifests of an application * [argocd app patch](argocd_app_patch.md) - Patch application * [argocd app patch-resource](argocd_app_patch-resource.md) - Patch resource in an application +* [argocd app remove-source](argocd_app_remove-source.md) - Remove a source from multiple sources application. Index starts with 0. * [argocd app resources](argocd_app_resources.md) - List resource of application * [argocd app rollback](argocd_app_rollback.md) - Rollback application to a previous deployed version by History ID, omitted will Rollback to the previous version * [argocd app set](argocd_app_set.md) - Set application parameters diff --git a/docs/user-guide/commands/argocd_app_add-source.md b/docs/user-guide/commands/argocd_app_add-source.md new file mode 100644 index 0000000000000..9ce5ce5a941c7 --- /dev/null +++ b/docs/user-guide/commands/argocd_app_add-source.md @@ -0,0 +1,108 @@ +# `argocd app add-source` Command Reference + +## argocd app add-source + +Adds a source to the list of sources in the application + +``` +argocd app add-source APPNAME [flags] +``` + +### Examples + +``` + # Append a source to the list of sources in the application + argocd app add-source guestbook --repo https://github.com/argoproj/argocd-example-apps.git --path guestbook +``` + +### Options + +``` + --allow-empty Set allow zero live resources when sync is automated + --auto-prune Set automatic pruning when sync is automated + --config-management-plugin string Config management plugin name + --dest-name string K8s cluster Name (e.g. minikube) + --dest-namespace string K8s target namespace + --dest-server string K8s cluster URL (e.g. https://kubernetes.default.svc) + --directory-exclude string Set glob expression used to exclude files from application source path + --directory-include string Set glob expression used to include files from application source path + --directory-recurse Recurse directory + --env string Application environment to monitor + --helm-chart string Helm Chart name + --helm-pass-credentials Pass credentials to all domain + --helm-set stringArray Helm set values on the command line (can be repeated to set several values: --helm-set key1=val1 --helm-set key2=val2) + --helm-set-file stringArray Helm set values from respective files specified via the command line (can be repeated to set several values: --helm-set-file key1=path1 --helm-set-file key2=path2) + --helm-set-string stringArray Helm set STRING values on the command line (can be repeated to set several values: --helm-set-string key1=val1 --helm-set-string key2=val2) + --helm-skip-crds Skip helm crd installation step + --helm-version string Helm version + -h, --help help for add-source + --ignore-missing-value-files Ignore locally missing valueFiles when setting helm template --values + --jsonnet-ext-var-code stringArray Jsonnet ext var + --jsonnet-ext-var-str stringArray Jsonnet string ext var + --jsonnet-libs stringArray Additional jsonnet libs (prefixed by repoRoot) + --jsonnet-tla-code stringArray Jsonnet top level code arguments + --jsonnet-tla-str stringArray Jsonnet top level string arguments + --kustomize-common-annotation stringArray Set common labels in Kustomize + --kustomize-common-label stringArray Set common labels in Kustomize + --kustomize-force-common-annotation Force common annotations in Kustomize + --kustomize-force-common-label Force common labels in Kustomize + --kustomize-image stringArray Kustomize images (e.g. --kustomize-image node:8.15.0 --kustomize-image mysql=mariadb,alpine@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d) + --kustomize-label-without-selector Do not apply common label to selectors or templates + --kustomize-namespace string Kustomize namespace + --kustomize-replica stringArray Kustomize replicas (e.g. --kustomize-replica my-development=2 --kustomize-replica my-statefulset=4) + --kustomize-version string Kustomize version + --nameprefix string Kustomize nameprefix + --namesuffix string Kustomize namesuffix + -p, --parameter stringArray set a parameter override (e.g. -p guestbook=image=example/guestbook:latest) + --path string Path in repository to the app directory, ignored if a file is set + --plugin-env stringArray Additional plugin envs + --project string Application project name + --ref string Ref is reference to another source within sources field + --release-name string Helm release-name + --repo string Repository URL, ignored if a file is set + --revision string The tracking source branch, tag, commit or Helm chart version the application will sync to + --revision-history-limit int How many items to keep in revision history (default 10) + --self-heal Set self healing when sync is automated + --sync-option Prune=false Add or remove a sync option, e.g add Prune=false. Remove using `!` prefix, e.g. `!Prune=false` + --sync-policy string Set the sync policy (one of: none, automated (aliases of automated: auto, automatic)) + --sync-retry-backoff-duration duration Sync retry backoff base duration. Input needs to be a duration (e.g. 2m, 1h) (default 5s) + --sync-retry-backoff-factor int Factor multiplies the base duration after each failed sync retry (default 2) + --sync-retry-backoff-max-duration duration Max sync retry backoff duration. Input needs to be a duration (e.g. 2m, 1h) (default 3m0s) + --sync-retry-limit int Max number of allowed sync retries + --validate Validation of repo and cluster (default true) + --values stringArray Helm values file(s) to use + --values-literal-file string Filename or URL to import as a literal Helm values block +``` + +### Options inherited from parent commands + +``` + --auth-token string Authentication token + --client-crt string Client certificate file + --client-crt-key string Client certificate key file + --config string Path to Argo CD config (default "/home/user/.config/argocd/config") + --controller-name string Name of the Argo CD Application controller; set this or the ARGOCD_APPLICATION_CONTROLLER_NAME environment variable when the controller's name label differs from the default, for example when installing via the Helm chart (default "argocd-application-controller") + --core If set to true then CLI talks directly to Kubernetes instead of talking to Argo CD API server + --grpc-web Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. + --grpc-web-root-path string Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. Set web root. + -H, --header strings Sets additional header to all requests made by Argo CD CLI. (Can be repeated multiple times to add multiple headers, also supports comma separated headers) + --http-retry-max int Maximum number of retries to establish http connection to Argo CD server + --insecure Skip server certificate and domain verification + --kube-context string Directs the command to the given kube-context + --logformat string Set the logging format. One of: text|json (default "text") + --loglevel string Set the logging level. One of: debug|info|warn|error (default "info") + --plaintext Disable TLS + --port-forward Connect to a random argocd-server port using port forwarding + --port-forward-namespace string Namespace name which should be used for port forwarding + --redis-haproxy-name string Name of the Redis HA Proxy; set this or the ARGOCD_REDIS_HAPROXY_NAME environment variable when the HA Proxy's name label differs from the default, for example when installing via the Helm chart (default "argocd-redis-ha-haproxy") + --redis-name string Name of the Redis deployment; set this or the ARGOCD_REDIS_NAME environment variable when the Redis's name label differs from the default, for example when installing via the Helm chart (default "argocd-redis") + --repo-server-name string Name of the Argo CD Repo server; set this or the ARGOCD_REPO_SERVER_NAME environment variable when the server's name label differs from the default, for example when installing via the Helm chart (default "argocd-repo-server") + --server string Argo CD server address + --server-crt string Server certificate file + --server-name string Name of the Argo CD API server; set this or the ARGOCD_SERVER_NAME environment variable when the server's name label differs from the default, for example when installing via the Helm chart (default "argocd-server") +``` + +### SEE ALSO + +* [argocd app](argocd_app.md) - Manage applications + diff --git a/docs/user-guide/commands/argocd_app_create.md b/docs/user-guide/commands/argocd_app_create.md index 41a671f3efdcd..0171f257c671c 100644 --- a/docs/user-guide/commands/argocd_app_create.md +++ b/docs/user-guide/commands/argocd_app_create.md @@ -65,6 +65,7 @@ argocd app create APPNAME [flags] --kustomize-force-common-annotation Force common annotations in Kustomize --kustomize-force-common-label Force common labels in Kustomize --kustomize-image stringArray Kustomize images (e.g. --kustomize-image node:8.15.0 --kustomize-image mysql=mariadb,alpine@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d) + --kustomize-label-without-selector Do not apply common label to selectors or templates --kustomize-namespace string Kustomize namespace --kustomize-replica stringArray Kustomize replicas (e.g. --kustomize-replica my-development=2 --kustomize-replica my-statefulset=4) --kustomize-version string Kustomize version @@ -76,6 +77,7 @@ argocd app create APPNAME [flags] --path string Path in repository to the app directory, ignored if a file is set --plugin-env stringArray Additional plugin envs --project string Application project name + --ref string Ref is reference to another source within sources field --release-name string Helm release-name --repo string Repository URL, ignored if a file is set --revision string The tracking source branch, tag, commit or Helm chart version the application will sync to diff --git a/docs/user-guide/commands/argocd_app_delete.md b/docs/user-guide/commands/argocd_app_delete.md index f4ff666a4b919..827eeaab4ce7a 100644 --- a/docs/user-guide/commands/argocd_app_delete.md +++ b/docs/user-guide/commands/argocd_app_delete.md @@ -28,10 +28,12 @@ argocd app delete APPNAME [flags] ### Options ``` + -N, --app-namespace string Namespace where the application will be deleted from --cascade Perform a cascaded deletion of all application resources (default true) -h, --help help for delete -p, --propagation-policy string Specify propagation policy for deletion of application's resources. One of: foreground|background (default "foreground") -l, --selector string Delete all apps with matching label. Supports '=', '==', '!=', in, notin, exists & not exists. Matching apps must satisfy all of the specified label constraints. + --wait Wait until deletion of the application(s) completes -y, --yes Turn off prompting to confirm cascaded deletion of application resources ``` diff --git a/docs/user-guide/commands/argocd_app_diff.md b/docs/user-guide/commands/argocd_app_diff.md index 139584d4fead5..18cc8f4751324 100644 --- a/docs/user-guide/commands/argocd_app_diff.md +++ b/docs/user-guide/commands/argocd_app_diff.md @@ -17,6 +17,7 @@ argocd app diff APPNAME [flags] ### Options ``` + -N, --app-namespace string Only render the difference in namespace --exit-code Return non-zero exit code when there is a diff (default true) --hard-refresh Refresh application data as well as target manifests cache -h, --help help for diff diff --git a/docs/user-guide/commands/argocd_app_edit.md b/docs/user-guide/commands/argocd_app_edit.md index 204e96cb76c0f..e581677b79c12 100644 --- a/docs/user-guide/commands/argocd_app_edit.md +++ b/docs/user-guide/commands/argocd_app_edit.md @@ -11,7 +11,8 @@ argocd app edit APPNAME [flags] ### Options ``` - -h, --help help for edit + -N, --app-namespace string Only edit application in namespace + -h, --help help for edit ``` ### Options inherited from parent commands diff --git a/docs/user-guide/commands/argocd_app_get.md b/docs/user-guide/commands/argocd_app_get.md index cf766ed9eb0d7..d0bf744054c38 100644 --- a/docs/user-guide/commands/argocd_app_get.md +++ b/docs/user-guide/commands/argocd_app_get.md @@ -42,12 +42,13 @@ argocd app get APPNAME [flags] ### Options ``` - --hard-refresh Refresh application data as well as target manifests cache - -h, --help help for get - -o, --output string Output format. One of: json|yaml|wide|tree (default "wide") - --refresh Refresh application data when retrieving - --show-operation Show application operation - --show-params Show application parameters and overrides + -N, --app-namespace string Only get application from namespace + --hard-refresh Refresh application data as well as target manifests cache + -h, --help help for get + -o, --output string Output format. One of: json|yaml|wide|tree (default "wide") + --refresh Refresh application data when retrieving + --show-operation Show application operation + --show-params Show application parameters and overrides ``` ### Options inherited from parent commands diff --git a/docs/user-guide/commands/argocd_app_history.md b/docs/user-guide/commands/argocd_app_history.md index 253a1dec64dd5..eefadef01f417 100644 --- a/docs/user-guide/commands/argocd_app_history.md +++ b/docs/user-guide/commands/argocd_app_history.md @@ -11,8 +11,9 @@ argocd app history APPNAME [flags] ### Options ``` - -h, --help help for history - -o, --output string Output format. One of: wide|id (default "wide") + -N, --app-namespace string Only show application deployment history in namespace + -h, --help help for history + -o, --output string Output format. One of: wide|id (default "wide") ``` ### Options inherited from parent commands diff --git a/docs/user-guide/commands/argocd_app_patch.md b/docs/user-guide/commands/argocd_app_patch.md index 01147f022c755..0c453ea159e64 100644 --- a/docs/user-guide/commands/argocd_app_patch.md +++ b/docs/user-guide/commands/argocd_app_patch.md @@ -21,9 +21,10 @@ argocd app patch APPNAME [flags] ### Options ``` - -h, --help help for patch - --patch string Patch body - --type string The type of patch being provided; one of [json merge] (default "json") + -N, --app-namespace string Only patch application in namespace + -h, --help help for patch + --patch string Patch body + --type string The type of patch being provided; one of [json merge] (default "json") ``` ### Options inherited from parent commands diff --git a/docs/user-guide/commands/argocd_app_remove-source.md b/docs/user-guide/commands/argocd_app_remove-source.md new file mode 100644 index 0000000000000..a334cbd37b5b1 --- /dev/null +++ b/docs/user-guide/commands/argocd_app_remove-source.md @@ -0,0 +1,56 @@ +# `argocd app remove-source` Command Reference + +## argocd app remove-source + +Remove a source from multiple sources application. Index starts with 0. + +``` +argocd app remove-source APPNAME [flags] +``` + +### Examples + +``` + # Remove the source at index 1 from application's sources + argocd app remove-source myapplication --source-index 1 +``` + +### Options + +``` + -h, --help help for remove-source + --source-index int Index of the source from the list of sources of the app. Index starts from 0. (default -1) +``` + +### Options inherited from parent commands + +``` + --auth-token string Authentication token + --client-crt string Client certificate file + --client-crt-key string Client certificate key file + --config string Path to Argo CD config (default "/home/user/.config/argocd/config") + --controller-name string Name of the Argo CD Application controller; set this or the ARGOCD_APPLICATION_CONTROLLER_NAME environment variable when the controller's name label differs from the default, for example when installing via the Helm chart (default "argocd-application-controller") + --core If set to true then CLI talks directly to Kubernetes instead of talking to Argo CD API server + --grpc-web Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. + --grpc-web-root-path string Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. Set web root. + -H, --header strings Sets additional header to all requests made by Argo CD CLI. (Can be repeated multiple times to add multiple headers, also supports comma separated headers) + --http-retry-max int Maximum number of retries to establish http connection to Argo CD server + --insecure Skip server certificate and domain verification + --kube-context string Directs the command to the given kube-context + --logformat string Set the logging format. One of: text|json (default "text") + --loglevel string Set the logging level. One of: debug|info|warn|error (default "info") + --plaintext Disable TLS + --port-forward Connect to a random argocd-server port using port forwarding + --port-forward-namespace string Namespace name which should be used for port forwarding + --redis-haproxy-name string Name of the Redis HA Proxy; set this or the ARGOCD_REDIS_HAPROXY_NAME environment variable when the HA Proxy's name label differs from the default, for example when installing via the Helm chart (default "argocd-redis-ha-haproxy") + --redis-name string Name of the Redis deployment; set this or the ARGOCD_REDIS_NAME environment variable when the Redis's name label differs from the default, for example when installing via the Helm chart (default "argocd-redis") + --repo-server-name string Name of the Argo CD Repo server; set this or the ARGOCD_REPO_SERVER_NAME environment variable when the server's name label differs from the default, for example when installing via the Helm chart (default "argocd-repo-server") + --server string Argo CD server address + --server-crt string Server certificate file + --server-name string Name of the Argo CD API server; set this or the ARGOCD_SERVER_NAME environment variable when the server's name label differs from the default, for example when installing via the Helm chart (default "argocd-server") +``` + +### SEE ALSO + +* [argocd app](argocd_app.md) - Manage applications + diff --git a/docs/user-guide/commands/argocd_app_rollback.md b/docs/user-guide/commands/argocd_app_rollback.md index bfcbf89631854..923023e35a2e8 100644 --- a/docs/user-guide/commands/argocd_app_rollback.md +++ b/docs/user-guide/commands/argocd_app_rollback.md @@ -11,10 +11,11 @@ argocd app rollback APPNAME [ID] [flags] ### Options ``` - -h, --help help for rollback - -o, --output string Output format. One of: json|yaml|wide|tree|tree=detailed (default "wide") - --prune Allow deleting unexpected resources - --timeout uint Time out after this many seconds + -N, --app-namespace string Rollback application in namespace + -h, --help help for rollback + -o, --output string Output format. One of: json|yaml|wide|tree|tree=detailed (default "wide") + --prune Allow deleting unexpected resources + --timeout uint Time out after this many seconds ``` ### Options inherited from parent commands diff --git a/docs/user-guide/commands/argocd_app_set.md b/docs/user-guide/commands/argocd_app_set.md index 18096e16f256a..75a50a77f3379 100644 --- a/docs/user-guide/commands/argocd_app_set.md +++ b/docs/user-guide/commands/argocd_app_set.md @@ -59,6 +59,7 @@ argocd app set APPNAME [flags] --kustomize-force-common-annotation Force common annotations in Kustomize --kustomize-force-common-label Force common labels in Kustomize --kustomize-image stringArray Kustomize images (e.g. --kustomize-image node:8.15.0 --kustomize-image mysql=mariadb,alpine@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d) + --kustomize-label-without-selector Do not apply common label to selectors or templates --kustomize-namespace string Kustomize namespace --kustomize-replica stringArray Kustomize replicas (e.g. --kustomize-replica my-development=2 --kustomize-replica my-statefulset=4) --kustomize-version string Kustomize version @@ -68,6 +69,7 @@ argocd app set APPNAME [flags] --path string Path in repository to the app directory, ignored if a file is set --plugin-env stringArray Additional plugin envs --project string Application project name + --ref string Ref is reference to another source within sources field --release-name string Helm release-name --repo string Repository URL, ignored if a file is set --revision string The tracking source branch, tag, commit or Helm chart version the application will sync to diff --git a/docs/user-guide/commands/argocd_app_sync.md b/docs/user-guide/commands/argocd_app_sync.md index 81ce3fd024c5c..a0a8f8459eeaa 100644 --- a/docs/user-guide/commands/argocd_app_sync.md +++ b/docs/user-guide/commands/argocd_app_sync.md @@ -38,6 +38,7 @@ argocd app sync [APPNAME... | -l selector | --project project-name] [flags] ### Options ``` + -N, --app-namespace string Only sync an application in namespace --apply-out-of-sync-only Sync only out-of-sync resources --assumeYes Assume yes as answer for all user queries or prompts --async Do not wait for application to sync before continuing diff --git a/docs/user-guide/commands/argocd_app_wait.md b/docs/user-guide/commands/argocd_app_wait.md index 99e422167b76f..e2d3886f4d3ab 100644 --- a/docs/user-guide/commands/argocd_app_wait.md +++ b/docs/user-guide/commands/argocd_app_wait.md @@ -38,7 +38,9 @@ argocd app wait [APPNAME.. | -l selector] [flags] ### Options ``` + -N, --app-namespace string Only wait for an application in namespace --degraded Wait for degraded + --delete Wait for delete --health Wait for health -h, --help help for wait --operation Wait for pending operations diff --git a/docs/user-guide/commands/argocd_cluster_add.md b/docs/user-guide/commands/argocd_cluster_add.md index 6d3a094b4bf83..8a80a12f5a4d5 100644 --- a/docs/user-guide/commands/argocd_cluster_add.md +++ b/docs/user-guide/commands/argocd_cluster_add.md @@ -13,6 +13,7 @@ argocd cluster add CONTEXT [flags] ``` --annotation stringArray Set metadata annotations (e.g. --annotation key=value) --aws-cluster-name string AWS Cluster name if set then aws cli eks token command will be used to access cluster + --aws-profile string Optional AWS profile. If set then AWS IAM Authenticator uses this profile to perform cluster operations instead of the default AWS credential provider chain. --aws-role-arn string Optional AWS role arn. If set then AWS IAM Authenticator assumes a role to perform cluster operations instead of the default AWS credential provider chain. --cluster-endpoint string Cluster endpoint to use. Can be one of the following: 'kubeconfig', 'kube-public', or 'internal'. --cluster-resources Indicates if cluster level resources should be managed. The setting is used only if list of managed namespaces is not empty. diff --git a/docs/user-guide/commands/argocd_proj.md b/docs/user-guide/commands/argocd_proj.md index 17aeef0cdfc27..5586463adee6e 100644 --- a/docs/user-guide/commands/argocd_proj.md +++ b/docs/user-guide/commands/argocd_proj.md @@ -84,6 +84,7 @@ argocd proj [flags] * [argocd proj add-orphaned-ignore](argocd_proj_add-orphaned-ignore.md) - Add a resource to orphaned ignore list * [argocd proj add-signature-key](argocd_proj_add-signature-key.md) - Add GnuPG signature key to project * [argocd proj add-source](argocd_proj_add-source.md) - Add project source repository +* [argocd proj add-source-namespace](argocd_proj_add-source-namespace.md) - Add source namespace to the AppProject * [argocd proj allow-cluster-resource](argocd_proj_allow-cluster-resource.md) - Adds a cluster-scoped API resource to the allow list and removes it from deny list * [argocd proj allow-namespace-resource](argocd_proj_allow-namespace-resource.md) - Removes a namespaced API resource from the deny list or add a namespaced API resource to the allow list * [argocd proj create](argocd_proj_create.md) - Create a project @@ -97,6 +98,7 @@ argocd proj [flags] * [argocd proj remove-orphaned-ignore](argocd_proj_remove-orphaned-ignore.md) - Remove a resource from orphaned ignore list * [argocd proj remove-signature-key](argocd_proj_remove-signature-key.md) - Remove GnuPG signature key from project * [argocd proj remove-source](argocd_proj_remove-source.md) - Remove project source repository +* [argocd proj remove-source-namespace](argocd_proj_remove-source-namespace.md) - Removes the source namespace from the AppProject * [argocd proj role](argocd_proj_role.md) - Manage a project's roles * [argocd proj set](argocd_proj_set.md) - Set project parameters * [argocd proj windows](argocd_proj_windows.md) - Manage a project's sync windows diff --git a/docs/user-guide/commands/argocd_proj_add-source-namespace.md b/docs/user-guide/commands/argocd_proj_add-source-namespace.md new file mode 100644 index 0000000000000..ced1f6fa3c67d --- /dev/null +++ b/docs/user-guide/commands/argocd_proj_add-source-namespace.md @@ -0,0 +1,55 @@ +# `argocd proj add-source-namespace` Command Reference + +## argocd proj add-source-namespace + +Add source namespace to the AppProject + +``` +argocd proj add-source-namespace PROJECT NAMESPACE [flags] +``` + +### Examples + +``` + # Add Kubernetes namespace as source namespace to the AppProject where application resources are allowed to be created in. + argocd proj add-source-namespace PROJECT NAMESPACE +``` + +### Options + +``` + -h, --help help for add-source-namespace +``` + +### Options inherited from parent commands + +``` + --auth-token string Authentication token + --client-crt string Client certificate file + --client-crt-key string Client certificate key file + --config string Path to Argo CD config (default "/home/user/.config/argocd/config") + --controller-name string Name of the Argo CD Application controller; set this or the ARGOCD_APPLICATION_CONTROLLER_NAME environment variable when the controller's name label differs from the default, for example when installing via the Helm chart (default "argocd-application-controller") + --core If set to true then CLI talks directly to Kubernetes instead of talking to Argo CD API server + --grpc-web Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. + --grpc-web-root-path string Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. Set web root. + -H, --header strings Sets additional header to all requests made by Argo CD CLI. (Can be repeated multiple times to add multiple headers, also supports comma separated headers) + --http-retry-max int Maximum number of retries to establish http connection to Argo CD server + --insecure Skip server certificate and domain verification + --kube-context string Directs the command to the given kube-context + --logformat string Set the logging format. One of: text|json (default "text") + --loglevel string Set the logging level. One of: debug|info|warn|error (default "info") + --plaintext Disable TLS + --port-forward Connect to a random argocd-server port using port forwarding + --port-forward-namespace string Namespace name which should be used for port forwarding + --redis-haproxy-name string Name of the Redis HA Proxy; set this or the ARGOCD_REDIS_HAPROXY_NAME environment variable when the HA Proxy's name label differs from the default, for example when installing via the Helm chart (default "argocd-redis-ha-haproxy") + --redis-name string Name of the Redis deployment; set this or the ARGOCD_REDIS_NAME environment variable when the Redis's name label differs from the default, for example when installing via the Helm chart (default "argocd-redis") + --repo-server-name string Name of the Argo CD Repo server; set this or the ARGOCD_REPO_SERVER_NAME environment variable when the server's name label differs from the default, for example when installing via the Helm chart (default "argocd-repo-server") + --server string Argo CD server address + --server-crt string Server certificate file + --server-name string Name of the Argo CD API server; set this or the ARGOCD_SERVER_NAME environment variable when the server's name label differs from the default, for example when installing via the Helm chart (default "argocd-server") +``` + +### SEE ALSO + +* [argocd proj](argocd_proj.md) - Manage projects + diff --git a/docs/user-guide/commands/argocd_proj_remove-source-namespace.md b/docs/user-guide/commands/argocd_proj_remove-source-namespace.md new file mode 100644 index 0000000000000..6a0ee319c7b9b --- /dev/null +++ b/docs/user-guide/commands/argocd_proj_remove-source-namespace.md @@ -0,0 +1,55 @@ +# `argocd proj remove-source-namespace` Command Reference + +## argocd proj remove-source-namespace + +Removes the source namespace from the AppProject + +``` +argocd proj remove-source-namespace PROJECT NAMESPACE [flags] +``` + +### Examples + +``` + # Remove source NAMESPACE in PROJECT + argocd proj remove-source-namespace PROJECT NAMESPACE +``` + +### Options + +``` + -h, --help help for remove-source-namespace +``` + +### Options inherited from parent commands + +``` + --auth-token string Authentication token + --client-crt string Client certificate file + --client-crt-key string Client certificate key file + --config string Path to Argo CD config (default "/home/user/.config/argocd/config") + --controller-name string Name of the Argo CD Application controller; set this or the ARGOCD_APPLICATION_CONTROLLER_NAME environment variable when the controller's name label differs from the default, for example when installing via the Helm chart (default "argocd-application-controller") + --core If set to true then CLI talks directly to Kubernetes instead of talking to Argo CD API server + --grpc-web Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. + --grpc-web-root-path string Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. Set web root. + -H, --header strings Sets additional header to all requests made by Argo CD CLI. (Can be repeated multiple times to add multiple headers, also supports comma separated headers) + --http-retry-max int Maximum number of retries to establish http connection to Argo CD server + --insecure Skip server certificate and domain verification + --kube-context string Directs the command to the given kube-context + --logformat string Set the logging format. One of: text|json (default "text") + --loglevel string Set the logging level. One of: debug|info|warn|error (default "info") + --plaintext Disable TLS + --port-forward Connect to a random argocd-server port using port forwarding + --port-forward-namespace string Namespace name which should be used for port forwarding + --redis-haproxy-name string Name of the Redis HA Proxy; set this or the ARGOCD_REDIS_HAPROXY_NAME environment variable when the HA Proxy's name label differs from the default, for example when installing via the Helm chart (default "argocd-redis-ha-haproxy") + --redis-name string Name of the Redis deployment; set this or the ARGOCD_REDIS_NAME environment variable when the Redis's name label differs from the default, for example when installing via the Helm chart (default "argocd-redis") + --repo-server-name string Name of the Argo CD Repo server; set this or the ARGOCD_REPO_SERVER_NAME environment variable when the server's name label differs from the default, for example when installing via the Helm chart (default "argocd-repo-server") + --server string Argo CD server address + --server-crt string Server certificate file + --server-name string Name of the Argo CD API server; set this or the ARGOCD_SERVER_NAME environment variable when the server's name label differs from the default, for example when installing via the Helm chart (default "argocd-server") +``` + +### SEE ALSO + +* [argocd proj](argocd_proj.md) - Manage projects + diff --git a/docs/user-guide/helm.md b/docs/user-guide/helm.md index 866f9c6d935aa..7a763336abcc8 100644 --- a/docs/user-guide/helm.md +++ b/docs/user-guide/helm.md @@ -25,9 +25,28 @@ spec: namespace: kubeseal ``` +Another example using a public OCI helm chart: +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: nginx +spec: + project: default + source: + chart: nginx + repoURL: registry-1.docker.io/bitnamicharts # note: the oci:// syntax is not included. + targetRevision: 15.9.0 + destination: + name: "in-cluster" + namespace: nginx +``` + !!! note "When using multiple ways to provide values" Order of precedence is `parameters > valuesObject > values > valueFiles > helm repository values.yaml` (see [Here](./helm.md#helm-value-precedence) for a more detailed example) +See [here](../operator-manual/declarative-setup.md#helm-chart-repositories) for more info about how to configure private Helm repositories. + ## Values Files Helm has the ability to use a different, or even multiple "values.yaml" files to derive its diff --git a/docs/user-guide/kustomize.md b/docs/user-guide/kustomize.md index 647e753649cce..4e45eb685e75f 100644 --- a/docs/user-guide/kustomize.md +++ b/docs/user-guide/kustomize.md @@ -7,6 +7,7 @@ The following configuration options are available for Kustomize: * `images` is a list of Kustomize image overrides * `replicas` is a list of Kustomize replica overrides * `commonLabels` is a string map of additional labels +* `labelWithoutSelector` is a boolean value which defines if the common label(s) should be applied to resource selectors and templates. * `forceCommonLabels` is a boolean value which defines if it's allowed to override existing labels * `commonAnnotations` is a string map of additional annotations * `namespace` is a Kubernetes resources namespace @@ -106,6 +107,37 @@ spec: namespace: default ``` +## Components +Kustomize [components](https://github.com/kubernetes-sigs/kustomize/blob/master/examples/components.md) encapsulate both resources and patches together. They provide a powerful way to modularize and reuse configuration in Kubernetes applications. + +Outside of Argo CD, to utilize components, you must add the following to the `kustomization.yaml` that the Application references. For example: +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +... +components: +- ../component +``` + +With support added for components in `v2.10.0`, you can now reference a component directly in the Application: +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: application-kustomize-components +spec: + ... + source: + path: examples/application-kustomize-components/base + repoURL: https://github.com/my-user/my-repo + targetRevision: main + + # This! + kustomize: + components: + - ../component # relative to the kustomization.yaml (`source.path`). +``` + ## Private Remote Bases If you have remote bases that are either (a) HTTPS and need username/password (b) SSH and need SSH private key, then they'll inherit that from the app's repo. diff --git a/docs/user-guide/resource_hooks.md b/docs/user-guide/resource_hooks.md index a6fdaf8bd2e05..6e15a55bb20c2 100644 --- a/docs/user-guide/resource_hooks.md +++ b/docs/user-guide/resource_hooks.md @@ -8,9 +8,9 @@ and after a Sync operation. Hooks can also be run if a Sync operation fails at a * Using a `Sync` hook to orchestrate a complex deployment requiring more sophistication than the Kubernetes rolling update strategy. * Using a `PostSync` hook to run integration and health checks after a deployment. -* Using a `SyncFail` hook to run clean-up or finalizer logic if a Sync operation fails. _`SyncFail` hooks are only available starting in v1.2_ -* Using a `PostDelete` hook to run clean-up or finalizer logic after an all Application resources are deleted. Please note that - `PostDelete` hooks are only deleted if delete policy matches to the aggregated deletion hooks status and not garbage collected after the application is deleted. +* Using a `SyncFail` hook to run clean-up or finalizer logic if a Sync operation fails. +* Using a `PostDelete` hook to run clean-up or finalizer logic after all Application resources are deleted. Please note that + `PostDelete` hooks are only deleted if the delete policy matches the aggregated deletion hooks status and not garbage collected after the application is deleted. ## Usage @@ -39,7 +39,8 @@ The following hooks are defined: | `Sync` | Executes after all `PreSync` hooks completed and were successful, at the same time as the application of the manifests. | | `Skip` | Indicates to Argo CD to skip the application of the manifest. | | `PostSync` | Executes after all `Sync` hooks completed and were successful, a successful application, and all resources in a `Healthy` state. | -| `SyncFail` | Executes when the sync operation fails. _Available starting in v1.2_ | +| `SyncFail` | Executes when the sync operation fails. | +| `PostDelete` | Executes after all Application resources are deleted. _Available starting in v2.10._ | ### Generate Name diff --git a/docs/user-guide/resource_tracking.md b/docs/user-guide/resource_tracking.md index 79eda63ce5d5a..e62a7c094f4e2 100644 --- a/docs/user-guide/resource_tracking.md +++ b/docs/user-guide/resource_tracking.md @@ -65,6 +65,11 @@ metadata: The advantages of using the tracking id annotation is that there are no clashes any more with other Kubernetes tools and Argo CD is never confused about the owner of a resource. The `annotation+label` can also be used if you want other tools to understand resources managed by Argo CD. +### Non self-referencing annotations +When using the tracking method `annotation` or `annotation+label`, Argo CD will consider the resource properties in the annotation (name, namespace, group and kind) to determine whether the resource should be compared against the desired state. If the tracking annotation does not reference the resource it is applied to, the resource will neither affect the application's sync status nor be marked for pruning. + +This allows other kubernetes tools (e.g. [HNC](https://github.com/kubernetes-sigs/hierarchical-namespaces)) to copy a resource to a different namespace without impacting the Argo CD application's sync status. Copied resources will be visible on the UI at top level. They will have no sync status and won't impact the application's sync status. + ## Choosing a tracking method To actually select your preferred tracking method edit the `resourceTrackingMethod` value contained inside the `argocd-cm` configmap. diff --git a/docs/user-guide/status-badge.md b/docs/user-guide/status-badge.md index 8355be458f026..3363227997309 100644 --- a/docs/user-guide/status-badge.md +++ b/docs/user-guide/status-badge.md @@ -9,7 +9,12 @@ To show this badge, use the following URL format `${argoCdBaseUrl}/api/badge?nam The URLs for status image are available on application details page: 1. Navigate to application details page and click on 'Details' button. -1. Scroll down to 'Status Badge' section. -1. Select required template such as URL, Markdown etc. +2. Scroll down to 'Status Badge' section. +3. Select required template such as URL, Markdown etc. for the status image URL in markdown, html, etc are available . -1. Copy the text and paste it into your README or website. \ No newline at end of file +4. Copy the text and paste it into your README or website. + +The application name may optionally be displayed in the status badge by adding the `?showAppName=true` query parameter. + +For example, `${argoCdBaseUrl}/api/badge?name=${appName}&showAppName=true`. +To remove the application name from the badge, remove the query parameter from the URL or set it to `false`. \ No newline at end of file diff --git a/docs/user-guide/sync-options.md b/docs/user-guide/sync-options.md index e5b1fe55e8e66..985f9fcf3c974 100644 --- a/docs/user-guide/sync-options.md +++ b/docs/user-guide/sync-options.md @@ -270,7 +270,7 @@ spec: - RespectIgnoreDifferences=true ``` -The example above shows how an Argo CD Application can be configured so it will ignore the `spec.replicas` field from the desired state (git) during the sync stage. This is achieve by calculating and pre-patching the desired state before applying it in the cluster. Note that the `RespectIgnoreDifferences` sync option is only effective when the resource is already created in the cluster. If the Application is being created and no live state exists, the desired state is applied as-is. +The example above shows how an Argo CD Application can be configured so it will ignore the `spec.replicas` field from the desired state (git) during the sync stage. This is achieved by calculating and pre-patching the desired state before applying it in the cluster. Note that the `RespectIgnoreDifferences` sync option is only effective when the resource is already created in the cluster. If the Application is being created and no live state exists, the desired state is applied as-is. ## Create Namespace diff --git a/docs/user-guide/sync-waves.md b/docs/user-guide/sync-waves.md index 932ba396d68d2..8b17237c87571 100644 --- a/docs/user-guide/sync-waves.md +++ b/docs/user-guide/sync-waves.md @@ -37,7 +37,7 @@ Hooks and resources are assigned to wave zero by default. The wave can be negati When Argo CD starts a sync, it orders the resources in the following precedence: * The phase -* The wave they are in (lower values first) +* The wave they are in (lower values first for creation & updation and higher values first for deletion) * By kind (e.g. [namespaces first and then other Kubernetes resources, followed by custom resources](https://github.com/argoproj/gitops-engine/blob/bc9ce5764fa306f58cf59199a94f6c968c775a2d/pkg/sync/sync_tasks.go#L27-L66)) * By name @@ -49,6 +49,8 @@ It repeats this process until all phases and waves are in-sync and healthy. Because an application can have resources that are unhealthy in the first wave, it may be that the app can never get to healthy. +During pruning of resources, resources from higher waves are processed first before moving to lower waves. If, for any reason, a resource isn't removed/pruned in a wave, the resources in next waves won't be processed. This is to ensure proper resource cleanup between waves. + Note that there's currently a delay between each sync wave in order give other controllers a chance to react to the spec change that we just applied. This also prevent Argo CD from assessing resource health too quickly (against the stale object), causing hooks to fire prematurely. The current delay between each sync wave is 2 seconds and can be configured via environment diff --git a/examples/dashboard.json b/examples/dashboard.json index 7e992a5363324..108ac81918ba3 100644 --- a/examples/dashboard.json +++ b/examples/dashboard.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -13,15 +16,17 @@ ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, - "iteration": 1605574886303, + "id": 28, "links": [], + "liveNow": false, "panels": [ { "collapsed": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 1, "w": 24, @@ -30,12 +35,21 @@ }, "id": 68, "panels": [], + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], "title": "Overview", "type": "row" }, { - "content": "![argoimage](https://avatars1.githubusercontent.com/u/30269780?s=110&v=4)", - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 4, "w": 2, @@ -44,29 +58,64 @@ }, "id": 26, "links": [], - "mode": "markdown", - "options": {}, - "title": "", + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "![argoimage](https://avatars1.githubusercontent.com/u/30269780?s=110&v=4)", + "mode": "markdown" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], "transparent": true, "type": "text" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "dtdurations", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurations", + "unitScale": true + }, + "overrides": [] }, "gridPos": { "h": 4, @@ -75,79 +124,77 @@ "y": 1 }, "id": 32, - "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "tableColumn": "", + "pluginVersion": "10.3.1", "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "time() - max(process_start_time_seconds{job=\"argocd-server-metrics\",namespace=~\"$namespace\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], - "thresholds": "", "title": "Uptime", - "type": "singlestat", - "valueFontSize": "70%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "type": "stat" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] }, "gridPos": { "h": 4, @@ -156,43 +203,30 @@ "y": 1 }, "id": 94, - "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "tableColumn": "", + "pluginVersion": "10.3.1", "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "count(count by (server) (argocd_cluster_info{namespace=~\"$namespace\"}))", "format": "time_series", "instant": false, @@ -200,40 +234,47 @@ "refId": "A" } ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, "title": "Clusters", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "type": "stat" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorPostfix": false, - "colorPrefix": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] }, "gridPos": { "h": 4, @@ -242,45 +283,31 @@ "y": 1 }, "id": 75, - "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": null, - "repeatDirection": "h", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "tableColumn": "", + "pluginVersion": "10.3.1", + "repeatDirection": "h", "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(argocd_app_info{namespace=~\"$namespace\",dest_server=~\"$cluster\",health_status=~\"$health_status\",sync_status=~\"$sync_status\"})", "format": "time_series", "instant": false, @@ -288,38 +315,47 @@ "refId": "A" } ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, "title": "Applications", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "type": "stat" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] }, "gridPos": { "h": 4, @@ -328,43 +364,30 @@ "y": 1 }, "id": 107, - "interval": null, "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "tableColumn": "", + "pluginVersion": "10.3.1", "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "count(count by (repo) (argocd_app_info{namespace=~\"$namespace\"}))", "format": "time_series", "instant": false, @@ -372,24 +395,47 @@ "refId": "A" } ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, "title": "Repositories", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" + "type": "stat" }, { - "cacheTimeout": null, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "0", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 4, "w": 3, @@ -399,47 +445,27 @@ "id": 100, "links": [], "options": { - "fieldOptions": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { "calcs": [ "lastNotNull" ], - "defaults": { - "mappings": [ - { - "id": 0, - "op": "=", - "text": "0", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ], - "unit": "none" - }, - "override": {}, - "overrides": [], + "fields": "", "values": false }, - "orientation": "horizontal", "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "6.5.2", + "pluginVersion": "10.3.1", "repeatDirection": "h", "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(argocd_app_info{namespace=~\"$namespace\",dest_server=~\"$cluster\",operation!=\"\"})", "format": "time_series", "instant": true, @@ -448,19 +474,24 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Operations", "type": "gauge" }, { "aliasColors": {}, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "$datasource", - "decimals": null, + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -491,10 +522,11 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -504,6 +536,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(argocd_app_info{namespace=~\"$namespace\",dest_server=~\"$cluster\",health_status=~\"$health_status\",sync_status=~\"$sync_status\"}) by (namespace)", "format": "time_series", "instant": false, @@ -513,9 +548,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Applications", "tooltip": { "shared": false, @@ -524,9 +557,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -534,29 +565,24 @@ { "decimals": 0, "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { "collapsed": true, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 1, "w": 24, @@ -575,11 +601,18 @@ "Unknown": "rgb(255, 255, 255)" }, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "$datasource", - "decimals": null, + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -601,7 +634,6 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, @@ -612,10 +644,11 @@ "links": [], "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -625,6 +658,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(argocd_app_info{namespace=~\"$namespace\",dest_server=~\"$cluster\",health_status=~\"$health_status\",sync_status=~\"$sync_status\",health_status!=\"\"}) by (health_status)", "format": "time_series", "instant": false, @@ -634,9 +670,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Health Status", "tooltip": { "shared": true, @@ -645,33 +679,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 2, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -686,11 +711,18 @@ "Unknown": "rgb(255, 255, 255)" }, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "$datasource", - "decimals": null, + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -712,7 +744,6 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, @@ -723,10 +754,11 @@ "links": [], "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -736,6 +768,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(argocd_app_info{namespace=~\"$namespace\",dest_server=~\"$cluster\",health_status=~\"$health_status\",sync_status=~\"$sync_status\",health_status!=\"\"}) by (sync_status)", "format": "time_series", "instant": false, @@ -745,9 +780,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Sync Status", "tooltip": { "shared": true, @@ -756,42 +789,43 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 2, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], "title": "Application Status", "type": "row" }, { "collapsed": true, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 1, "w": 24, @@ -805,8 +839,9 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", - "decimals": null, + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -851,6 +886,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(round(increase(argocd_app_sync_total{namespace=~\"$namespace\",dest_server=~\"$cluster\"}[$interval]))) by ($grouping)", "format": "time_series", "intervalFactor": 1, @@ -859,9 +897,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Sync Activity", "tooltip": { "shared": true, @@ -870,9 +906,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -882,7 +916,6 @@ "format": "short", "label": "", "logBase": 1, - "max": null, "min": "0", "show": true }, @@ -891,14 +924,11 @@ "format": "short", "label": "", "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -906,8 +936,9 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", - "decimals": null, + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -951,6 +982,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(round(increase(argocd_app_sync_total{namespace=~\"$namespace\",phase=~\"Error|Failed\",dest_server=~\"$cluster\"}[$interval]))) by ($grouping, phase)", "format": "time_series", "intervalFactor": 1, @@ -959,9 +993,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Sync Failures", "tooltip": { "shared": true, @@ -970,9 +1002,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -982,7 +1012,6 @@ "format": "none", "label": "", "logBase": 1, - "max": null, "min": "0", "show": true }, @@ -990,23 +1019,30 @@ "format": "short", "label": "", "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], "title": "Sync Stats", "type": "row" }, { "collapsed": true, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 1, "w": 24, @@ -1020,7 +1056,9 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1062,6 +1100,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(argocd_app_reconcile_count{namespace=~\"$namespace\",dest_server=~\"$cluster\"}[$interval])) by ($grouping)", "format": "time_series", "intervalFactor": 1, @@ -1070,9 +1111,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Reconciliation Activity", "tooltip": { "shared": false, @@ -1081,50 +1120,39 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, + "cards": {}, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, - "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 7, "w": 24, @@ -1143,6 +1171,9 @@ "reverseYBuckets": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(argocd_app_reconcile_bucket{namespace=~\"$namespace\"}[$interval])) by (le)", "format": "heatmap", "instant": false, @@ -1151,8 +1182,6 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Reconciliation Performance", "tooltip": { "show": true, @@ -1163,27 +1192,21 @@ "xAxis": { "show": true }, - "xBucketNumber": null, - "xBucketSize": null, "yAxis": { - "decimals": null, "format": "short", "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null + "show": true }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yBucketBound": "auto" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1204,7 +1227,6 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, @@ -1215,11 +1237,315 @@ "links": [], "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(argocd_app_k8s_request_total{namespace=~\"$namespace\",server=~\"$cluster\"}[$interval])) by (verb, resource_kind)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{verb}} {{resource_kind}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s API Activity", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(workqueue_depth{namespace=~\"$namespace\",name=~\"app_.*\"}) by (name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Workqueue Depth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(argocd_kubectl_exec_pending{namespace=~\"$namespace\"}) by (command)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{command}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Pending kubectl run", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Controller Stats", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "uid": "$datasource" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 102, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true }, "paceLength": 10, "percentage": false, - "pointradius": 2, + "pluginVersion": "10.3.1", + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -1228,53 +1554,44 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(argocd_app_k8s_request_total{namespace=~\"$namespace\",server=~\"$cluster\"}[$interval])) by (verb, resource_kind)", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_memstats_heap_alloc_bytes{job=\"argocd-server-metrics\",namespace=~\"$namespace\"}", "format": "time_series", - "instant": false, "intervalFactor": 1, - "legendFormat": "{{verb}} {{resource_kind}}", + "legendFormat": "{{namespace}}", "refId": "A" } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, - "title": "K8s API Activity", + "title": "Memory Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { - "format": "short", - "label": null, + "format": "bytes", "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1282,39 +1599,52 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 12, + "w": 24, "x": 0, - "y": 31 + "y": 16 }, "hiddenSeries": false, - "id": 96, + "id": 108, "legend": { "alignAsTable": true, "avg": true, "current": true, + "hideEmpty": true, "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, - "sideWidth": null, + "sort": "avg", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, + "paceLength": 10, "percentage": false, - "pointradius": 2, + "pluginVersion": "10.3.1", + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -1323,52 +1653,45 @@ "steppedLine": false, "targets": [ { - "expr": "sum(workqueue_depth{namespace=~\"$namespace\",name=~\"app_.*\"}) by (name)", + "datasource": { + "uid": "$datasource" + }, + "expr": "irate(process_cpu_seconds_total{job=\"argocd-server-metrics\",namespace=~\"$namespace\"}[1m])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{name}}", + "legendFormat": "{{namespace}}", "refId": "A" } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, - "title": "Workqueue Depth", + "title": "CPU Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { - "format": "short", - "label": null, + "decimals": 1, + "format": "none", "logBase": 1, - "max": null, - "min": "0", "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1376,26 +1699,38 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", - "decimals": null, + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 31 + "w": 24, + "x": 0, + "y": 23 }, "hiddenSeries": false, - "id": 98, + "id": 62, "legend": { "alignAsTable": true, "avg": true, "current": true, + "hideEmpty": false, "hideZero": false, "max": true, "min": false, + "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -1404,10 +1739,12 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, + "paceLength": 10, "percentage": false, - "pointradius": 2, + "pluginVersion": "10.3.1", + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -1416,63 +1753,64 @@ "steppedLine": false, "targets": [ { - "expr": "sum(argocd_kubectl_exec_pending{namespace=~\"$namespace\"}) by (command)", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_goroutines{job=\"argocd-server-metrics\",namespace=~\"$namespace\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{command}}", + "legendFormat": "{{namespace}}", "refId": "A" } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, - "title": "Pending kubectl run", + "title": "Goroutines", "tooltip": { - "shared": true, + "shared": false, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { - "decimals": 0, "format": "short", - "label": "", "logBase": 1, - "max": null, - "min": "0", "show": true }, { - "decimals": 0, "format": "short", - "label": "", "logBase": 1, - "max": null, - "min": "0", "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], - "title": "Controller Stats", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Controller Telemetry", "type": "row" }, + { "collapsed": true, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 1, "w": 24, @@ -1486,14 +1824,23 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 26 + "y": 9 }, "hiddenSeries": false, "id": 34, @@ -1515,10 +1862,11 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -1528,7 +1876,10 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_alloc_bytes{job=\"argocd-metrics\",namespace=~\"$namespace\"}", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_memstats_heap_alloc_bytes{job=\"argocd-applicationset-controller-metrics\",namespace=~\"$namespace\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{namespace}}", @@ -1536,9 +1887,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Memory Usage", "tooltip": { "shared": false, @@ -1547,33 +1896,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1581,14 +1921,23 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 33 + "y": 16 }, "hiddenSeries": false, "id": 108, @@ -1612,10 +1961,11 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -1625,7 +1975,10 @@ "steppedLine": false, "targets": [ { - "expr": "irate(process_cpu_seconds_total{job=\"argocd-metrics\",namespace=~\"$namespace\"}[1m])", + "datasource": { + "uid": "$datasource" + }, + "expr": "irate(process_cpu_seconds_total{job=\"argocd-applicationset-controller-metrics\",namespace=~\"$namespace\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{namespace}}", @@ -1633,9 +1986,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "CPU Usage", "tooltip": { "shared": false, @@ -1644,9 +1995,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1654,24 +2003,17 @@ { "decimals": 1, "format": "none", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1679,14 +2021,23 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 40 + "y": 23 }, "hiddenSeries": false, "id": 62, @@ -1710,10 +2061,11 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -1723,7 +2075,10 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"argocd-metrics\",namespace=~\"$namespace\"}", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_goroutines{job=\"argocd-applicationset-controller-metrics\",namespace=~\"$namespace\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{namespace}}", @@ -1731,9 +2086,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Goroutines", "tooltip": { "shared": false, @@ -1742,42 +2095,43 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], - "title": "Controller Telemetry", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "AppSet Controller Telemetry", "type": "row" }, { "collapsed": true, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 1, "w": 24, @@ -1791,7 +2145,9 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1832,6 +2188,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(argocd_cluster_api_resource_objects{namespace=~\"$namespace\",server=~\"$cluster\"}) by (server)", "format": "time_series", "intervalFactor": 1, @@ -1840,9 +2199,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Resource Objects Count", "tooltip": { "shared": false, @@ -1851,33 +2208,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1885,7 +2233,9 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1927,6 +2277,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": " sum(argocd_cluster_api_resources{namespace=~\"$namespace\",server=~\"$cluster\"}) by (server)", "format": "time_series", "intervalFactor": 1, @@ -1935,9 +2288,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "API Resources Count", "tooltip": { "shared": false, @@ -1946,33 +2297,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1980,7 +2322,9 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2021,6 +2365,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(argocd_cluster_events_total{namespace=~\"$namespace\",server=~\"$cluster\"}[$interval])) by (server)", "format": "time_series", "intervalFactor": 1, @@ -2029,9 +2376,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Cluster Events Count", "tooltip": { "shared": false, @@ -2040,42 +2385,43 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], "title": "Cluster Stats", "type": "row" }, { "collapsed": true, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 1, "w": 24, @@ -2089,14 +2435,23 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 11 }, "hiddenSeries": false, "id": 82, @@ -2114,9 +2469,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -2126,6 +2482,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(argocd_git_request_total{request_type=\"ls-remote\", namespace=~\"$namespace\"}[10m])) by (namespace)", "format": "time_series", "intervalFactor": 1, @@ -2134,9 +2493,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Git Requests (ls-remote)", "tooltip": { "shared": true, @@ -2145,33 +2502,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -2179,14 +2527,23 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 7 + "y": 11 }, "hiddenSeries": false, "id": 84, @@ -2204,9 +2561,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -2216,6 +2574,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(argocd_git_request_total{request_type=\"fetch\", namespace=~\"$namespace\"}[10m])) by (namespace)", "format": "time_series", "intervalFactor": 1, @@ -2224,9 +2585,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Git Requests (checkout)", "tooltip": { "shared": true, @@ -2235,9 +2594,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2246,29 +2603,20 @@ "format": "short", "label": "", "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, + "cards": {}, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", @@ -2277,12 +2625,30 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 19 }, "heatmap": {}, "hideZeroBuckets": false, @@ -2291,10 +2657,51 @@ "legend": { "show": false }, - "options": {}, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "10.3.1", "reverseYBuckets": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(argocd_git_request_duration_seconds_bucket{request_type=\"fetch\", namespace=~\"$namespace\"}[$interval])) by (le)", "format": "heatmap", "intervalFactor": 10, @@ -2302,8 +2709,6 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Git Fetch Performance", "tooltip": { "show": true, @@ -2313,26 +2718,15 @@ "xAxis": { "show": true }, - "xBucketNumber": null, - "xBucketSize": null, "yAxis": { - "decimals": null, "format": "short", "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null + "show": true }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yBucketBound": "auto" }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, + "cards": {}, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", @@ -2341,12 +2735,30 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 19 }, "heatmap": {}, "hideZeroBuckets": false, @@ -2355,10 +2767,51 @@ "legend": { "show": false }, - "options": {}, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "10.3.1", "reverseYBuckets": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(argocd_git_request_duration_seconds_bucket{request_type=\"ls-remote\", namespace=~\"$namespace\"}[$interval])) by (le)", "format": "heatmap", "intervalFactor": 10, @@ -2366,8 +2819,6 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Git Ls-Remote Performance", "tooltip": { "show": true, @@ -2377,34 +2828,28 @@ "xAxis": { "show": true }, - "xBucketNumber": null, - "xBucketSize": null, "yAxis": { - "decimals": null, "format": "short", "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null + "show": true }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yBucketBound": "auto" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 23 + "y": 27 }, "hiddenSeries": false, "id": 71, @@ -2435,7 +2880,10 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_alloc_bytes{job=\"argocd-repo-server\",namespace=~\"$namespace\"}", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_memstats_heap_alloc_bytes{job=\"argocd-repo-server-metrics\",namespace=~\"$namespace\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -2443,9 +2891,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Memory Used", "tooltip": { "shared": true, @@ -2454,33 +2900,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -2488,14 +2925,16 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 31 + "y": 35 }, "hiddenSeries": false, "id": 72, @@ -2526,7 +2965,10 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"argocd-repo-server\",namespace=~\"$namespace\"}", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_goroutines{job=\"argocd-repo-server-metrics\",namespace=~\"$namespace\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -2534,9 +2976,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Goroutines", "tooltip": { "shared": true, @@ -2545,42 +2985,43 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], "title": "Repo Server Stats", "type": "row" }, { "collapsed": true, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "gridPos": { "h": 1, "w": 24, @@ -2594,14 +3035,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 89 + "y": 12 }, + "hiddenSeries": false, "id": 61, "legend": { "avg": false, @@ -2616,8 +3067,12 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -2627,7 +3082,10 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_alloc_bytes{job=\"argocd-server-metrics\",namespace=~\"$namespace\"}", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_memstats_heap_alloc_bytes{job=\"argocd-repo-server-metrics\",namespace=~\"$namespace\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -2635,9 +3093,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Memory Used", "tooltip": { "shared": true, @@ -2646,33 +3102,25 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", - "label": null, "logBase": 1, - "max": null, "min": "0", "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -2680,14 +3128,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 24, "x": 0, - "y": 97 + "y": 20 }, + "hiddenSeries": false, "id": 36, "legend": { "avg": false, @@ -2702,8 +3160,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -2713,7 +3175,10 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"argocd-server-metrics\",namespace=~\"$namespace\"}", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_goroutines{job=\"argocd-repo-server-metrics\",namespace=~\"$namespace\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -2721,9 +3186,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Goroutines", "tooltip": { "shared": true, @@ -2732,33 +3195,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -2766,14 +3220,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 24, "x": 0, - "y": 106 + "y": 29 }, + "hiddenSeries": false, "id": 38, "legend": { "avg": false, @@ -2788,8 +3252,12 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -2799,7 +3267,10 @@ "steppedLine": false, "targets": [ { - "expr": "go_gc_duration_seconds{job=\"argocd-server-metrics\", quantile=\"1\", namespace=~\"$namespace\"}", + "datasource": { + "uid": "$datasource" + }, + "expr": "go_gc_duration_seconds{job=\"argocd-repo-server-metrics\", quantile=\"1\", namespace=~\"$namespace\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -2807,9 +3278,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "GC Time Quantiles", "tooltip": { "shared": true, @@ -2818,33 +3287,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -2853,12 +3313,11 @@ "h": 2, "w": 24, "x": 0, - "y": 115 + "y": 38 }, "id": 54, "links": [], "mode": "markdown", - "title": "", "transparent": true, "type": "text" }, @@ -2867,14 +3326,15 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", - "decimals": null, + "datasource": { + "uid": "$datasource" + }, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 117 + "y": 40 }, "id": 40, "legend": { @@ -2907,6 +3367,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"application.ApplicationService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", "format": "time_series", "intervalFactor": 1, @@ -2915,9 +3378,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "ApplicationService Requests", "tooltip": { "shared": false, @@ -2926,33 +3387,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -2960,13 +3412,15 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 117 + "y": 40 }, "id": 42, "legend": { @@ -2997,6 +3451,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"cluster.ClusterService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", "format": "time_series", "intervalFactor": 1, @@ -3005,9 +3462,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "ClusterService Requests", "tooltip": { "shared": false, @@ -3016,33 +3471,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -3050,13 +3496,15 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 126 + "y": 49 }, "id": 44, "legend": { @@ -3087,6 +3535,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"project.ProjectService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", "format": "time_series", "intervalFactor": 1, @@ -3095,9 +3546,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "ProjectService Requests", "tooltip": { "shared": true, @@ -3106,33 +3555,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -3140,13 +3580,15 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 126 + "y": 49 }, "id": 46, "legend": { @@ -3176,6 +3618,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"repository.RepositoryService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", "format": "time_series", "intervalFactor": 1, @@ -3192,9 +3637,7 @@ "yaxis": "left" } ], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "RepositoryService Requests", "tooltip": { "shared": true, @@ -3203,33 +3646,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -3237,13 +3671,15 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 135 + "y": 58 }, "id": 48, "legend": { @@ -3273,6 +3709,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"session.SessionService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", "format": "time_series", "intervalFactor": 1, @@ -3281,9 +3720,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "SessionService Requests", "tooltip": { "shared": true, @@ -3292,33 +3729,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -3326,13 +3754,15 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 135 + "y": 58 }, "id": 49, "legend": { @@ -3362,6 +3792,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"version.VersionService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", "format": "time_series", "intervalFactor": 1, @@ -3370,9 +3803,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "VersionService Requests", "tooltip": { "shared": true, @@ -3381,33 +3812,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -3415,13 +3837,15 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 144 + "y": 67 }, "id": 50, "legend": { @@ -3451,6 +3875,9 @@ "steppedLine": false, "targets": [ { + "datasource": { + "uid": "$datasource" + }, "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"account.AccountService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", "format": "time_series", "intervalFactor": 1, @@ -3459,9 +3886,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "AccountService Requests", "tooltip": { "shared": true, @@ -3470,33 +3895,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -3504,13 +3920,15 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": { + "uid": "$datasource" + }, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 144 + "y": 67 }, "id": 99, "legend": { @@ -3540,7 +3958,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"settings.SettingsService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(grpc_server_handled_total{job=\"argocd-server-metrics\",grpc_service=\"cluster.SettingsService\",namespace=~\"$namespace\"}[$interval])) by (grpc_code, grpc_method)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{grpc_code}},{{grpc_method}}", @@ -3548,9 +3969,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "SettingsService Requests", "tooltip": { "shared": true, @@ -3559,42 +3978,44 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "refId": "A" + } + ], "title": "Server Stats", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "gridPos": { "h": 1, "w": 24, @@ -3608,14 +4029,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 9 + "y": 13 }, "hiddenSeries": false, "id": 112, @@ -3632,9 +4063,10 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -3644,14 +4076,16 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "sum(increase(argocd_redis_request_total{namespace=~\"$namespace\"}[$interval])) by (failed)", "refId": "A" } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Requests by result", "tooltip": { "shared": true, @@ -3660,58 +4094,58 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A" + } + ], "title": "Redis Stats", "type": "row" } ], - "refresh": false, - "schemaVersion": 21, - "style": "dark", + "refresh": "", + "schemaVersion": 39, "tags": [], "templating": { "list": [ { "current": { + "selected": false, "text": "Prometheus", - "value": "Prometheus" + "value": "prometheus" }, "hide": 0, "includeAll": false, - "label": null, "multi": false, "name": "datasource", "options": [], "query": "prometheus", + "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -3724,11 +4158,13 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, "definition": "label_values(kube_pod_info, namespace)", "hide": 0, "includeAll": true, - "label": null, "multi": false, "name": "namespace", "options": [], @@ -3738,7 +4174,6 @@ "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -3753,7 +4188,6 @@ "value": "$__auto_interval_interval" }, "hide": 0, - "label": null, "name": "interval", "options": [ { @@ -3810,13 +4244,12 @@ { "allValue": "", "current": { - "selected": true, + "selected": false, "text": "namespace", "value": "namespace" }, "hide": 0, "includeAll": false, - "label": null, "multi": false, "name": "grouping", "options": [ @@ -3837,6 +4270,7 @@ } ], "query": "namespace,name,project", + "queryValue": "", "skipUrlSync": false, "type": "custom" }, @@ -3847,11 +4281,13 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, "definition": "label_values(argocd_cluster_info, server)", "hide": 0, "includeAll": true, - "label": null, "multi": false, "name": "cluster", "options": [], @@ -3861,7 +4297,6 @@ "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -3869,13 +4304,12 @@ { "allValue": ".*", "current": { - "selected": true, + "selected": false, "text": "All", "value": "$__all" }, "hide": 0, "includeAll": true, - "label": null, "multi": false, "name": "health_status", "options": [ @@ -3922,13 +4356,12 @@ { "allValue": ".*", "current": { - "selected": true, + "selected": false, "text": "All", "value": "$__all" }, "hide": 0, "includeAll": true, - "label": null, "multi": false, "name": "sync_status", "options": [ @@ -3991,5 +4424,6 @@ "timezone": "", "title": "ArgoCD", "uid": "LCAgc9rWz", - "version": 1 + "version": 2, + "weekStart": "" } \ No newline at end of file diff --git a/go.mod b/go.mod index 07dd99e4beff1..2f3bdec276c7c 100644 --- a/go.mod +++ b/go.mod @@ -13,10 +13,10 @@ require ( github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d github.com/alicebob/miniredis/v2 v2.30.4 github.com/antonmedv/expr v1.15.2 - github.com/argoproj/gitops-engine v0.7.1-0.20231218194513-aba38192fb16 - github.com/argoproj/notifications-engine v0.4.1-0.20231027194313-a8d185ecc0a9 + github.com/argoproj/gitops-engine v0.7.1-0.20240124052710-5fd9f449e757 + github.com/argoproj/notifications-engine v0.4.1-0.20240206192038-2daee6022f41 github.com/argoproj/pkg v0.13.7-0.20230626144333-d56162821bd1 - github.com/aws/aws-sdk-go v1.44.317 + github.com/aws/aws-sdk-go v1.50.8 github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/bombsimon/logrusr/v2 v2.0.1 github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 @@ -25,10 +25,12 @@ require ( github.com/coreos/go-oidc/v3 v3.6.0 github.com/cyphar/filepath-securejoin v0.2.4 github.com/dustin/go-humanize v1.0.1 - github.com/evanphx/json-patch v5.6.0+incompatible + github.com/evanphx/json-patch v5.9.0+incompatible + github.com/felixge/httpsnoop v1.0.3 github.com/fsnotify/fsnotify v1.6.0 github.com/gfleury/go-bitbucket-v1 v0.0.0-20220301131131-8e7ed04b843e - github.com/go-git/go-git/v5 v5.10.1 + github.com/go-git/go-git/v5 v5.11.0 + github.com/go-jose/go-jose/v3 v3.0.1 github.com/go-logr/logr v1.3.0 github.com/go-openapi/loads v0.21.2 github.com/go-openapi/runtime v0.26.0 @@ -88,11 +90,10 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 - gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.26.11 - k8s.io/apiextensions-apiserver v0.26.4 + k8s.io/apiextensions-apiserver v0.26.10 k8s.io/apimachinery v0.26.11 k8s.io/apiserver v0.26.11 k8s.io/client-go v0.26.11 @@ -103,7 +104,7 @@ require ( k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 layeh.com/gopher-json v0.0.0-20190114024228-97fed8db8427 oras.land/oras-go/v2 v2.3.0 - sigs.k8s.io/controller-runtime v0.14.6 + sigs.k8s.io/controller-runtime v0.14.7 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 sigs.k8s.io/yaml v1.3.0 ) @@ -114,19 +115,20 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.5.2 // indirect - github.com/aws/aws-sdk-go-v2 v1.17.3 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.8 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sqs v1.20.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.0 // indirect - github.com/aws/smithy-go v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.25.12 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect + github.com/aws/smithy-go v1.19.0 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect @@ -178,13 +180,11 @@ require ( github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/camelcase v1.0.0 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fvbommel/sortorder v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.21.4 // indirect github.com/go-openapi/errors v0.20.3 // indirect @@ -242,7 +242,7 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/client_model v0.3.0 github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -267,7 +267,7 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.starlark.net v0.0.0-20220328144851-d1966c6b9fcd // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.18.0 + golang.org/x/net v0.19.0 golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 diff --git a/go.sum b/go.sum index 0c5e889f6bdf6..495ba3ed9ba29 100644 --- a/go.sum +++ b/go.sum @@ -694,10 +694,10 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/appscode/go v0.0.0-20191119085241-0887d8ec2ecc/go.mod h1:OawnOmAL4ZX3YaPdN+8HTNwBveT1jMsqP74moa9XUbE= -github.com/argoproj/gitops-engine v0.7.1-0.20231218194513-aba38192fb16 h1:kR15L8UsSVr7oitABKU88msQirMT0/RO/KRla1jkq/s= -github.com/argoproj/gitops-engine v0.7.1-0.20231218194513-aba38192fb16/go.mod h1:gWE8uROi7hIkWGNAVM+8FWkMfo0vZ03SLx/aFw/DBzg= -github.com/argoproj/notifications-engine v0.4.1-0.20231027194313-a8d185ecc0a9 h1:1lt0VXzmLK7Vv0kaeal3S6/JIfzPyBORkUWXhiqF3l0= -github.com/argoproj/notifications-engine v0.4.1-0.20231027194313-a8d185ecc0a9/go.mod h1:E/vv4+by868m0mmflaRfGBmKBtAupoF+mmyfekP8QCk= +github.com/argoproj/gitops-engine v0.7.1-0.20240124052710-5fd9f449e757 h1:5fKAhTQcTBom0vin56cz/UTPx2GMuvdb+lJRAUOPbHA= +github.com/argoproj/gitops-engine v0.7.1-0.20240124052710-5fd9f449e757/go.mod h1:gWE8uROi7hIkWGNAVM+8FWkMfo0vZ03SLx/aFw/DBzg= +github.com/argoproj/notifications-engine v0.4.1-0.20240206192038-2daee6022f41 h1:PQE8LbcbRHdtnQzeEWwVU2QHXACKOA30yS3No5HSoTQ= +github.com/argoproj/notifications-engine v0.4.1-0.20240206192038-2daee6022f41/go.mod h1:TsyusmXQWIL0ST7YMRG/ered7WlWDmbmnPpXnS2LJmM= github.com/argoproj/pkg v0.13.7-0.20230626144333-d56162821bd1 h1:qsHwwOJ21K2Ao0xPju1sNuqphyMnMYkyB3ZLoLtxWpo= github.com/argoproj/pkg v0.13.7-0.20230626144333-d56162821bd1/go.mod h1:CZHlkyAD1/+FbEn6cB2DQTj48IoLGvEYsWEvtzP3238= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -713,35 +713,37 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.44.289/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.44.317 h1:+8XWrLmGMwPPXSRSLPzhgcGnzJ2mYkgkrcB9C/GnSOU= -github.com/aws/aws-sdk-go v1.44.317/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.50.8 h1:gY0WoOW+/Wz6XmYSgDH9ge3wnAevYDSQWPxxJvqAkP4= +github.com/aws/aws-sdk-go v1.50.8/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= -github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.8 h1:lDpy0WM8AHsywOnVrOHaSMfpaiV2igOw8D7svkFkXVA= -github.com/aws/aws-sdk-go-v2/config v1.18.8/go.mod h1:5XCmmyutmzzgkpk/6NYTjeWb6lgo9N170m1j6pQkIBs= -github.com/aws/aws-sdk-go-v2/credentials v1.13.8 h1:vTrwTvv5qAwjWIGhZDSBH/oQHuIQjGmD232k01FUh6A= -github.com/aws/aws-sdk-go-v2/credentials v1.13.8/go.mod h1:lVa4OHbvgjVot4gmh1uouF1ubgexSCN92P6CJQpT0t8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= -github.com/aws/aws-sdk-go-v2/service/sqs v1.20.0 h1:tQoMg8i4nFAB70cJ4wiAYEiZRYo2P6uDmU2D6ys/igo= -github.com/aws/aws-sdk-go-v2/service/sqs v1.20.0/go.mod h1:jQhN5f4p3PALMNlUtfb/0wGIFlV7vGtJlPDVfxfNfPY= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 h1:/2gzjhQowRLarkkBOGPXSRnb8sQ2RVsjdG1C/UliK/c= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.0/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 h1:Jfly6mRxk2ZOSlbCvZfKNS7TukSx1mIzhSsqZ/IGSZI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.0 h1:kOO++CYo50RcTFISESluhWEi5Prhg+gaSs4whWabiZU= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.0/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= +github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/config v1.25.12 h1:mF4cMuNh/2G+d19nWnm1vJ/ak0qK6SbqF0KtSX9pxu0= +github.com/aws/aws-sdk-go-v2/config v1.25.12/go.mod h1:lOvvqtZP9p29GIjOTuA/76HiVk0c/s8qRcFRq2+E2uc= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= +github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 h1:tRNrFDGRm81e6nTX5Q4CFblea99eAfm0dxXazGpLceU= +github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7/go.mod h1:8GWUDux5Z2h6z2efAtr54RdHXtLm8sq7Rg85ZNY/CZM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -875,8 +877,8 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBF github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= @@ -927,8 +929,8 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+ github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.10.1 h1:tu8/D8i+TWxgKpzQ3Vc43e+kkhXqtsZCKI/egajKnxk= -github.com/go-git/go-git/v5 v5.10.1/go.mod h1:uEuHjxkHap8kAl//V5F/nNWwqIYtP/402ddd05mp0wg= +github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= +github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -1960,8 +1962,8 @@ golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2597,8 +2599,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/retry.v1 v1.0.3 h1:a9CArYczAVv6Qs6VGoLMio99GEs7kY9UzSF9+LD+iGs= gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= @@ -2706,8 +2706,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= -sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/controller-runtime v0.14.7 h1:Vrnm2vk9ZFlRkXATHz0W0wXcqNl7kPat8q2JyxVy0Q8= +sigs.k8s.io/controller-runtime v0.14.7/go.mod h1:ErTs3SJCOujNUnTz4AS+uh8hp6DHMo1gj6fFndJT1X8= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= diff --git a/hack/installers/checksums/helm-v3.14.0-linux-amd64.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.0-linux-amd64.tar.gz.sha256 new file mode 100644 index 0000000000000..6f9aaf5a270d5 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.0-linux-amd64.tar.gz.sha256 @@ -0,0 +1 @@ +f43e1c3387de24547506ab05d24e5309c0ce0b228c23bd8aa64e9ec4b8206651 helm-v3.14.0-linux-amd64.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.0-linux-arm64.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.0-linux-arm64.tar.gz.sha256 new file mode 100644 index 0000000000000..d0e09bd4b41f7 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.0-linux-arm64.tar.gz.sha256 @@ -0,0 +1 @@ +b29e61674731b15f6ad3d1a3118a99d3cc2ab25a911aad1b8ac8c72d5a9d2952 helm-v3.14.0-linux-arm64.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.0-linux-ppc64le.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.0-linux-ppc64le.tar.gz.sha256 new file mode 100644 index 0000000000000..d179322b99dd5 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.0-linux-ppc64le.tar.gz.sha256 @@ -0,0 +1 @@ +f1f9d3561724863edd4c06d89acb2e2fd8ae0f1b72058ceb891fa1c346ce5dbc helm-v3.14.0-linux-ppc64le.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.0-linux-s390x.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.0-linux-s390x.tar.gz.sha256 new file mode 100644 index 0000000000000..31ff04397b29e --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.0-linux-s390x.tar.gz.sha256 @@ -0,0 +1 @@ +82298ef39936f1bef848959a29f77bff92d1309d8646657e3a7733702e81288c helm-v3.14.0-linux-s390x.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.1-linux-amd64.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.1-linux-amd64.tar.gz.sha256 new file mode 100644 index 0000000000000..cc06e12986311 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.1-linux-amd64.tar.gz.sha256 @@ -0,0 +1 @@ +75496ea824f92305ff7d28af37f4af57536bf5138399c824dff997b9d239dd42 helm-v3.14.1-linux-amd64.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.1-linux-arm64.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.1-linux-arm64.tar.gz.sha256 new file mode 100644 index 0000000000000..63f791b234ec4 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.1-linux-arm64.tar.gz.sha256 @@ -0,0 +1 @@ +f865b8ad4228fd0990bbc5b50615eb6cb9eb31c9a9ca7238401ed897bbbe9033 helm-v3.14.1-linux-arm64.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.1-linux-ppc64le.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.1-linux-ppc64le.tar.gz.sha256 new file mode 100644 index 0000000000000..17b9b1e625fac --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.1-linux-ppc64le.tar.gz.sha256 @@ -0,0 +1 @@ +4d853ab8fe3462287c7272fbadd5f73531ecdd6fa0db37d31630e41ae1ae21de helm-v3.14.1-linux-ppc64le.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.1-linux-s390x.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.1-linux-s390x.tar.gz.sha256 new file mode 100644 index 0000000000000..232ec10e03fc6 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.1-linux-s390x.tar.gz.sha256 @@ -0,0 +1 @@ +19bf07999c7244bfeb0fd27152919b9faa1148cf43910edbb98efa9150058a98 helm-v3.14.1-linux-s390x.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.2-linux-amd64.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.2-linux-amd64.tar.gz.sha256 new file mode 100644 index 0000000000000..22049267fd24e --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.2-linux-amd64.tar.gz.sha256 @@ -0,0 +1 @@ +0885a501d586c1e949e9b113bf3fb3290b0bbf74db9444a1d8c2723a143006a5 helm-v3.14.2-linux-amd64.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.2-linux-arm64.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.2-linux-arm64.tar.gz.sha256 new file mode 100644 index 0000000000000..17320419ee7e6 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.2-linux-arm64.tar.gz.sha256 @@ -0,0 +1 @@ +c65d6a9557bb359abc2c0d26670de850b52327dc3976ad6f9e14c298ea3e1b61 helm-v3.14.2-linux-arm64.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.2-linux-ppc64le.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.2-linux-ppc64le.tar.gz.sha256 new file mode 100644 index 0000000000000..8ffe4ebe40e62 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.2-linux-ppc64le.tar.gz.sha256 @@ -0,0 +1 @@ +f3bc8582ff151e619cd285d9cdf9fef1c5733ee5522d8bed2ef680ef07f87223 helm-v3.14.2-linux-ppc64le.tar.gz diff --git a/hack/installers/checksums/helm-v3.14.2-linux-s390x.tar.gz.sha256 b/hack/installers/checksums/helm-v3.14.2-linux-s390x.tar.gz.sha256 new file mode 100644 index 0000000000000..d14a74799e6a2 --- /dev/null +++ b/hack/installers/checksums/helm-v3.14.2-linux-s390x.tar.gz.sha256 @@ -0,0 +1 @@ +7bda34aa26638e5116b31385f3b781172572175bf4c1ae00c87d8b154458ed94 helm-v3.14.2-linux-s390x.tar.gz diff --git a/hack/tool-versions.sh b/hack/tool-versions.sh index ecc1c424febfa..964dec3e6c8f1 100644 --- a/hack/tool-versions.sh +++ b/hack/tool-versions.sh @@ -11,7 +11,7 @@ # Use ./hack/installers/checksums/add-helm-checksums.sh and # add-kustomize-checksums.sh to help download checksums. ############################################################################### -helm3_version=3.13.2 +helm3_version=3.14.2 kubectl_version=1.17.8 kubectx_version=0.6.3 kustomize5_version=5.2.1 diff --git a/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml b/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml index 0fbf979809c97..68dd75de2f47f 100644 --- a/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml +++ b/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml @@ -20,8 +20,6 @@ spec: - args: - /usr/local/bin/argocd-application-controller env: - - name: ARGOCD_CONTROLLER_REPLICAS - value: "1" - name: ARGOCD_RECONCILIATION_TIMEOUT valueFrom: configMapKeyRef: @@ -34,6 +32,12 @@ spec: name: argocd-cm key: timeout.hard.reconciliation optional: true + - name: ARGOCD_RECONCILIATION_JITTER + valueFrom: + configMapKeyRef: + key: timeout.reconciliation.jitter + name: argocd-cm + optional: true - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS valueFrom: configMapKeyRef: diff --git a/manifests/base/application-controller-deployment/argocd-application-controller-statefulset.yaml b/manifests/base/application-controller-deployment/argocd-application-controller-statefulset.yaml new file mode 100644 index 0000000000000..10e4ea2ac7e3e --- /dev/null +++ b/manifests/base/application-controller-deployment/argocd-application-controller-statefulset.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: argocd-application-controller +spec: + replicas: 0 + template: + spec: + containers: + - name: argocd-application-controller + args: + - /usr/local/bin/argocd-application-controller + env: + - name: ARGOCD_CONTROLLER_REPLICAS + value: "0" \ No newline at end of file diff --git a/manifests/base/application-controller-deployment/kustomization.yaml b/manifests/base/application-controller-deployment/kustomization.yaml index 8f35ec8bd388f..733a378e013e0 100644 --- a/manifests/base/application-controller-deployment/kustomization.yaml +++ b/manifests/base/application-controller-deployment/kustomization.yaml @@ -2,5 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: +- ../application-controller-roles - argocd-application-controller-service.yaml +- argocd-application-controller-statefulset.yaml - argocd-application-controller-deployment.yaml + diff --git a/manifests/base/application-controller/argocd-application-controller-role.yaml b/manifests/base/application-controller-roles/argocd-application-controller-role.yaml similarity index 100% rename from manifests/base/application-controller/argocd-application-controller-role.yaml rename to manifests/base/application-controller-roles/argocd-application-controller-role.yaml diff --git a/manifests/base/application-controller/argocd-application-controller-rolebinding.yaml b/manifests/base/application-controller-roles/argocd-application-controller-rolebinding.yaml similarity index 100% rename from manifests/base/application-controller/argocd-application-controller-rolebinding.yaml rename to manifests/base/application-controller-roles/argocd-application-controller-rolebinding.yaml diff --git a/manifests/base/application-controller/argocd-application-controller-sa.yaml b/manifests/base/application-controller-roles/argocd-application-controller-sa.yaml similarity index 100% rename from manifests/base/application-controller/argocd-application-controller-sa.yaml rename to manifests/base/application-controller-roles/argocd-application-controller-sa.yaml diff --git a/manifests/base/application-controller-roles/kustomization.yaml b/manifests/base/application-controller-roles/kustomization.yaml new file mode 100644 index 0000000000000..f834d2ef3dbc4 --- /dev/null +++ b/manifests/base/application-controller-roles/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- argocd-application-controller-sa.yaml +- argocd-application-controller-role.yaml +- argocd-application-controller-rolebinding.yaml diff --git a/manifests/base/application-controller/argocd-application-controller-statefulset.yaml b/manifests/base/application-controller/argocd-application-controller-statefulset.yaml index 62f98a1449215..d974edffdd618 100644 --- a/manifests/base/application-controller/argocd-application-controller-statefulset.yaml +++ b/manifests/base/application-controller/argocd-application-controller-statefulset.yaml @@ -35,6 +35,12 @@ spec: name: argocd-cm key: timeout.hard.reconciliation optional: true + - name: ARGOCD_RECONCILIATION_JITTER + valueFrom: + configMapKeyRef: + key: timeout.reconciliation.jitter + name: argocd-cm + optional: true - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS valueFrom: configMapKeyRef: diff --git a/manifests/base/application-controller/kustomization.yaml b/manifests/base/application-controller/kustomization.yaml index 9a801ad877bd2..616977fb9b08b 100644 --- a/manifests/base/application-controller/kustomization.yaml +++ b/manifests/base/application-controller/kustomization.yaml @@ -2,9 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- argocd-application-controller-sa.yaml -- argocd-application-controller-role.yaml -- argocd-application-controller-rolebinding.yaml +- ../application-controller-roles - argocd-application-controller-statefulset.yaml - argocd-metrics.yaml - argocd-application-controller-network-policy.yaml \ No newline at end of file diff --git a/manifests/base/dex/argocd-dex-server-deployment.yaml b/manifests/base/dex/argocd-dex-server-deployment.yaml index 8d3b37d177913..7ff5985f44a90 100644 --- a/manifests/base/dex/argocd-dex-server-deployment.yaml +++ b/manifests/base/dex/argocd-dex-server-deployment.yaml @@ -37,7 +37,7 @@ spec: type: RuntimeDefault containers: - name: dex - image: ghcr.io/dexidp/dex:v2.37.0 + image: ghcr.io/dexidp/dex:v2.38.0 imagePullPolicy: Always command: [/shared/argocd-dex, rundex] env: diff --git a/manifests/base/server/argocd-server-deployment.yaml b/manifests/base/server/argocd-server-deployment.yaml index 6df5f9701713f..0ebeb70e08531 100644 --- a/manifests/base/server/argocd-server-deployment.yaml +++ b/manifests/base/server/argocd-server-deployment.yaml @@ -25,136 +25,136 @@ spec: env: - name: ARGOCD_SERVER_INSECURE valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.insecure - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.insecure + optional: true - name: ARGOCD_SERVER_BASEHREF valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.basehref - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.basehref + optional: true - name: ARGOCD_SERVER_ROOTPATH valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.rootpath - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.rootpath + optional: true - name: ARGOCD_SERVER_LOGFORMAT valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.log.format - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.log.format + optional: true - name: ARGOCD_SERVER_LOG_LEVEL valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.log.level - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.log.level + optional: true - name: ARGOCD_SERVER_REPO_SERVER valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: repo.server - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: repo.server + optional: true - name: ARGOCD_SERVER_DEX_SERVER valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.dex.server - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.dex.server + optional: true - name: ARGOCD_SERVER_DISABLE_AUTH valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.disable.auth - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.disable.auth + optional: true - name: ARGOCD_SERVER_ENABLE_GZIP valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.enable.gzip - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.enable.gzip + optional: true - name: ARGOCD_SERVER_REPO_SERVER_TIMEOUT_SECONDS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.repo.server.timeout.seconds - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.repo.server.timeout.seconds + optional: true - name: ARGOCD_SERVER_X_FRAME_OPTIONS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.x.frame.options - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.x.frame.options + optional: true - name: ARGOCD_SERVER_CONTENT_SECURITY_POLICY valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.content.security.policy - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.content.security.policy + optional: true - name: ARGOCD_SERVER_REPO_SERVER_PLAINTEXT valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.repo.server.plaintext - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.repo.server.plaintext + optional: true - name: ARGOCD_SERVER_REPO_SERVER_STRICT_TLS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.repo.server.strict.tls - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.repo.server.strict.tls + optional: true - name: ARGOCD_SERVER_DEX_SERVER_PLAINTEXT valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.dex.server.plaintext - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.dex.server.plaintext + optional: true - name: ARGOCD_SERVER_DEX_SERVER_STRICT_TLS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.dex.server.strict.tls - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.dex.server.strict.tls + optional: true - name: ARGOCD_TLS_MIN_VERSION valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.tls.minversion - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.tls.minversion + optional: true - name: ARGOCD_TLS_MAX_VERSION valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.tls.maxversion - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.tls.maxversion + optional: true - name: ARGOCD_TLS_CIPHERS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.tls.ciphers - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.tls.ciphers + optional: true - name: ARGOCD_SERVER_CONNECTION_STATUS_CACHE_EXPIRATION valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.connection.status.cache.expiration - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.connection.status.cache.expiration + optional: true - name: ARGOCD_SERVER_OIDC_CACHE_EXPIRATION valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.oidc.cache.expiration - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.oidc.cache.expiration + optional: true - name: ARGOCD_SERVER_LOGIN_ATTEMPTS_EXPIRATION valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.login.attempts.expiration - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.login.attempts.expiration + optional: true - name: ARGOCD_SERVER_STATIC_ASSETS valueFrom: configMapKeyRef: @@ -163,16 +163,16 @@ spec: optional: true - name: ARGOCD_APP_STATE_CACHE_EXPIRATION valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.app.state.cache.expiration - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.app.state.cache.expiration + optional: true - name: REDIS_SERVER valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: redis.server - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: redis.server + optional: true - name: REDIS_COMPRESSION valueFrom: configMapKeyRef: @@ -181,76 +181,82 @@ spec: optional: true - name: REDISDB valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: redis.db - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: redis.db + optional: true - name: ARGOCD_DEFAULT_CACHE_EXPIRATION valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.default.cache.expiration - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.default.cache.expiration + optional: true - name: ARGOCD_MAX_COOKIE_NUMBER valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.http.cookie.maxnumber - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.http.cookie.maxnumber + optional: true - name: ARGOCD_SERVER_LISTEN_ADDRESS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.listen.address - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.listen.address + optional: true - name: ARGOCD_SERVER_METRICS_LISTEN_ADDRESS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.metrics.listen.address - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.metrics.listen.address + optional: true - name: ARGOCD_SERVER_OTLP_ADDRESS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: otlp.address - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: otlp.address + optional: true - name: ARGOCD_SERVER_OTLP_INSECURE valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: otlp.insecure - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: otlp.insecure + optional: true - name: ARGOCD_SERVER_OTLP_HEADERS valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: otlp.headers - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: otlp.headers + optional: true - name: ARGOCD_APPLICATION_NAMESPACES valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: application.namespaces - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: application.namespaces + optional: true - name: ARGOCD_SERVER_ENABLE_PROXY_EXTENSION valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.enable.proxy.extension - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.enable.proxy.extension + optional: true - name: ARGOCD_K8SCLIENT_RETRY_MAX valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.k8sclient.retry.max - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.k8sclient.retry.max + optional: true - name: ARGOCD_K8SCLIENT_RETRY_BASE_BACKOFF valueFrom: - configMapKeyRef: - name: argocd-cmd-params-cm - key: server.k8sclient.retry.base.backoff - optional: true + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.k8sclient.retry.base.backoff + optional: true + - name: ARGOCD_API_CONTENT_TYPES + valueFrom: + configMapKeyRef: + name: argocd-cmd-params-cm + key: server.api.content.types + optional: true volumeMounts: - name: ssh-known-hosts mountPath: /app/config/ssh diff --git a/manifests/cluster-rbac/kustomization.yaml b/manifests/cluster-rbac/kustomization.yaml index 7f791905b661b..55e6e2d72df9e 100644 --- a/manifests/cluster-rbac/kustomization.yaml +++ b/manifests/cluster-rbac/kustomization.yaml @@ -3,4 +3,5 @@ kind: Kustomization resources: - ./application-controller +- ./applicationset-controller - ./server diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml index 08d7d972e6362..3cbaa4946e3cc 100644 --- a/manifests/core-install.yaml +++ b/manifests/core-install.yaml @@ -343,6 +343,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to + apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -678,6 +682,10 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1127,6 +1135,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to apply + common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1452,6 +1464,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to apply + common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1943,6 +1959,10 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -2281,6 +2301,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -2764,6 +2789,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors + or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3120,6 +3150,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies + whether to apply common labels to resource + selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3590,6 +3625,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3939,6 +3979,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -4431,6 +4476,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -4780,6 +4830,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -5209,6 +5264,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5423,6 +5480,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5796,6 +5855,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6010,6 +6071,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6387,6 +6450,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6601,6 +6666,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6958,6 +7025,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7172,6 +7241,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7553,6 +7624,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7767,6 +7840,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8140,6 +8215,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8354,6 +8431,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8731,6 +8810,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8945,6 +9026,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9302,6 +9385,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9516,6 +9601,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9883,6 +9970,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10097,6 +10186,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10644,6 +10735,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10858,6 +10951,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11400,6 +11495,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11614,6 +11711,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11985,6 +12084,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12199,6 +12300,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12580,6 +12683,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12794,6 +12899,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13167,6 +13274,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13381,6 +13490,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13758,6 +13869,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13972,6 +14085,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14329,6 +14444,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14543,6 +14660,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14910,6 +15029,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15124,6 +15245,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15671,6 +15794,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15885,6 +16010,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -16427,6 +16554,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -16641,6 +16770,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17016,6 +17147,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17230,6 +17363,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17594,6 +17729,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17808,6 +17945,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -18355,6 +18494,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -18569,6 +18710,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19111,6 +19254,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19325,6 +19470,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19771,6 +19918,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19985,6 +20134,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -21514,6 +21665,12 @@ spec: key: timeout.hard.reconciliation name: argocd-cm optional: true + - name: ARGOCD_RECONCILIATION_JITTER + valueFrom: + configMapKeyRef: + key: timeout.reconciliation.jitter + name: argocd-cm + optional: true - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS valueFrom: configMapKeyRef: diff --git a/manifests/crds/application-crd.yaml b/manifests/crds/application-crd.yaml index f325dda7da6f7..aaf1347f64dfb 100644 --- a/manifests/crds/application-crd.yaml +++ b/manifests/crds/application-crd.yaml @@ -342,6 +342,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to + apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -677,6 +681,10 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1126,6 +1134,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to apply + common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1451,6 +1463,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to apply + common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1942,6 +1958,10 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -2280,6 +2300,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -2763,6 +2788,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors + or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3119,6 +3149,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies + whether to apply common labels to resource + selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3589,6 +3624,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3938,6 +3978,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -4430,6 +4475,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -4779,6 +4829,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps diff --git a/manifests/crds/applicationset-crd.yaml b/manifests/crds/applicationset-crd.yaml index 758785832ea78..8d4fbb5c748fa 100644 --- a/manifests/crds/applicationset-crd.yaml +++ b/manifests/crds/applicationset-crd.yaml @@ -256,6 +256,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -470,6 +472,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -843,6 +847,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -1057,6 +1063,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -1434,6 +1442,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -1648,6 +1658,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -2005,6 +2017,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -2219,6 +2233,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -2600,6 +2616,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -2814,6 +2832,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -3187,6 +3207,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -3401,6 +3423,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -3778,6 +3802,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -3992,6 +4018,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -4349,6 +4377,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -4563,6 +4593,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -4930,6 +4962,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5144,6 +5178,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5691,6 +5727,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5905,6 +5943,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6447,6 +6487,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6661,6 +6703,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7032,6 +7076,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7246,6 +7292,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7627,6 +7675,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7841,6 +7891,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8214,6 +8266,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8428,6 +8482,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8805,6 +8861,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9019,6 +9077,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9376,6 +9436,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9590,6 +9652,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9957,6 +10021,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10171,6 +10237,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10718,6 +10786,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10932,6 +11002,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11474,6 +11546,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11688,6 +11762,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12063,6 +12139,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12277,6 +12355,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12641,6 +12721,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12855,6 +12937,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13402,6 +13486,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13616,6 +13702,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14158,6 +14246,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14372,6 +14462,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14818,6 +14910,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15032,6 +15126,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: diff --git a/manifests/ha/base/controller-deployment/kustomization.yaml b/manifests/ha/base/controller-deployment/kustomization.yaml index d6d20d99b4516..e98bd250d699e 100644 --- a/manifests/ha/base/controller-deployment/kustomization.yaml +++ b/manifests/ha/base/controller-deployment/kustomization.yaml @@ -1,20 +1,17 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - patches: +- path: argocd-application-controller-statefulset.yaml - path: argocd-repo-server-deployment.yaml - path: argocd-server-deployment.yaml -- path: argocd-application-controller-statefulset.yaml - path: argocd-cmd-params-cm.yaml - images: - name: quay.io/argoproj/argocd newName: quay.io/argoproj/argocd newTag: latest resources: -- ../../../base/application-controller - ../../../base/application-controller-deployment - ../../../base/applicationset-controller - ../../../base/dex diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml index a7086ae8a6c06..322d9534a8371 100644 --- a/manifests/ha/install.yaml +++ b/manifests/ha/install.yaml @@ -343,6 +343,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to + apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -678,6 +682,10 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1127,6 +1135,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to apply + common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1452,6 +1464,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to apply + common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1943,6 +1959,10 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -2281,6 +2301,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -2764,6 +2789,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors + or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3120,6 +3150,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies + whether to apply common labels to resource + selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3590,6 +3625,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3939,6 +3979,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -4431,6 +4476,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -4780,6 +4830,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -5209,6 +5264,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5423,6 +5480,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5796,6 +5855,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6010,6 +6071,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6387,6 +6450,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6601,6 +6666,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6958,6 +7025,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7172,6 +7241,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7553,6 +7624,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7767,6 +7840,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8140,6 +8215,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8354,6 +8431,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8731,6 +8810,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8945,6 +9026,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9302,6 +9385,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9516,6 +9601,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9883,6 +9970,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10097,6 +10186,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10644,6 +10735,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10858,6 +10951,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11400,6 +11495,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11614,6 +11711,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11985,6 +12084,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12199,6 +12300,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12580,6 +12683,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12794,6 +12899,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13167,6 +13274,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13381,6 +13490,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13758,6 +13869,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13972,6 +14085,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14329,6 +14444,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14543,6 +14660,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14910,6 +15029,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15124,6 +15245,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15671,6 +15794,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15885,6 +16010,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -16427,6 +16554,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -16641,6 +16770,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17016,6 +17147,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17230,6 +17363,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17594,6 +17729,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17808,6 +17945,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -18355,6 +18494,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -18569,6 +18710,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19111,6 +19254,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19325,6 +19470,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19771,6 +19918,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19985,6 +20134,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -20868,6 +21019,95 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: applicationset-controller + app.kubernetes.io/name: argocd-applicationset-controller + app.kubernetes.io/part-of: argocd + name: argocd-applicationset-controller +rules: +- apiGroups: + - argoproj.io + resources: + - applications + - applicationsets + - applicationsets/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - argoproj.io + resources: + - applicationsets/status + verbs: + - get + - patch + - update +- apiGroups: + - argoproj.io + resources: + - appprojects + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - update + - delete + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + - extensions + resources: + - deployments + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: labels: app.kubernetes.io/component: server @@ -21049,6 +21289,23 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: applicationset-controller + app.kubernetes.io/name: argocd-applicationset-controller + app.kubernetes.io/part-of: argocd + name: argocd-applicationset-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: argocd-applicationset-controller +subjects: +- kind: ServiceAccount + name: argocd-applicationset-controller + namespace: argocd +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/component: server @@ -22390,7 +22647,7 @@ spec: key: dexserver.disable.tls name: argocd-cmd-params-cm optional: true - image: ghcr.io/dexidp/dex:v2.37.0 + image: ghcr.io/dexidp/dex:v2.38.0 imagePullPolicy: Always name: dex ports: @@ -23221,6 +23478,12 @@ spec: key: server.k8sclient.retry.base.backoff name: argocd-cmd-params-cm optional: true + - name: ARGOCD_API_CONTENT_TYPES + valueFrom: + configMapKeyRef: + key: server.api.content.types + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:latest imagePullPolicy: Always livenessProbe: @@ -23347,6 +23610,12 @@ spec: key: timeout.hard.reconciliation name: argocd-cm optional: true + - name: ARGOCD_RECONCILIATION_JITTER + valueFrom: + configMapKeyRef: + key: timeout.reconciliation.jitter + name: argocd-cm + optional: true - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS valueFrom: configMapKeyRef: diff --git a/manifests/ha/namespace-install.yaml b/manifests/ha/namespace-install.yaml index 01a8da2ffd7b9..044a061bf0cb1 100644 --- a/manifests/ha/namespace-install.yaml +++ b/manifests/ha/namespace-install.yaml @@ -1762,7 +1762,7 @@ spec: key: dexserver.disable.tls name: argocd-cmd-params-cm optional: true - image: ghcr.io/dexidp/dex:v2.37.0 + image: ghcr.io/dexidp/dex:v2.38.0 imagePullPolicy: Always name: dex ports: @@ -2593,6 +2593,12 @@ spec: key: server.k8sclient.retry.base.backoff name: argocd-cmd-params-cm optional: true + - name: ARGOCD_API_CONTENT_TYPES + valueFrom: + configMapKeyRef: + key: server.api.content.types + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:latest imagePullPolicy: Always livenessProbe: @@ -2719,6 +2725,12 @@ spec: key: timeout.hard.reconciliation name: argocd-cm optional: true + - name: ARGOCD_RECONCILIATION_JITTER + valueFrom: + configMapKeyRef: + key: timeout.reconciliation.jitter + name: argocd-cm + optional: true - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS valueFrom: configMapKeyRef: diff --git a/manifests/install.yaml b/manifests/install.yaml index 8d30e076d8bf7..8da7f4c8306b4 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -343,6 +343,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to + apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -678,6 +682,10 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1127,6 +1135,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to apply + common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1452,6 +1464,10 @@ spec: definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether to apply + common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -1943,6 +1959,10 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -2281,6 +2301,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -2764,6 +2789,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors + or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3120,6 +3150,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies + whether to apply common labels to resource + selectors or not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3590,6 +3625,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -3939,6 +3979,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -4431,6 +4476,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -4780,6 +4830,11 @@ spec: image definition in the format [old_image_name=]: type: string type: array + labelWithoutSelector: + description: LabelWithoutSelector specifies whether + to apply common labels to resource selectors or + not + type: boolean namePrefix: description: NamePrefix is a prefix appended to resources for Kustomize apps @@ -5209,6 +5264,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5423,6 +5480,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -5796,6 +5855,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6010,6 +6071,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6387,6 +6450,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6601,6 +6666,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -6958,6 +7025,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7172,6 +7241,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7553,6 +7624,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -7767,6 +7840,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8140,6 +8215,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8354,6 +8431,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8731,6 +8810,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -8945,6 +9026,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9302,6 +9385,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9516,6 +9601,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -9883,6 +9970,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10097,6 +10186,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10644,6 +10735,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -10858,6 +10951,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11400,6 +11495,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11614,6 +11711,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -11985,6 +12084,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12199,6 +12300,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12580,6 +12683,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -12794,6 +12899,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13167,6 +13274,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13381,6 +13490,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13758,6 +13869,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -13972,6 +14085,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14329,6 +14444,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14543,6 +14660,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -14910,6 +15029,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15124,6 +15245,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15671,6 +15794,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -15885,6 +16010,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -16427,6 +16554,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -16641,6 +16770,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17016,6 +17147,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17230,6 +17363,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17594,6 +17729,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -17808,6 +17945,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -18355,6 +18494,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -18569,6 +18710,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19111,6 +19254,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19325,6 +19470,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19771,6 +19918,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -19985,6 +20134,8 @@ spec: items: type: string type: array + labelWithoutSelector: + type: boolean namePrefix: type: string nameSuffix: @@ -20827,6 +20978,95 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: applicationset-controller + app.kubernetes.io/name: argocd-applicationset-controller + app.kubernetes.io/part-of: argocd + name: argocd-applicationset-controller +rules: +- apiGroups: + - argoproj.io + resources: + - applications + - applicationsets + - applicationsets/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - argoproj.io + resources: + - applicationsets/status + verbs: + - get + - patch + - update +- apiGroups: + - argoproj.io + resources: + - appprojects + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - update + - delete + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + - extensions + resources: + - deployments + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: labels: app.kubernetes.io/component: server @@ -20976,6 +21216,23 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: applicationset-controller + app.kubernetes.io/name: argocd-applicationset-controller + app.kubernetes.io/part-of: argocd + name: argocd-applicationset-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: argocd-applicationset-controller +subjects: +- kind: ServiceAccount + name: argocd-applicationset-controller + namespace: argocd +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/component: server @@ -21485,7 +21742,7 @@ spec: key: dexserver.disable.tls name: argocd-cmd-params-cm optional: true - image: ghcr.io/dexidp/dex:v2.37.0 + image: ghcr.io/dexidp/dex:v2.38.0 imagePullPolicy: Always name: dex ports: @@ -22265,6 +22522,12 @@ spec: key: server.k8sclient.retry.base.backoff name: argocd-cmd-params-cm optional: true + - name: ARGOCD_API_CONTENT_TYPES + valueFrom: + configMapKeyRef: + key: server.api.content.types + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:latest imagePullPolicy: Always livenessProbe: @@ -22391,6 +22654,12 @@ spec: key: timeout.hard.reconciliation name: argocd-cm optional: true + - name: ARGOCD_RECONCILIATION_JITTER + valueFrom: + configMapKeyRef: + key: timeout.reconciliation.jitter + name: argocd-cm + optional: true - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS valueFrom: configMapKeyRef: diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml index 76301680f195a..cb58228423c11 100644 --- a/manifests/namespace-install.yaml +++ b/manifests/namespace-install.yaml @@ -857,7 +857,7 @@ spec: key: dexserver.disable.tls name: argocd-cmd-params-cm optional: true - image: ghcr.io/dexidp/dex:v2.37.0 + image: ghcr.io/dexidp/dex:v2.38.0 imagePullPolicy: Always name: dex ports: @@ -1637,6 +1637,12 @@ spec: key: server.k8sclient.retry.base.backoff name: argocd-cmd-params-cm optional: true + - name: ARGOCD_API_CONTENT_TYPES + valueFrom: + configMapKeyRef: + key: server.api.content.types + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:latest imagePullPolicy: Always livenessProbe: @@ -1763,6 +1769,12 @@ spec: key: timeout.hard.reconciliation name: argocd-cm optional: true + - name: ARGOCD_RECONCILIATION_JITTER + valueFrom: + configMapKeyRef: + key: timeout.reconciliation.jitter + name: argocd-cm + optional: true - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS valueFrom: configMapKeyRef: diff --git a/notifications_catalog/install.yaml b/notifications_catalog/install.yaml index 59b3665b9a2e3..7457b25ddad89 100644 --- a/notifications_catalog/install.yaml +++ b/notifications_catalog/install.yaml @@ -40,8 +40,7 @@ data: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -68,8 +67,7 @@ data: "value": "{{.app.status.sync.revision}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -119,8 +117,7 @@ data: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -143,8 +140,7 @@ data: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -194,8 +190,7 @@ data: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -222,8 +217,7 @@ data: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -273,8 +267,7 @@ data: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -301,8 +294,7 @@ data: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -356,8 +348,7 @@ data: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -380,8 +371,7 @@ data: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" @@ -430,8 +420,7 @@ data: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -458,8 +447,7 @@ data: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" diff --git a/notifications_catalog/templates/app-deployed.yaml b/notifications_catalog/templates/app-deployed.yaml index 843bf57e21a89..ee58c775f1fd8 100644 --- a/notifications_catalog/templates/app-deployed.yaml +++ b/notifications_catalog/templates/app-deployed.yaml @@ -25,8 +25,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -52,8 +51,7 @@ teams: "value": "{{.app.status.sync.revision}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" diff --git a/notifications_catalog/templates/app-health-degraded.yaml b/notifications_catalog/templates/app-health-degraded.yaml index 46c39b2e9ca0c..59115c9a14935 100644 --- a/notifications_catalog/templates/app-health-degraded.yaml +++ b/notifications_catalog/templates/app-health-degraded.yaml @@ -21,8 +21,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -44,8 +43,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" diff --git a/notifications_catalog/templates/app-sync-failed.yaml b/notifications_catalog/templates/app-sync-failed.yaml index 4a5ece85ba541..a4c23787dde8b 100644 --- a/notifications_catalog/templates/app-sync-failed.yaml +++ b/notifications_catalog/templates/app-sync-failed.yaml @@ -21,8 +21,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -48,8 +47,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" diff --git a/notifications_catalog/templates/app-sync-running.yaml b/notifications_catalog/templates/app-sync-running.yaml index b2a86042e3ce2..434132ad86d89 100644 --- a/notifications_catalog/templates/app-sync-running.yaml +++ b/notifications_catalog/templates/app-sync-running.yaml @@ -21,8 +21,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -47,8 +46,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" diff --git a/notifications_catalog/templates/app-sync-status-unknown.yaml b/notifications_catalog/templates/app-sync-status-unknown.yaml index b1af244fb6d2d..c893070bfcc63 100644 --- a/notifications_catalog/templates/app-sync-status-unknown.yaml +++ b/notifications_catalog/templates/app-sync-status-unknown.yaml @@ -26,8 +26,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -48,8 +47,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" diff --git a/notifications_catalog/templates/app-sync-succeeded.yaml b/notifications_catalog/templates/app-sync-succeeded.yaml index d791de55149a4..76e467bd1c37d 100644 --- a/notifications_catalog/templates/app-sync-succeeded.yaml +++ b/notifications_catalog/templates/app-sync-succeeded.yaml @@ -21,8 +21,7 @@ slack: "short": true } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "title": "{{$c.type}}", "value": "{{$c.message}}", @@ -48,8 +47,7 @@ teams: "value": "{{.app.spec.source.repoURL}}" } {{range $index, $c := .app.status.conditions}} - {{if not $index}},{{end}} - {{if $index}},{{end}} + , { "name": "{{$c.type}}", "value": "{{$c.message}}" diff --git a/pkg/apis/application/v1alpha1/generated.pb.go b/pkg/apis/application/v1alpha1/generated.pb.go index cccbc3f7f15a4..f6a253d23ed7d 100644 --- a/pkg/apis/application/v1alpha1/generated.pb.go +++ b/pkg/apis/application/v1alpha1/generated.pb.go @@ -4448,13 +4448,13 @@ func init() { } var fileDescriptor_030104ce3b95bcac = []byte{ - // 10990 bytes of a gzipped FileDescriptorProto + // 11030 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x6d, 0x70, 0x1c, 0xc9, 0x75, 0x98, 0x66, 0x17, 0x0b, 0xec, 0x3e, 0x7c, 0x90, 0x6c, 0x92, 0x77, 0x20, 0x75, 0x77, 0xa0, 0xe7, 0xe2, 0xd3, 0x39, 0xba, 0x03, 0x7c, 0xf4, 0x9d, 0x7c, 0xf1, 0xd9, 0x92, 0xb1, 0x00, 0x09, 0x82, 0x04, 0x08, 0x5c, 0x03, 0x24, 0xa5, 0x93, 0x4f, 0xa7, 0xc1, 0x6e, 0x63, 0x31, 0xc4, 0xec, 0xcc, 0xdc, 0xcc, 0x2c, 0x08, 0x9c, 0x25, 0x59, 0xb2, 0x64, 0x5b, 0x89, 0x3e, 0x4e, 0x91, 0x92, - 0xf2, 0x39, 0x89, 0x14, 0xd9, 0x72, 0x52, 0x76, 0x25, 0xaa, 0x38, 0xc9, 0x8f, 0x38, 0x71, 0x52, + 0xf2, 0x39, 0xb1, 0x14, 0xd9, 0x72, 0x52, 0x71, 0x25, 0xaa, 0x38, 0xc9, 0x8f, 0x38, 0x71, 0x52, 0x2e, 0xdb, 0xa9, 0x94, 0x52, 0x4a, 0xca, 0x2e, 0x97, 0xcb, 0x72, 0x12, 0x1b, 0x91, 0x98, 0x4a, 0x25, 0x95, 0xaa, 0xb8, 0xca, 0x89, 0x7f, 0x24, 0x4c, 0x7e, 0xa4, 0xfa, 0xbb, 0x67, 0x76, 0x16, 0x58, 0x00, 0x03, 0x92, 0x52, 0xee, 0xdf, 0x6e, 0xbf, 0x37, 0xef, 0xf5, 0xf4, 0x74, 0xbf, 0xf7, @@ -4471,671 +4471,674 @@ var fileDescriptor_030104ce3b95bcac = []byte{ 0xdb, 0x69, 0x6c, 0xb8, 0x3e, 0x89, 0x76, 0xf4, 0xe3, 0x6d, 0x92, 0x38, 0x79, 0x4f, 0x4d, 0xf5, 0x7a, 0x2a, 0xea, 0xf8, 0x89, 0xdb, 0x26, 0x5d, 0x0f, 0xbc, 0x67, 0xbf, 0x07, 0xe2, 0xc6, 0x06, 0x69, 0x3b, 0x5d, 0xcf, 0xfd, 0x50, 0xaf, 0xe7, 0x3a, 0x89, 0xeb, 0x4d, 0xb9, 0x7e, 0x12, 0x27, - 0x51, 0xf6, 0x21, 0xfb, 0x75, 0x18, 0x9d, 0xbe, 0xb5, 0x32, 0xdd, 0x49, 0x36, 0x66, 0x02, 0x7f, - 0xdd, 0x6d, 0xa1, 0x17, 0x60, 0xb8, 0xe1, 0x75, 0xe2, 0x84, 0x44, 0xd7, 0x9d, 0x36, 0x19, 0xb7, - 0x2e, 0x58, 0x4f, 0xd7, 0xea, 0xa7, 0xbf, 0xb1, 0x3b, 0xf1, 0x8e, 0xbb, 0xbb, 0x13, 0xc3, 0x33, - 0x1a, 0x84, 0x4d, 0x3c, 0xf4, 0x03, 0x30, 0x14, 0x05, 0x1e, 0x99, 0xc6, 0xd7, 0xc7, 0x4b, 0xec, - 0x91, 0x13, 0xe2, 0x91, 0x21, 0xcc, 0x9b, 0xb1, 0x84, 0xdb, 0x7f, 0x58, 0x02, 0x98, 0x0e, 0xc3, - 0xe5, 0x28, 0xb8, 0x4d, 0x1a, 0x09, 0xfa, 0x30, 0x54, 0xe9, 0xd0, 0x35, 0x9d, 0xc4, 0x61, 0xdc, - 0x86, 0x2f, 0xfe, 0xe0, 0x24, 0x7f, 0x93, 0x49, 0xf3, 0x4d, 0xf4, 0xc4, 0xa1, 0xd8, 0x93, 0x5b, - 0xcf, 0x4d, 0x2e, 0xad, 0xd1, 0xe7, 0x17, 0x49, 0xe2, 0xd4, 0x91, 0x60, 0x06, 0xba, 0x0d, 0x2b, - 0xaa, 0xc8, 0x87, 0x81, 0x38, 0x24, 0x0d, 0xd6, 0xb1, 0xe1, 0x8b, 0x0b, 0x93, 0x47, 0x99, 0xa1, - 0x93, 0xba, 0xe7, 0x2b, 0x21, 0x69, 0xd4, 0x47, 0x04, 0xe7, 0x01, 0xfa, 0x0f, 0x33, 0x3e, 0x68, - 0x0b, 0x06, 0xe3, 0xc4, 0x49, 0x3a, 0xf1, 0x78, 0x99, 0x71, 0xbc, 0x5e, 0x18, 0x47, 0x46, 0xb5, - 0x3e, 0x26, 0x78, 0x0e, 0xf2, 0xff, 0x58, 0x70, 0xb3, 0xff, 0xc4, 0x82, 0x31, 0x8d, 0xbc, 0xe0, - 0xc6, 0x09, 0xfa, 0x89, 0xae, 0xc1, 0x9d, 0xec, 0x6f, 0x70, 0xe9, 0xd3, 0x6c, 0x68, 0x4f, 0x0a, - 0x66, 0x55, 0xd9, 0x62, 0x0c, 0x6c, 0x1b, 0x2a, 0x6e, 0x42, 0xda, 0xf1, 0x78, 0xe9, 0x42, 0xf9, - 0xe9, 0xe1, 0x8b, 0x57, 0x8a, 0x7a, 0xcf, 0xfa, 0xa8, 0x60, 0x5a, 0x99, 0xa7, 0xe4, 0x31, 0xe7, - 0x62, 0xff, 0xea, 0x88, 0xf9, 0x7e, 0x74, 0xc0, 0xd1, 0x73, 0x30, 0x1c, 0x07, 0x9d, 0xa8, 0x41, - 0x30, 0x09, 0x83, 0x78, 0xdc, 0xba, 0x50, 0xa6, 0x53, 0x8f, 0xce, 0xd4, 0x15, 0xdd, 0x8c, 0x4d, - 0x1c, 0xf4, 0x79, 0x0b, 0x46, 0x9a, 0x24, 0x4e, 0x5c, 0x9f, 0xf1, 0x97, 0x9d, 0x5f, 0x3d, 0x72, - 0xe7, 0x65, 0xe3, 0xac, 0x26, 0x5e, 0x3f, 0x23, 0x5e, 0x64, 0xc4, 0x68, 0x8c, 0x71, 0x8a, 0x3f, - 0x5d, 0x71, 0x4d, 0x12, 0x37, 0x22, 0x37, 0xa4, 0xff, 0xd9, 0x9c, 0x31, 0x56, 0xdc, 0xac, 0x06, - 0x61, 0x13, 0x0f, 0xf9, 0x50, 0xa1, 0x2b, 0x2a, 0x1e, 0x1f, 0x60, 0xfd, 0x9f, 0x3f, 0x5a, 0xff, - 0xc5, 0xa0, 0xd2, 0xc5, 0xaa, 0x47, 0x9f, 0xfe, 0x8b, 0x31, 0x67, 0x83, 0x3e, 0x67, 0xc1, 0xb8, - 0x58, 0xf1, 0x98, 0xf0, 0x01, 0xbd, 0xb5, 0xe1, 0x26, 0xc4, 0x73, 0xe3, 0x64, 0xbc, 0xc2, 0xfa, - 0x30, 0xd5, 0xdf, 0xdc, 0x9a, 0x8b, 0x82, 0x4e, 0x78, 0xcd, 0xf5, 0x9b, 0xf5, 0x0b, 0x82, 0xd3, - 0xf8, 0x4c, 0x0f, 0xc2, 0xb8, 0x27, 0x4b, 0xf4, 0x25, 0x0b, 0xce, 0xfb, 0x4e, 0x9b, 0xc4, 0xa1, - 0x43, 0x3f, 0x2d, 0x07, 0xd7, 0x3d, 0xa7, 0xb1, 0xc9, 0x7a, 0x34, 0x78, 0xb8, 0x1e, 0xd9, 0xa2, - 0x47, 0xe7, 0xaf, 0xf7, 0x24, 0x8d, 0xf7, 0x60, 0x8b, 0xbe, 0x66, 0xc1, 0xa9, 0x20, 0x0a, 0x37, - 0x1c, 0x9f, 0x34, 0x25, 0x34, 0x1e, 0x1f, 0x62, 0x4b, 0xef, 0x43, 0x47, 0xfb, 0x44, 0x4b, 0x59, - 0xb2, 0x8b, 0x81, 0xef, 0x26, 0x41, 0xb4, 0x42, 0x92, 0xc4, 0xf5, 0x5b, 0x71, 0xfd, 0xec, 0xdd, - 0xdd, 0x89, 0x53, 0x5d, 0x58, 0xb8, 0xbb, 0x3f, 0xe8, 0x27, 0x61, 0x38, 0xde, 0xf1, 0x1b, 0xb7, - 0x5c, 0xbf, 0x19, 0xdc, 0x89, 0xc7, 0xab, 0x45, 0x2c, 0xdf, 0x15, 0x45, 0x50, 0x2c, 0x40, 0xcd, - 0x00, 0x9b, 0xdc, 0xf2, 0x3f, 0x9c, 0x9e, 0x4a, 0xb5, 0xa2, 0x3f, 0x9c, 0x9e, 0x4c, 0x7b, 0xb0, - 0x45, 0x3f, 0x67, 0xc1, 0x68, 0xec, 0xb6, 0x7c, 0x27, 0xe9, 0x44, 0xe4, 0x1a, 0xd9, 0x89, 0xc7, - 0x81, 0x75, 0xe4, 0xea, 0x11, 0x47, 0xc5, 0x20, 0x59, 0x3f, 0x2b, 0xfa, 0x38, 0x6a, 0xb6, 0xc6, - 0x38, 0xcd, 0x37, 0x6f, 0xa1, 0xe9, 0x69, 0x3d, 0x5c, 0xec, 0x42, 0xd3, 0x93, 0xba, 0x27, 0x4b, - 0xf4, 0xe3, 0x70, 0x92, 0x37, 0xa9, 0x91, 0x8d, 0xc7, 0x47, 0x98, 0xa0, 0x3d, 0x73, 0x77, 0x77, - 0xe2, 0xe4, 0x4a, 0x06, 0x86, 0xbb, 0xb0, 0xd1, 0xeb, 0x30, 0x11, 0x92, 0xa8, 0xed, 0x26, 0x4b, - 0xbe, 0xb7, 0x23, 0xc5, 0x77, 0x23, 0x08, 0x49, 0x53, 0x74, 0x27, 0x1e, 0x1f, 0xbd, 0x60, 0x3d, - 0x5d, 0xad, 0xbf, 0x4b, 0x74, 0x73, 0x62, 0x79, 0x6f, 0x74, 0xbc, 0x1f, 0x3d, 0xfb, 0x5f, 0x97, - 0xe0, 0x64, 0x56, 0x71, 0xa2, 0xbf, 0x6b, 0xc1, 0x89, 0xdb, 0x77, 0x92, 0xd5, 0x60, 0x93, 0xf8, - 0x71, 0x7d, 0x87, 0x8a, 0x37, 0xa6, 0x32, 0x86, 0x2f, 0x36, 0x8a, 0x55, 0xd1, 0x93, 0x57, 0xd3, - 0x5c, 0x2e, 0xf9, 0x49, 0xb4, 0x53, 0x7f, 0x54, 0xbc, 0xdd, 0x89, 0xab, 0xb7, 0x56, 0x4d, 0x28, - 0xce, 0x76, 0xea, 0xfc, 0x67, 0x2c, 0x38, 0x93, 0x47, 0x02, 0x9d, 0x84, 0xf2, 0x26, 0xd9, 0xe1, - 0x56, 0x19, 0xa6, 0x3f, 0xd1, 0xab, 0x50, 0xd9, 0x72, 0xbc, 0x0e, 0x11, 0xd6, 0xcd, 0xdc, 0xd1, - 0x5e, 0x44, 0xf5, 0x0c, 0x73, 0xaa, 0x3f, 0x52, 0x7a, 0xd1, 0xb2, 0x7f, 0xb7, 0x0c, 0xc3, 0x86, - 0x7e, 0xbb, 0x0f, 0x16, 0x5b, 0x90, 0xb2, 0xd8, 0x16, 0x0b, 0x53, 0xcd, 0x3d, 0x4d, 0xb6, 0x3b, - 0x19, 0x93, 0x6d, 0xa9, 0x38, 0x96, 0x7b, 0xda, 0x6c, 0x28, 0x81, 0x5a, 0x10, 0x52, 0x8b, 0x9c, - 0xaa, 0xfe, 0x81, 0x22, 0x3e, 0xe1, 0x92, 0x24, 0x57, 0x1f, 0xbd, 0xbb, 0x3b, 0x51, 0x53, 0x7f, - 0xb1, 0x66, 0x64, 0x7f, 0xcb, 0x82, 0x33, 0x46, 0x1f, 0x67, 0x02, 0xbf, 0xe9, 0xb2, 0x4f, 0x7b, - 0x01, 0x06, 0x92, 0x9d, 0x50, 0x9a, 0xfd, 0x6a, 0xa4, 0x56, 0x77, 0x42, 0x82, 0x19, 0x84, 0x1a, - 0xfa, 0x6d, 0x12, 0xc7, 0x4e, 0x8b, 0x64, 0x0d, 0xfd, 0x45, 0xde, 0x8c, 0x25, 0x1c, 0x45, 0x80, - 0x3c, 0x27, 0x4e, 0x56, 0x23, 0xc7, 0x8f, 0x19, 0xf9, 0x55, 0xb7, 0x4d, 0xc4, 0x00, 0xff, 0xc5, - 0xfe, 0x66, 0x0c, 0x7d, 0xa2, 0xfe, 0xc8, 0xdd, 0xdd, 0x09, 0xb4, 0xd0, 0x45, 0x09, 0xe7, 0x50, - 0xb7, 0xbf, 0x64, 0xc1, 0x23, 0xf9, 0xb6, 0x18, 0x7a, 0x0a, 0x06, 0xf9, 0x96, 0x4f, 0xbc, 0x9d, - 0xfe, 0x24, 0xac, 0x15, 0x0b, 0x28, 0x9a, 0x82, 0x9a, 0xd2, 0x13, 0xe2, 0x1d, 0x4f, 0x09, 0xd4, - 0x9a, 0x56, 0x2e, 0x1a, 0x87, 0x0e, 0x1a, 0xfd, 0x23, 0x2c, 0x37, 0x35, 0x68, 0x6c, 0x93, 0xc4, - 0x20, 0xf6, 0x7f, 0xb4, 0xe0, 0x84, 0xd1, 0xab, 0xfb, 0x60, 0x9a, 0xfb, 0x69, 0xd3, 0x7c, 0xbe, - 0xb0, 0xf9, 0xdc, 0xc3, 0x36, 0xff, 0x9c, 0x05, 0xe7, 0x0d, 0xac, 0x45, 0x27, 0x69, 0x6c, 0x5c, - 0xda, 0x0e, 0x23, 0x12, 0xd3, 0xed, 0x34, 0x7a, 0xdc, 0x90, 0x5b, 0xf5, 0x61, 0x41, 0xa1, 0x7c, - 0x8d, 0xec, 0x70, 0x21, 0xf6, 0x0c, 0x54, 0xf9, 0xe4, 0x0c, 0x22, 0x31, 0xe2, 0xea, 0xdd, 0x96, - 0x44, 0x3b, 0x56, 0x18, 0xc8, 0x86, 0x41, 0x26, 0x9c, 0xe8, 0x62, 0xa5, 0x6a, 0x08, 0xe8, 0x47, - 0xbc, 0xc9, 0x5a, 0xb0, 0x80, 0xd8, 0x71, 0xaa, 0x3b, 0xcb, 0x11, 0x61, 0x1f, 0xb7, 0x79, 0xd9, - 0x25, 0x5e, 0x33, 0xa6, 0xdb, 0x06, 0xc7, 0xf7, 0x83, 0x44, 0xec, 0x00, 0x8c, 0x6d, 0xc3, 0xb4, - 0x6e, 0xc6, 0x26, 0x0e, 0x65, 0xea, 0x39, 0x6b, 0xc4, 0xe3, 0x23, 0x2a, 0x98, 0x2e, 0xb0, 0x16, - 0x2c, 0x20, 0xf6, 0xdd, 0x12, 0xdb, 0xa0, 0xa8, 0xa5, 0x4f, 0xee, 0xc7, 0xee, 0x36, 0x4a, 0xc9, - 0xca, 0xe5, 0xe2, 0x04, 0x17, 0xe9, 0xbd, 0xc3, 0x7d, 0x23, 0x23, 0x2e, 0x71, 0xa1, 0x5c, 0xf7, - 0xde, 0xe5, 0xfe, 0x56, 0x09, 0x26, 0xd2, 0x0f, 0x74, 0x49, 0x5b, 0xba, 0xa5, 0x32, 0x18, 0x65, - 0x9d, 0x18, 0x06, 0x3e, 0x36, 0xf1, 0x7a, 0x08, 0xac, 0xd2, 0x71, 0x0a, 0x2c, 0x53, 0x9e, 0x96, - 0xf7, 0x91, 0xa7, 0x4f, 0xa9, 0x51, 0x1f, 0xc8, 0x08, 0xb0, 0xb4, 0x4e, 0xb9, 0x00, 0x03, 0x71, - 0x42, 0xc2, 0xf1, 0x4a, 0x5a, 0x1e, 0xad, 0x24, 0x24, 0xc4, 0x0c, 0x62, 0xff, 0xb7, 0x12, 0x3c, - 0x9a, 0x1e, 0x43, 0xad, 0x02, 0xde, 0x97, 0x52, 0x01, 0xef, 0x36, 0x55, 0xc0, 0xbd, 0xdd, 0x89, - 0x77, 0xf6, 0x78, 0xec, 0xbb, 0x46, 0x43, 0xa0, 0xb9, 0xcc, 0x28, 0x4e, 0xa5, 0x47, 0xf1, 0xde, - 0xee, 0xc4, 0xe3, 0x3d, 0xde, 0x31, 0x33, 0xcc, 0x4f, 0xc1, 0x60, 0x44, 0x9c, 0x38, 0xf0, 0xc5, - 0x40, 0xab, 0xcf, 0x81, 0x59, 0x2b, 0x16, 0x50, 0xfb, 0xf7, 0x6b, 0xd9, 0xc1, 0x9e, 0xe3, 0x4e, - 0xb8, 0x20, 0x42, 0x2e, 0x0c, 0x30, 0xb3, 0x9e, 0x8b, 0x86, 0x6b, 0x47, 0x5b, 0x46, 0x54, 0x0d, - 0x28, 0xd2, 0xf5, 0x2a, 0xfd, 0x6a, 0xb4, 0x09, 0x33, 0x16, 0x68, 0x1b, 0xaa, 0x0d, 0x69, 0x6d, - 0x97, 0x8a, 0xf0, 0x4b, 0x09, 0x5b, 0x5b, 0x73, 0x1c, 0xa1, 0xf2, 0x5a, 0x99, 0xe8, 0x8a, 0x1b, - 0x22, 0x50, 0x6e, 0xb9, 0x89, 0xf8, 0xac, 0x47, 0xdc, 0x4f, 0xcd, 0xb9, 0xc6, 0x2b, 0x0e, 0x51, - 0x25, 0x32, 0xe7, 0x26, 0x98, 0xd2, 0x47, 0x3f, 0x63, 0xc1, 0x70, 0xdc, 0x68, 0x2f, 0x47, 0xc1, - 0x96, 0xdb, 0x24, 0x91, 0xb0, 0xa6, 0x8e, 0x28, 0x9a, 0x56, 0x66, 0x16, 0x25, 0x41, 0xcd, 0x97, - 0xef, 0x6f, 0x35, 0x04, 0x9b, 0x7c, 0xe9, 0x2e, 0xe3, 0x51, 0xf1, 0xee, 0xb3, 0xa4, 0xe1, 0x52, - 0xfd, 0x27, 0x37, 0x55, 0x6c, 0xa6, 0x1c, 0xd9, 0xba, 0x9c, 0xed, 0x34, 0x36, 0xe9, 0x7a, 0xd3, - 0x1d, 0x7a, 0xe7, 0xdd, 0xdd, 0x89, 0x47, 0x67, 0xf2, 0x79, 0xe2, 0x5e, 0x9d, 0x61, 0x03, 0x16, - 0x76, 0x3c, 0x0f, 0x93, 0xd7, 0x3b, 0x84, 0xb9, 0x4c, 0x0a, 0x18, 0xb0, 0x65, 0x4d, 0x30, 0x33, - 0x60, 0x06, 0x04, 0x9b, 0x7c, 0xd1, 0xeb, 0x30, 0xd8, 0x76, 0x92, 0xc8, 0xdd, 0x16, 0x7e, 0x92, - 0x23, 0xda, 0xfb, 0x8b, 0x8c, 0x96, 0x66, 0xce, 0x34, 0x35, 0x6f, 0xc4, 0x82, 0x11, 0x6a, 0x43, - 0xa5, 0x4d, 0xa2, 0x16, 0x19, 0xaf, 0x16, 0xe1, 0x13, 0x5e, 0xa4, 0xa4, 0x34, 0xc3, 0x1a, 0xb5, - 0x8e, 0x58, 0x1b, 0xe6, 0x5c, 0xd0, 0xab, 0x50, 0x8d, 0x89, 0x47, 0x1a, 0xd4, 0xbe, 0xa9, 0x31, - 0x8e, 0x3f, 0xd4, 0xa7, 0xad, 0x47, 0x0d, 0x8b, 0x15, 0xf1, 0x28, 0x5f, 0x60, 0xf2, 0x1f, 0x56, - 0x24, 0xe9, 0x00, 0x86, 0x5e, 0xa7, 0xe5, 0xfa, 0xe3, 0x50, 0xc4, 0x00, 0x2e, 0x33, 0x5a, 0x99, - 0x01, 0xe4, 0x8d, 0x58, 0x30, 0xb2, 0xff, 0xb3, 0x05, 0x28, 0x2d, 0xd4, 0xee, 0x83, 0x51, 0xfb, - 0x7a, 0xda, 0xa8, 0x5d, 0x28, 0xd2, 0xea, 0xe8, 0x61, 0xd7, 0xfe, 0x46, 0x0d, 0x32, 0xea, 0xe0, - 0x3a, 0x89, 0x13, 0xd2, 0x7c, 0x5b, 0x84, 0xbf, 0x2d, 0xc2, 0xdf, 0x16, 0xe1, 0x4a, 0x84, 0xaf, - 0x65, 0x44, 0xf8, 0x7b, 0x8d, 0x55, 0xaf, 0x0f, 0x55, 0x5f, 0x53, 0xa7, 0xae, 0x66, 0x0f, 0x0c, - 0x04, 0x2a, 0x09, 0xae, 0xae, 0x2c, 0x5d, 0xcf, 0x95, 0xd9, 0xaf, 0xa5, 0x65, 0xf6, 0x51, 0x59, - 0xfc, 0xff, 0x20, 0xa5, 0xff, 0x95, 0x05, 0xef, 0x4a, 0x4b, 0x2f, 0x39, 0x73, 0xe6, 0x5b, 0x7e, - 0x10, 0x91, 0x59, 0x77, 0x7d, 0x9d, 0x44, 0xc4, 0x6f, 0x90, 0x58, 0x79, 0x31, 0xac, 0x5e, 0x5e, - 0x0c, 0xf4, 0x3c, 0x8c, 0xdc, 0x8e, 0x03, 0x7f, 0x39, 0x70, 0x7d, 0x21, 0x82, 0xe8, 0x46, 0xf8, - 0xe4, 0xdd, 0xdd, 0x89, 0x11, 0x3a, 0xa2, 0xb2, 0x1d, 0xa7, 0xb0, 0xd0, 0x0c, 0x9c, 0xba, 0xfd, - 0xfa, 0xb2, 0x93, 0x18, 0xee, 0x00, 0xb9, 0x71, 0x67, 0x07, 0x16, 0x57, 0x5f, 0xce, 0x00, 0x71, - 0x37, 0xbe, 0xfd, 0x37, 0x4b, 0x70, 0x2e, 0xf3, 0x22, 0x81, 0xe7, 0x05, 0x9d, 0x84, 0x6e, 0x6a, - 0xd0, 0x57, 0x2c, 0x38, 0xd9, 0x4e, 0x7b, 0x1c, 0x62, 0xe1, 0xd8, 0x7d, 0x7f, 0x61, 0x3a, 0x22, - 0xe3, 0xd2, 0xa8, 0x8f, 0x8b, 0x11, 0x3a, 0x99, 0x01, 0xc4, 0xb8, 0xab, 0x2f, 0xe8, 0x55, 0xa8, - 0xb5, 0x9d, 0xed, 0x1b, 0x61, 0xd3, 0x49, 0xe4, 0x7e, 0xb2, 0xb7, 0x1b, 0xa0, 0x93, 0xb8, 0xde, - 0x24, 0x3f, 0xae, 0x9f, 0x9c, 0xf7, 0x93, 0xa5, 0x68, 0x25, 0x89, 0x5c, 0xbf, 0xc5, 0xdd, 0x79, - 0x8b, 0x92, 0x0c, 0xd6, 0x14, 0xed, 0x2f, 0x5b, 0x59, 0x25, 0xa5, 0x46, 0x27, 0x72, 0x12, 0xd2, - 0xda, 0x41, 0x1f, 0x81, 0x0a, 0xdd, 0xf8, 0xc9, 0x51, 0xb9, 0x55, 0xa4, 0xe6, 0x34, 0xbe, 0x84, - 0x56, 0xa2, 0xf4, 0x5f, 0x8c, 0x39, 0x53, 0xfb, 0x2b, 0xb5, 0xac, 0xb1, 0xc0, 0x0e, 0x6f, 0x2f, - 0x02, 0xb4, 0x82, 0x55, 0xd2, 0x0e, 0x3d, 0x3a, 0x2c, 0x16, 0x3b, 0x01, 0x50, 0xbe, 0x8e, 0x39, - 0x05, 0xc1, 0x06, 0x16, 0xfa, 0xcb, 0x16, 0x40, 0x4b, 0xce, 0x79, 0x69, 0x08, 0xdc, 0x28, 0xf2, - 0x75, 0xf4, 0x8a, 0xd2, 0x7d, 0x51, 0x0c, 0xb1, 0xc1, 0x1c, 0xfd, 0xb4, 0x05, 0xd5, 0x44, 0x76, - 0x9f, 0xab, 0xc6, 0xd5, 0x22, 0x7b, 0x22, 0x5f, 0x5a, 0xdb, 0x44, 0x6a, 0x48, 0x14, 0x5f, 0xf4, - 0xb3, 0x16, 0x40, 0xbc, 0xe3, 0x37, 0x96, 0x03, 0xcf, 0x6d, 0xec, 0x08, 0x8d, 0x79, 0xb3, 0x50, - 0x7f, 0x8c, 0xa2, 0x5e, 0x1f, 0xa3, 0xa3, 0xa1, 0xff, 0x63, 0x83, 0x33, 0xfa, 0x18, 0x54, 0x63, - 0x31, 0xdd, 0x84, 0x8e, 0x5c, 0x2d, 0xd6, 0x2b, 0xc4, 0x69, 0x0b, 0xf1, 0x2a, 0xfe, 0x61, 0xc5, - 0x13, 0xfd, 0xbc, 0x05, 0x27, 0xc2, 0xb4, 0x9f, 0x4f, 0xa8, 0xc3, 0xe2, 0x64, 0x40, 0xc6, 0x8f, - 0x58, 0x3f, 0x7d, 0x77, 0x77, 0xe2, 0x44, 0xa6, 0x11, 0x67, 0x7b, 0x41, 0x25, 0xa0, 0x9e, 0xc1, - 0x4b, 0x21, 0xf7, 0x39, 0x0e, 0x69, 0x09, 0x38, 0x97, 0x05, 0xe2, 0x6e, 0x7c, 0xb4, 0x0c, 0x67, - 0x68, 0xef, 0x76, 0xb8, 0xf9, 0x29, 0xd5, 0x4b, 0xcc, 0x94, 0x61, 0xb5, 0xfe, 0x98, 0x98, 0x21, - 0xcc, 0xab, 0x9f, 0xc5, 0xc1, 0xb9, 0x4f, 0xa2, 0xdf, 0xb5, 0xe0, 0x31, 0x97, 0xa9, 0x01, 0xd3, - 0x61, 0xae, 0x35, 0x82, 0x38, 0x89, 0x25, 0x85, 0xca, 0x8a, 0x5e, 0xea, 0xa7, 0xfe, 0x17, 0xc4, - 0x1b, 0x3c, 0x36, 0xbf, 0x47, 0x97, 0xf0, 0x9e, 0x1d, 0x46, 0x3f, 0x0c, 0xa3, 0x72, 0x5d, 0x2c, - 0x53, 0x11, 0xcc, 0x14, 0x6d, 0xad, 0x7e, 0xea, 0xee, 0xee, 0xc4, 0xe8, 0xaa, 0x09, 0xc0, 0x69, - 0x3c, 0xfb, 0x9b, 0xa5, 0xd4, 0x79, 0x88, 0x72, 0x42, 0x32, 0x71, 0xd3, 0x90, 0xfe, 0x1f, 0x29, - 0x3d, 0x0b, 0x15, 0x37, 0xca, 0xbb, 0xa4, 0xc5, 0x8d, 0x6a, 0x8a, 0xb1, 0xc1, 0x9c, 0x1a, 0xa5, - 0xa7, 0x9c, 0xac, 0xab, 0x53, 0x48, 0xc0, 0x57, 0x8b, 0xec, 0x52, 0xf7, 0xe9, 0xd5, 0x39, 0xd1, - 0xb5, 0x53, 0x5d, 0x20, 0xdc, 0xdd, 0x25, 0xfb, 0x9b, 0xe9, 0x33, 0x18, 0x63, 0xf1, 0xf6, 0x71, - 0xbe, 0xf4, 0x79, 0x0b, 0x86, 0xa3, 0xc0, 0xf3, 0x5c, 0xbf, 0x45, 0x05, 0x8d, 0xd0, 0x96, 0x1f, - 0x3c, 0x16, 0x85, 0x25, 0x24, 0x0a, 0x33, 0x6d, 0xb1, 0xe6, 0x89, 0xcd, 0x0e, 0xd8, 0x7f, 0x62, - 0xc1, 0x78, 0x2f, 0x81, 0x88, 0x08, 0xbc, 0x53, 0xae, 0x76, 0x15, 0x5d, 0xb1, 0xe4, 0xcf, 0x12, - 0x8f, 0x28, 0xc7, 0x73, 0xb5, 0xfe, 0xa4, 0x78, 0xcd, 0x77, 0x2e, 0xf7, 0x46, 0xc5, 0x7b, 0xd1, - 0x41, 0xaf, 0xc0, 0x49, 0xe3, 0xbd, 0x62, 0x35, 0x30, 0xb5, 0xfa, 0x24, 0xb5, 0x40, 0xa6, 0x33, - 0xb0, 0x7b, 0xbb, 0x13, 0x8f, 0x64, 0xdb, 0x84, 0xc4, 0xee, 0xa2, 0x63, 0xff, 0x72, 0x29, 0xfb, - 0xb5, 0x94, 0xb2, 0x7d, 0xcb, 0xea, 0xda, 0xce, 0xbf, 0xff, 0x38, 0x14, 0x1c, 0xdb, 0xf8, 0xab, - 0x00, 0x8e, 0xde, 0x38, 0x0f, 0xf0, 0x84, 0xd8, 0xfe, 0x37, 0x03, 0xb0, 0x47, 0xcf, 0xfa, 0xb0, - 0x9e, 0x0f, 0x7c, 0xac, 0xf8, 0x59, 0x4b, 0x1d, 0x39, 0x95, 0xd9, 0x22, 0x6f, 0x1e, 0xd7, 0xd8, - 0xf3, 0x0d, 0x4c, 0xcc, 0xa3, 0x14, 0x94, 0x1b, 0x3b, 0x7d, 0xb8, 0x85, 0xbe, 0x6a, 0xa5, 0x0f, - 0xcd, 0x78, 0xd8, 0x99, 0x7b, 0x6c, 0x7d, 0x32, 0x4e, 0xe2, 0x78, 0xc7, 0xf4, 0xf9, 0x4d, 0xaf, - 0x33, 0xba, 0x49, 0x80, 0x75, 0xd7, 0x77, 0x3c, 0xf7, 0x0d, 0xba, 0x3d, 0xa9, 0x30, 0x0d, 0xcb, - 0x4c, 0x96, 0xcb, 0xaa, 0x15, 0x1b, 0x18, 0xe7, 0xff, 0x12, 0x0c, 0x1b, 0x6f, 0x9e, 0x13, 0x5c, - 0x71, 0xc6, 0x0c, 0xae, 0xa8, 0x19, 0x31, 0x11, 0xe7, 0xdf, 0x0b, 0x27, 0xb3, 0x1d, 0x3c, 0xc8, - 0xf3, 0xf6, 0xff, 0x1a, 0xca, 0x9e, 0x62, 0xad, 0x92, 0xa8, 0x4d, 0xbb, 0xf6, 0xb6, 0x67, 0xe9, - 0x6d, 0xcf, 0xd2, 0xdb, 0x9e, 0x25, 0xf3, 0x70, 0x40, 0x78, 0x4d, 0x86, 0xee, 0x93, 0xd7, 0x24, - 0xe5, 0x07, 0xaa, 0x16, 0xee, 0x07, 0xb2, 0xef, 0x56, 0x20, 0x65, 0x47, 0xf1, 0xf1, 0xfe, 0x01, - 0x18, 0x8a, 0x48, 0x18, 0xdc, 0xc0, 0x0b, 0x42, 0x87, 0xe8, 0x00, 0x7a, 0xde, 0x8c, 0x25, 0x9c, - 0xea, 0x9a, 0xd0, 0x49, 0x36, 0x84, 0x12, 0x51, 0xba, 0x66, 0xd9, 0x49, 0x36, 0x30, 0x83, 0xa0, - 0xf7, 0xc2, 0x58, 0xe2, 0x44, 0x2d, 0x6a, 0x6f, 0x6f, 0xb1, 0xcf, 0x2a, 0xce, 0x3a, 0x1f, 0x11, - 0xb8, 0x63, 0xab, 0x29, 0x28, 0xce, 0x60, 0xa3, 0xd7, 0x61, 0x60, 0x83, 0x78, 0x6d, 0x31, 0xe4, - 0x2b, 0xc5, 0xc9, 0x78, 0xf6, 0xae, 0x57, 0x88, 0xd7, 0xe6, 0x12, 0x88, 0xfe, 0xc2, 0x8c, 0x15, - 0x9d, 0x6f, 0xb5, 0xcd, 0x4e, 0x9c, 0x04, 0x6d, 0xf7, 0x0d, 0xe9, 0xe2, 0x7b, 0x7f, 0xc1, 0x8c, - 0xaf, 0x49, 0xfa, 0xdc, 0x97, 0xa2, 0xfe, 0x62, 0xcd, 0x99, 0xf5, 0xa3, 0xe9, 0x46, 0xec, 0x53, - 0xed, 0x08, 0x4f, 0x5d, 0xd1, 0xfd, 0x98, 0x95, 0xf4, 0x79, 0x3f, 0xd4, 0x5f, 0xac, 0x39, 0xa3, - 0x1d, 0x35, 0xef, 0x87, 0x59, 0x1f, 0x6e, 0x14, 0xdc, 0x07, 0x3e, 0xe7, 0x73, 0xe7, 0xff, 0x93, - 0x50, 0x69, 0x6c, 0x38, 0x51, 0x32, 0x3e, 0xc2, 0x26, 0x8d, 0xf2, 0xe9, 0xcc, 0xd0, 0x46, 0xcc, - 0x61, 0xe8, 0x71, 0x28, 0x47, 0x64, 0x9d, 0xc5, 0x6d, 0x1a, 0x11, 0x3d, 0x98, 0xac, 0x63, 0xda, - 0x6e, 0xff, 0x62, 0x29, 0x6d, 0x2e, 0xa5, 0xdf, 0x9b, 0xcf, 0xf6, 0x46, 0x27, 0x8a, 0xa5, 0xdf, - 0xc7, 0x98, 0xed, 0xac, 0x19, 0x4b, 0x38, 0xfa, 0x84, 0x05, 0x43, 0xb7, 0xe3, 0xc0, 0xf7, 0x49, - 0x22, 0x54, 0xd3, 0xcd, 0x82, 0x87, 0xe2, 0x2a, 0xa7, 0xae, 0xfb, 0x20, 0x1a, 0xb0, 0xe4, 0x4b, - 0xbb, 0x4b, 0xb6, 0x1b, 0x5e, 0xa7, 0xd9, 0x15, 0xa4, 0x71, 0x89, 0x37, 0x63, 0x09, 0xa7, 0xa8, - 0xae, 0xcf, 0x51, 0x07, 0xd2, 0xa8, 0xf3, 0xbe, 0x40, 0x15, 0x70, 0xfb, 0xaf, 0x0f, 0xc2, 0xd9, - 0xdc, 0xc5, 0x41, 0x0d, 0x19, 0x66, 0x2a, 0x5c, 0x76, 0x3d, 0x22, 0xc3, 0x93, 0x98, 0x21, 0x73, - 0x53, 0xb5, 0x62, 0x03, 0x03, 0xfd, 0x14, 0x40, 0xe8, 0x44, 0x4e, 0x9b, 0x28, 0xbf, 0xec, 0x91, - 0xed, 0x05, 0xda, 0x8f, 0x65, 0x49, 0x53, 0xef, 0x4d, 0x55, 0x53, 0x8c, 0x0d, 0x96, 0xe8, 0x05, - 0x18, 0x8e, 0x88, 0x47, 0x9c, 0x98, 0x85, 0xfd, 0x66, 0x73, 0x18, 0xb0, 0x06, 0x61, 0x13, 0x0f, - 0x3d, 0xa5, 0x22, 0xb9, 0x32, 0x11, 0x2d, 0xe9, 0x68, 0x2e, 0xf4, 0xa6, 0x05, 0x63, 0xeb, 0xae, - 0x47, 0x34, 0x77, 0x91, 0x71, 0xb0, 0x74, 0xf4, 0x97, 0xbc, 0x6c, 0xd2, 0xd5, 0x12, 0x32, 0xd5, - 0x1c, 0xe3, 0x0c, 0x7b, 0xfa, 0x99, 0xb7, 0x48, 0xc4, 0x44, 0xeb, 0x60, 0xfa, 0x33, 0xdf, 0xe4, - 0xcd, 0x58, 0xc2, 0xd1, 0x34, 0x9c, 0x08, 0x9d, 0x38, 0x9e, 0x89, 0x48, 0x93, 0xf8, 0x89, 0xeb, - 0x78, 0x3c, 0x1f, 0xa0, 0xaa, 0xe3, 0x81, 0x97, 0xd3, 0x60, 0x9c, 0xc5, 0x47, 0x1f, 0x80, 0x47, - 0xb9, 0xe3, 0x63, 0xd1, 0x8d, 0x63, 0xd7, 0x6f, 0xe9, 0x69, 0x20, 0xfc, 0x3f, 0x13, 0x82, 0xd4, - 0xa3, 0xf3, 0xf9, 0x68, 0xb8, 0xd7, 0xf3, 0xe8, 0x19, 0xa8, 0xc6, 0x9b, 0x6e, 0x38, 0x13, 0x35, - 0x63, 0x76, 0xe8, 0x51, 0xd5, 0xde, 0xc6, 0x15, 0xd1, 0x8e, 0x15, 0x06, 0x6a, 0xc0, 0x08, 0xff, - 0x24, 0x3c, 0x14, 0x4d, 0xc8, 0xc7, 0x67, 0x7b, 0xaa, 0x47, 0x91, 0xb2, 0x36, 0x89, 0x9d, 0x3b, - 0x97, 0xe4, 0x11, 0x0c, 0x3f, 0x31, 0xb8, 0x69, 0x90, 0xc1, 0x29, 0xa2, 0xf6, 0x2f, 0x94, 0xd2, - 0x3b, 0x6e, 0x73, 0x91, 0xa2, 0x98, 0x2e, 0xc5, 0xe4, 0xa6, 0x13, 0x49, 0x6f, 0xcc, 0x11, 0xd3, - 0x16, 0x04, 0xdd, 0x9b, 0x4e, 0x64, 0x2e, 0x6a, 0xc6, 0x00, 0x4b, 0x4e, 0xe8, 0x36, 0x0c, 0x24, - 0x9e, 0x53, 0x50, 0x9e, 0x93, 0xc1, 0x51, 0x3b, 0x40, 0x16, 0xa6, 0x63, 0xcc, 0x78, 0xa0, 0xc7, - 0xa8, 0xd5, 0xbf, 0x26, 0x8f, 0x48, 0x84, 0xa1, 0xbe, 0x16, 0x63, 0xd6, 0x6a, 0xff, 0x0a, 0xe4, - 0xc8, 0x55, 0xa5, 0xc8, 0xd0, 0x45, 0x00, 0xba, 0x81, 0x5c, 0x8e, 0xc8, 0xba, 0xbb, 0x2d, 0x0c, - 0x09, 0xb5, 0x76, 0xaf, 0x2b, 0x08, 0x36, 0xb0, 0xe4, 0x33, 0x2b, 0x9d, 0x75, 0xfa, 0x4c, 0xa9, - 0xfb, 0x19, 0x0e, 0xc1, 0x06, 0x16, 0x7a, 0x1e, 0x06, 0xdd, 0xb6, 0xd3, 0x52, 0x21, 0x98, 0x8f, - 0xd1, 0x45, 0x3b, 0xcf, 0x5a, 0xee, 0xed, 0x4e, 0x8c, 0xa9, 0x0e, 0xb1, 0x26, 0x2c, 0x70, 0xd1, - 0x2f, 0x5b, 0x30, 0xd2, 0x08, 0xda, 0xed, 0xc0, 0xe7, 0xdb, 0x2e, 0xb1, 0x87, 0xbc, 0x7d, 0x5c, - 0x6a, 0x7e, 0x72, 0xc6, 0x60, 0xc6, 0x37, 0x91, 0x2a, 0x21, 0xcb, 0x04, 0xe1, 0x54, 0xaf, 0xcc, - 0xb5, 0x5d, 0xd9, 0x67, 0x6d, 0xff, 0xba, 0x05, 0xa7, 0xf8, 0xb3, 0xc6, 0x6e, 0x50, 0xe4, 0x1e, - 0x05, 0xc7, 0xfc, 0x5a, 0x5d, 0x1b, 0x64, 0xe5, 0xa5, 0xeb, 0x82, 0xe3, 0xee, 0x4e, 0xa2, 0x39, - 0x38, 0xb5, 0x1e, 0x44, 0x0d, 0x62, 0x0e, 0x84, 0x10, 0x4c, 0x8a, 0xd0, 0xe5, 0x2c, 0x02, 0xee, - 0x7e, 0x06, 0xdd, 0x84, 0x47, 0x8c, 0x46, 0x73, 0x1c, 0xb8, 0x6c, 0x7a, 0x42, 0x50, 0x7b, 0xe4, - 0x72, 0x2e, 0x16, 0xee, 0xf1, 0x74, 0xda, 0x61, 0x52, 0xeb, 0xc3, 0x61, 0xf2, 0x1a, 0x9c, 0x6b, - 0x74, 0x8f, 0xcc, 0x56, 0xdc, 0x59, 0x8b, 0xb9, 0xa4, 0xaa, 0xd6, 0xbf, 0x4f, 0x10, 0x38, 0x37, - 0xd3, 0x0b, 0x11, 0xf7, 0xa6, 0x81, 0x3e, 0x02, 0xd5, 0x88, 0xb0, 0xaf, 0x12, 0x8b, 0x44, 0x9c, - 0x23, 0xee, 0x92, 0xb5, 0x05, 0xca, 0xc9, 0x6a, 0xd9, 0x2b, 0x1a, 0x62, 0xac, 0x38, 0xa2, 0x3b, - 0x30, 0x14, 0x3a, 0x49, 0x63, 0x43, 0xa4, 0xdf, 0x1c, 0x39, 0xfe, 0x45, 0x31, 0x67, 0x3e, 0x70, - 0x3d, 0xc9, 0x97, 0x39, 0x13, 0x2c, 0xb9, 0x51, 0x6b, 0xa4, 0x11, 0xb4, 0xc3, 0xc0, 0x27, 0x7e, - 0x12, 0x8f, 0x8f, 0x6a, 0x6b, 0x64, 0x46, 0xb5, 0x62, 0x03, 0xe3, 0xfc, 0xfb, 0xe0, 0x54, 0xd7, - 0xc2, 0x3b, 0x90, 0x73, 0x65, 0x16, 0x1e, 0xc9, 0x9f, 0xe2, 0x07, 0x72, 0xb1, 0xfc, 0xe3, 0x4c, - 0x90, 0xab, 0x61, 0xf6, 0xf6, 0xe1, 0xae, 0x73, 0xa0, 0x4c, 0xfc, 0x2d, 0x21, 0xf1, 0x2f, 0x1f, - 0x6d, 0xa4, 0x2f, 0xf9, 0x5b, 0x7c, 0x85, 0x32, 0x9f, 0xc4, 0x25, 0x7f, 0x0b, 0x53, 0xda, 0xe8, - 0x8b, 0x56, 0xca, 0x6c, 0xe3, 0x4e, 0xbe, 0x0f, 0x1d, 0x8b, 0x9d, 0xdf, 0xb7, 0x25, 0x67, 0xff, - 0xdb, 0x12, 0x5c, 0xd8, 0x8f, 0x48, 0x1f, 0xc3, 0xf7, 0x24, 0x0c, 0xc6, 0xec, 0xd8, 0x5a, 0x88, - 0xd0, 0x61, 0x3a, 0xb3, 0xf8, 0x41, 0xf6, 0x6b, 0x58, 0x80, 0x90, 0x07, 0xe5, 0xb6, 0x13, 0x0a, - 0xdf, 0xcf, 0xfc, 0x51, 0xd3, 0x5e, 0xe8, 0x7f, 0xc7, 0x5b, 0x74, 0x42, 0xee, 0x51, 0x30, 0x1a, - 0x30, 0x65, 0x83, 0x12, 0xa8, 0x38, 0x51, 0xe4, 0xc8, 0x33, 0xd2, 0x6b, 0xc5, 0xf0, 0x9b, 0xa6, - 0x24, 0xf9, 0x11, 0x53, 0xaa, 0x09, 0x73, 0x66, 0xf6, 0x67, 0x87, 0x52, 0xa9, 0x1f, 0xec, 0xe0, - 0x3b, 0x86, 0x41, 0xe1, 0xf2, 0xb1, 0x8a, 0xce, 0x36, 0xe2, 0xb9, 0x7b, 0x6c, 0x57, 0x27, 0x32, - 0xa0, 0x05, 0x2b, 0xf4, 0x19, 0x8b, 0xe5, 0x19, 0xcb, 0x74, 0x18, 0xb1, 0x97, 0x3a, 0x9e, 0xb4, - 0x67, 0x33, 0x7b, 0x59, 0x36, 0x62, 0x93, 0x3b, 0xd5, 0xb1, 0x21, 0xcf, 0x98, 0xcb, 0xee, 0xa8, - 0x64, 0x26, 0xb2, 0x84, 0xa3, 0xed, 0x9c, 0x03, 0xee, 0x02, 0x72, 0x55, 0xfb, 0x38, 0xd2, 0xfe, - 0xaa, 0x05, 0xa7, 0xdc, 0xec, 0x49, 0xa5, 0xd8, 0x79, 0x1c, 0x31, 0x84, 0xa2, 0xf7, 0x41, 0xa8, - 0x52, 0xbe, 0x5d, 0x20, 0xdc, 0xdd, 0x19, 0xd4, 0x84, 0x01, 0xd7, 0x5f, 0x0f, 0x84, 0xc9, 0x51, - 0x3f, 0x5a, 0xa7, 0xe6, 0xfd, 0xf5, 0x40, 0xaf, 0x66, 0xfa, 0x0f, 0x33, 0xea, 0x68, 0x01, 0xce, - 0x44, 0xc2, 0x37, 0x74, 0xc5, 0x8d, 0xe9, 0x0e, 0x7e, 0xc1, 0x6d, 0xbb, 0x09, 0x33, 0x17, 0xca, - 0xf5, 0xf1, 0xbb, 0xbb, 0x13, 0x67, 0x70, 0x0e, 0x1c, 0xe7, 0x3e, 0x85, 0xde, 0x80, 0x21, 0x99, - 0x18, 0x5d, 0x2d, 0x62, 0x17, 0xd7, 0x3d, 0xff, 0xd5, 0x64, 0x5a, 0x11, 0x39, 0xd0, 0x92, 0xa1, - 0xfd, 0xe6, 0x30, 0x74, 0x1f, 0x62, 0xa2, 0x8f, 0x42, 0x2d, 0x52, 0xc9, 0xda, 0x56, 0x11, 0xca, - 0x55, 0x7e, 0x5f, 0x71, 0x80, 0xaa, 0x0c, 0x17, 0x9d, 0x96, 0xad, 0x39, 0xd2, 0xed, 0x45, 0xac, - 0xcf, 0x3a, 0x0b, 0x98, 0xdb, 0x82, 0xab, 0x3e, 0xc7, 0xda, 0xf1, 0x1b, 0x98, 0xf1, 0x40, 0x11, - 0x0c, 0x6e, 0x10, 0xc7, 0x4b, 0x36, 0x8a, 0x71, 0xb9, 0x5f, 0x61, 0xb4, 0xb2, 0x29, 0x3b, 0xbc, - 0x15, 0x0b, 0x4e, 0x68, 0x1b, 0x86, 0x36, 0xf8, 0x04, 0x10, 0x16, 0xff, 0xe2, 0x51, 0x07, 0x37, - 0x35, 0xab, 0xf4, 0xe7, 0x16, 0x0d, 0x58, 0xb2, 0x63, 0xd1, 0x31, 0xc6, 0xf9, 0x3d, 0x5f, 0xba, - 0xc5, 0x65, 0x2b, 0xf5, 0x7f, 0x78, 0xff, 0x61, 0x18, 0x89, 0x48, 0x23, 0xf0, 0x1b, 0xae, 0x47, - 0x9a, 0xd3, 0xd2, 0x9d, 0x7e, 0x90, 0x1c, 0x17, 0xb6, 0x6b, 0xc6, 0x06, 0x0d, 0x9c, 0xa2, 0x88, - 0x3e, 0x6d, 0xc1, 0x98, 0xca, 0xf0, 0xa4, 0x1f, 0x84, 0x08, 0xf7, 0xed, 0x42, 0x41, 0xf9, 0xa4, - 0x8c, 0x66, 0x1d, 0xdd, 0xdd, 0x9d, 0x18, 0x4b, 0xb7, 0xe1, 0x0c, 0x5f, 0xf4, 0x0a, 0x40, 0xb0, - 0xc6, 0x43, 0x60, 0xa6, 0x13, 0xe1, 0xcb, 0x3d, 0xc8, 0xab, 0x8e, 0xf1, 0x64, 0x37, 0x49, 0x01, - 0x1b, 0xd4, 0xd0, 0x35, 0x00, 0xbe, 0x6c, 0x56, 0x77, 0x42, 0xb9, 0x2d, 0x90, 0x49, 0x4a, 0xb0, - 0xa2, 0x20, 0xf7, 0x76, 0x27, 0xba, 0x7d, 0x6b, 0x2c, 0xcc, 0xc0, 0x78, 0x1c, 0xfd, 0x24, 0x0c, - 0xc5, 0x9d, 0x76, 0xdb, 0x51, 0x9e, 0xde, 0x02, 0xd3, 0xe7, 0x38, 0x5d, 0x43, 0x14, 0xf1, 0x06, - 0x2c, 0x39, 0xa2, 0xdb, 0x54, 0xa8, 0xc6, 0xc2, 0xe9, 0xc7, 0x56, 0x11, 0xb7, 0x09, 0x86, 0xd9, - 0x3b, 0xbd, 0x47, 0x46, 0xf4, 0xe0, 0x1c, 0x9c, 0x7b, 0xbb, 0x13, 0x8f, 0xa4, 0xdb, 0x17, 0x02, - 0x91, 0xd0, 0x96, 0x4b, 0x13, 0x5d, 0x95, 0x75, 0x52, 0xe8, 0x6b, 0xcb, 0xf4, 0xfd, 0xa7, 0x75, - 0x9d, 0x14, 0xd6, 0xdc, 0x7b, 0xcc, 0xcc, 0x87, 0xd1, 0x22, 0x9c, 0x6e, 0x04, 0x7e, 0x12, 0x05, - 0x9e, 0xc7, 0x8b, 0xff, 0xf0, 0x1d, 0x1a, 0xf7, 0x04, 0xbf, 0x53, 0x74, 0xfb, 0xf4, 0x4c, 0x37, - 0x0a, 0xce, 0x7b, 0xce, 0xf6, 0xd3, 0xb1, 0x81, 0x62, 0x70, 0x9e, 0x87, 0x11, 0xb2, 0x9d, 0x90, - 0xc8, 0x77, 0xbc, 0x1b, 0x78, 0x41, 0xfa, 0x40, 0xd9, 0x1a, 0xb8, 0x64, 0xb4, 0xe3, 0x14, 0x16, - 0xb2, 0x95, 0x5b, 0xc2, 0x48, 0xd2, 0xe4, 0x6e, 0x09, 0xe9, 0x84, 0xb0, 0xff, 0x77, 0x29, 0x65, - 0x90, 0xad, 0x46, 0x84, 0xa0, 0x00, 0x2a, 0x7e, 0xd0, 0x54, 0xb2, 0xff, 0x6a, 0x31, 0xb2, 0xff, - 0x7a, 0xd0, 0x34, 0x8a, 0xa9, 0xd0, 0x7f, 0x31, 0xe6, 0x7c, 0x58, 0xb5, 0x09, 0x59, 0x96, 0x83, - 0x01, 0xc4, 0x46, 0xa3, 0x48, 0xce, 0xaa, 0xda, 0xc4, 0x92, 0xc9, 0x08, 0xa7, 0xf9, 0xa2, 0x4d, - 0xa8, 0x6c, 0x04, 0x71, 0x22, 0xb7, 0x1f, 0x47, 0xdc, 0xe9, 0x5c, 0x09, 0xe2, 0x84, 0x59, 0x11, - 0xea, 0xb5, 0x69, 0x4b, 0x8c, 0x39, 0x0f, 0xfb, 0xbf, 0x58, 0x29, 0x8f, 0xf7, 0x2d, 0x16, 0x27, - 0xbb, 0x45, 0x7c, 0xba, 0xac, 0xcd, 0xc0, 0xa0, 0x1f, 0xce, 0x64, 0x1d, 0xbe, 0xab, 0x57, 0x69, - 0xab, 0x3b, 0x94, 0xc2, 0x24, 0x23, 0x61, 0xc4, 0x10, 0x7d, 0xdc, 0x4a, 0xe7, 0x7f, 0x96, 0x8a, - 0xd8, 0x60, 0x98, 0x39, 0xd0, 0xfb, 0xa6, 0x92, 0xda, 0x5f, 0xb4, 0x60, 0xa8, 0xee, 0x34, 0x36, - 0x83, 0xf5, 0x75, 0xf4, 0x0c, 0x54, 0x9b, 0x9d, 0xc8, 0x4c, 0x45, 0x55, 0xdb, 0xfc, 0x59, 0xd1, - 0x8e, 0x15, 0x06, 0x9d, 0xc3, 0xeb, 0x4e, 0x43, 0x66, 0x42, 0x97, 0xf9, 0x1c, 0xbe, 0xcc, 0x5a, - 0xb0, 0x80, 0xa0, 0x17, 0x60, 0xb8, 0xed, 0x6c, 0xcb, 0x87, 0xb3, 0xee, 0xf6, 0x45, 0x0d, 0xc2, - 0x26, 0x9e, 0xfd, 0x2f, 0x2d, 0x18, 0xaf, 0x3b, 0xb1, 0xdb, 0x98, 0xee, 0x24, 0x1b, 0x75, 0x37, - 0x59, 0xeb, 0x34, 0x36, 0x49, 0xc2, 0xd3, 0xdf, 0x69, 0x2f, 0x3b, 0x31, 0x5d, 0x4a, 0x6a, 0x5f, - 0xa7, 0x7a, 0x79, 0x43, 0xb4, 0x63, 0x85, 0x81, 0xde, 0x80, 0xe1, 0xd0, 0x89, 0xe3, 0x3b, 0x41, - 0xd4, 0xc4, 0x64, 0xbd, 0x98, 0xe2, 0x13, 0x2b, 0xa4, 0x11, 0x91, 0x04, 0x93, 0x75, 0x71, 0x24, - 0xac, 0xe9, 0x63, 0x93, 0x99, 0xfd, 0x79, 0x0b, 0xce, 0xd5, 0x89, 0x13, 0x91, 0x88, 0xd5, 0xaa, - 0x50, 0x2f, 0x32, 0xe3, 0x05, 0x9d, 0x26, 0x7a, 0x1d, 0xaa, 0x09, 0x6d, 0xa6, 0xdd, 0xb2, 0x8a, - 0xed, 0x16, 0x3b, 0xd1, 0x5d, 0x15, 0xc4, 0xb1, 0x62, 0x63, 0xff, 0x0d, 0x0b, 0x46, 0xd8, 0xe1, - 0xd8, 0x2c, 0x49, 0x1c, 0xd7, 0xeb, 0x2a, 0xe9, 0x64, 0xf5, 0x59, 0xd2, 0xe9, 0x02, 0x0c, 0x6c, - 0x04, 0x6d, 0x92, 0x3d, 0xd8, 0xbd, 0x12, 0xd0, 0x6d, 0x35, 0x85, 0xa0, 0xe7, 0xe8, 0x87, 0x77, - 0xfd, 0xc4, 0xa1, 0x4b, 0x40, 0x3a, 0x5f, 0x4f, 0xf0, 0x8f, 0xae, 0x9a, 0xb1, 0x89, 0x63, 0xff, - 0x56, 0x0d, 0x86, 0xc4, 0xe9, 0x7f, 0xdf, 0x25, 0x10, 0xe4, 0xfe, 0xbe, 0xd4, 0x73, 0x7f, 0x1f, - 0xc3, 0x60, 0x83, 0x15, 0x8c, 0x13, 0x66, 0xe4, 0xb5, 0x42, 0xc2, 0x45, 0x78, 0x0d, 0x3a, 0xdd, - 0x2d, 0xfe, 0x1f, 0x0b, 0x56, 0xe8, 0x0b, 0x16, 0x9c, 0x68, 0x04, 0xbe, 0x4f, 0x1a, 0xda, 0xc6, - 0x19, 0x28, 0x22, 0x2a, 0x60, 0x26, 0x4d, 0x54, 0x9f, 0xcc, 0x64, 0x00, 0x38, 0xcb, 0x1e, 0xbd, - 0x04, 0xa3, 0x7c, 0xcc, 0x6e, 0xa6, 0x3c, 0xc6, 0xba, 0xd2, 0x8f, 0x09, 0xc4, 0x69, 0x5c, 0x34, - 0xc9, 0x3d, 0xef, 0xa2, 0xa6, 0xce, 0xa0, 0x76, 0xac, 0x19, 0xd5, 0x74, 0x0c, 0x0c, 0x14, 0x01, - 0x8a, 0xc8, 0x7a, 0x44, 0xe2, 0x0d, 0x11, 0x1d, 0xc1, 0xec, 0xab, 0xa1, 0xc3, 0xa5, 0x4b, 0xe3, - 0x2e, 0x4a, 0x38, 0x87, 0x3a, 0xda, 0x14, 0x1b, 0xcc, 0x6a, 0x11, 0x32, 0x54, 0x7c, 0xe6, 0x9e, - 0xfb, 0xcc, 0x09, 0xa8, 0xc4, 0x1b, 0x4e, 0xd4, 0x64, 0x76, 0x5d, 0x99, 0xa7, 0xe8, 0xac, 0xd0, - 0x06, 0xcc, 0xdb, 0xd1, 0x2c, 0x9c, 0xcc, 0xd4, 0x29, 0x8a, 0x85, 0x67, 0x57, 0xa5, 0x63, 0x64, - 0x2a, 0x1c, 0xc5, 0xb8, 0xeb, 0x09, 0xd3, 0xf9, 0x30, 0xbc, 0x8f, 0xf3, 0x61, 0x47, 0xc5, 0xe0, - 0x71, 0x9f, 0xeb, 0xcb, 0x85, 0x0c, 0x40, 0x5f, 0x01, 0x77, 0x9f, 0xcb, 0x04, 0xdc, 0x8d, 0xb2, - 0x0e, 0xdc, 0x2c, 0xa6, 0x03, 0x07, 0x8f, 0xae, 0x7b, 0x90, 0xd1, 0x72, 0x7f, 0x6e, 0x81, 0xfc, - 0xae, 0x33, 0x4e, 0x63, 0x83, 0xd0, 0x29, 0x83, 0xde, 0x0b, 0x63, 0x6a, 0x0b, 0x3d, 0x13, 0x74, - 0x7c, 0x1e, 0x28, 0x57, 0xd6, 0x47, 0xb8, 0x38, 0x05, 0xc5, 0x19, 0x6c, 0x34, 0x05, 0x35, 0x3a, - 0x4e, 0xfc, 0x51, 0xae, 0x6b, 0xd5, 0x36, 0x7d, 0x7a, 0x79, 0x5e, 0x3c, 0xa5, 0x71, 0x50, 0x00, - 0xa7, 0x3c, 0x27, 0x4e, 0x58, 0x0f, 0xe8, 0x8e, 0xfa, 0x90, 0xc5, 0x0a, 0x58, 0xcc, 0xff, 0x42, - 0x96, 0x10, 0xee, 0xa6, 0x6d, 0x7f, 0x6b, 0x00, 0x46, 0x53, 0x92, 0xf1, 0x80, 0x4a, 0xfa, 0x19, - 0xa8, 0x4a, 0xbd, 0x99, 0x2d, 0xab, 0xa2, 0x94, 0xab, 0xc2, 0xa0, 0x4a, 0x6b, 0x4d, 0x6b, 0xd5, - 0xac, 0x51, 0x61, 0x28, 0x5c, 0x6c, 0xe2, 0x31, 0xa1, 0x9c, 0x78, 0xf1, 0x8c, 0xe7, 0x12, 0x3f, - 0xe1, 0xdd, 0x2c, 0x46, 0x28, 0xaf, 0x2e, 0xac, 0x98, 0x44, 0xb5, 0x50, 0xce, 0x00, 0x70, 0x96, - 0x3d, 0xfa, 0x94, 0x05, 0xa3, 0xce, 0x9d, 0x58, 0x57, 0x35, 0x15, 0xa1, 0x75, 0x47, 0x54, 0x52, - 0xa9, 0x42, 0xa9, 0xdc, 0xe5, 0x9b, 0x6a, 0xc2, 0x69, 0xa6, 0xe8, 0x2d, 0x0b, 0x10, 0xd9, 0x26, - 0x0d, 0x19, 0xfc, 0x27, 0xfa, 0x32, 0x58, 0xc4, 0x4e, 0xf3, 0x52, 0x17, 0x5d, 0x2e, 0xd5, 0xbb, - 0xdb, 0x71, 0x4e, 0x1f, 0xec, 0x7f, 0x56, 0x56, 0x0b, 0x4a, 0xc7, 0x9b, 0x3a, 0x46, 0xdc, 0x9b, - 0x75, 0xf8, 0xb8, 0x37, 0x1d, 0x3f, 0xd0, 0x9d, 0x03, 0x99, 0x4a, 0x99, 0x2a, 0x3d, 0xa0, 0x94, - 0xa9, 0x9f, 0xb6, 0x52, 0x05, 0x84, 0x86, 0x2f, 0xbe, 0x52, 0x6c, 0xac, 0xeb, 0x24, 0x8f, 0x6d, - 0xc8, 0x48, 0xf7, 0x74, 0x48, 0x0b, 0x95, 0xa6, 0x06, 0xda, 0x81, 0xa4, 0xe1, 0xbf, 0x2f, 0xc3, - 0xb0, 0xa1, 0x49, 0x73, 0xcd, 0x22, 0xeb, 0x21, 0x33, 0x8b, 0x4a, 0x07, 0x30, 0x8b, 0x7e, 0x0a, - 0x6a, 0x0d, 0x29, 0xe5, 0x8b, 0x29, 0xa1, 0x9b, 0xd5, 0x1d, 0x5a, 0xd0, 0xab, 0x26, 0xac, 0x79, - 0xa2, 0xb9, 0x54, 0xa2, 0x8d, 0xd0, 0x10, 0x03, 0x4c, 0x43, 0xe4, 0x65, 0xc2, 0x08, 0x4d, 0xd1, - 0xfd, 0x0c, 0xab, 0x33, 0x15, 0xba, 0xe2, 0xbd, 0x64, 0x44, 0x3a, 0xaf, 0x33, 0xb5, 0x3c, 0x2f, - 0x9b, 0xb1, 0x89, 0x63, 0x7f, 0xcb, 0x52, 0x1f, 0xf7, 0x3e, 0x54, 0x54, 0xb8, 0x9d, 0xae, 0xa8, - 0x70, 0xa9, 0x90, 0x61, 0xee, 0x51, 0x4a, 0xe1, 0x3a, 0x0c, 0xcd, 0x04, 0xed, 0xb6, 0xe3, 0x37, - 0xd1, 0xf7, 0xc3, 0x50, 0x83, 0xff, 0x14, 0x8e, 0x1d, 0x76, 0x3c, 0x28, 0xa0, 0x58, 0xc2, 0xd0, - 0x63, 0x30, 0xe0, 0x44, 0x2d, 0xe9, 0xcc, 0x61, 0xa1, 0x30, 0xd3, 0x51, 0x2b, 0xc6, 0xac, 0xd5, - 0xfe, 0x47, 0x03, 0xc0, 0x4e, 0xa0, 0x9d, 0x88, 0x34, 0x57, 0x03, 0x56, 0xc2, 0xef, 0x58, 0x0f, - 0xd5, 0xf4, 0x66, 0xe9, 0x61, 0x3e, 0x58, 0x33, 0x0e, 0x57, 0xca, 0xf7, 0xf9, 0x70, 0xa5, 0xc7, - 0x79, 0xd9, 0xc0, 0x43, 0x74, 0x5e, 0x66, 0x7f, 0xd6, 0x02, 0xa4, 0xc2, 0x16, 0xf4, 0x81, 0xf6, - 0x14, 0xd4, 0x54, 0x00, 0x83, 0x30, 0xac, 0xb4, 0x88, 0x90, 0x00, 0xac, 0x71, 0xfa, 0xd8, 0x21, - 0x3f, 0x29, 0xe5, 0x77, 0x39, 0x1d, 0x45, 0xcb, 0xa4, 0xbe, 0x10, 0xe7, 0xf6, 0x6f, 0x97, 0xe0, - 0x11, 0xae, 0x92, 0x17, 0x1d, 0xdf, 0x69, 0x91, 0x36, 0xed, 0x55, 0xbf, 0x21, 0x0a, 0x0d, 0xba, - 0x35, 0x73, 0x65, 0x54, 0xec, 0x51, 0xd7, 0x2e, 0x5f, 0x73, 0x7c, 0x95, 0xcd, 0xfb, 0x6e, 0x82, - 0x19, 0x71, 0x14, 0x43, 0x55, 0xd6, 0x8c, 0x17, 0xb2, 0xb8, 0x20, 0x46, 0x4a, 0x2c, 0x09, 0xbd, - 0x49, 0xb0, 0x62, 0x44, 0x0d, 0x57, 0x2f, 0x68, 0x6c, 0x62, 0x12, 0x06, 0x4c, 0xee, 0x1a, 0x41, - 0x89, 0x0b, 0xa2, 0x1d, 0x2b, 0x0c, 0xfb, 0xb7, 0x2d, 0xc8, 0x6a, 0x24, 0xa3, 0x56, 0x9a, 0xb5, - 0x67, 0xad, 0xb4, 0x03, 0x14, 0x2b, 0xfb, 0x09, 0x18, 0x76, 0x12, 0x6a, 0x44, 0xf0, 0x6d, 0x77, - 0xf9, 0x70, 0xc7, 0x1a, 0x8b, 0x41, 0xd3, 0x5d, 0x77, 0xd9, 0x76, 0xdb, 0x24, 0x67, 0xff, 0x8f, - 0x01, 0x38, 0xd5, 0x95, 0xbb, 0x81, 0x5e, 0x84, 0x91, 0x86, 0x98, 0x1e, 0xa1, 0x74, 0x68, 0xd5, - 0xcc, 0x20, 0x36, 0x0d, 0xc3, 0x29, 0xcc, 0x3e, 0x26, 0xe8, 0x3c, 0x9c, 0x8e, 0xe8, 0x46, 0xbf, - 0x43, 0xa6, 0xd7, 0x13, 0x12, 0xad, 0x90, 0x46, 0xe0, 0x37, 0x79, 0x45, 0xbf, 0x72, 0xfd, 0xd1, - 0xbb, 0xbb, 0x13, 0xa7, 0x71, 0x37, 0x18, 0xe7, 0x3d, 0x83, 0x42, 0x18, 0xf5, 0x4c, 0x1b, 0x50, - 0x6c, 0x00, 0x0e, 0x65, 0x3e, 0x2a, 0x1b, 0x21, 0xd5, 0x8c, 0xd3, 0x0c, 0xd2, 0x86, 0x64, 0xe5, - 0x01, 0x19, 0x92, 0x9f, 0xd4, 0x86, 0x24, 0x3f, 0x7f, 0xff, 0x60, 0xc1, 0xb9, 0x3b, 0xc7, 0x6d, - 0x49, 0xbe, 0x0c, 0x55, 0x19, 0x9b, 0xd4, 0x57, 0x4c, 0x8f, 0x49, 0xa7, 0x87, 0x44, 0xbb, 0x57, - 0x82, 0x9c, 0x4d, 0x08, 0x5d, 0x67, 0x5a, 0xe3, 0xa7, 0xd6, 0xd9, 0xc1, 0xb4, 0x3e, 0xda, 0xe6, - 0x71, 0x59, 0x5c, 0xb7, 0x7d, 0xa0, 0xe8, 0x4d, 0x94, 0x0e, 0xd5, 0x52, 0x29, 0x0d, 0x2a, 0x5c, - 0xeb, 0x22, 0x80, 0x36, 0xd4, 0x44, 0xc0, 0xba, 0x3a, 0xf6, 0xd5, 0xf6, 0x1c, 0x36, 0xb0, 0xe8, - 0x9e, 0xda, 0xf5, 0xe3, 0xc4, 0xf1, 0xbc, 0x2b, 0xae, 0x9f, 0x08, 0xe7, 0xa0, 0x52, 0xe2, 0xf3, - 0x1a, 0x84, 0x4d, 0xbc, 0xf3, 0xef, 0x31, 0xbe, 0xcb, 0x41, 0xbe, 0xe7, 0x06, 0x9c, 0x9b, 0x73, - 0x13, 0x95, 0x66, 0xa1, 0xe6, 0x11, 0xb5, 0xc3, 0x54, 0xda, 0x90, 0xd5, 0x33, 0x6d, 0xc8, 0x48, - 0x73, 0x28, 0xa5, 0xb3, 0x32, 0xb2, 0x69, 0x0e, 0xf6, 0x8b, 0x70, 0x66, 0xce, 0x4d, 0x2e, 0xbb, - 0x1e, 0x39, 0x20, 0x13, 0xfb, 0x37, 0x07, 0x61, 0xc4, 0x4c, 0xd4, 0x3b, 0x48, 0xe6, 0xd3, 0xe7, - 0xa9, 0xa9, 0x25, 0xde, 0xce, 0x55, 0x87, 0x66, 0xb7, 0x8e, 0x9c, 0x35, 0x98, 0x3f, 0x62, 0x86, - 0xb5, 0xa5, 0x79, 0x62, 0xb3, 0x03, 0xe8, 0x0e, 0x54, 0xd6, 0x59, 0x18, 0x7e, 0xb9, 0x88, 0xc8, - 0x82, 0xbc, 0x11, 0xd5, 0xcb, 0x8c, 0x07, 0xf2, 0x73, 0x7e, 0x54, 0x43, 0x46, 0xe9, 0xdc, 0x2e, - 0x23, 0x74, 0x54, 0x64, 0x75, 0x29, 0x8c, 0x5e, 0xa2, 0xbe, 0x72, 0x08, 0x51, 0x9f, 0x12, 0xbc, - 0x83, 0x0f, 0x48, 0xf0, 0xb2, 0x94, 0x8a, 0x64, 0x83, 0xd9, 0x6f, 0x22, 0xd6, 0x7d, 0x88, 0x0d, - 0x82, 0x91, 0x52, 0x91, 0x02, 0xe3, 0x2c, 0x3e, 0xfa, 0x98, 0x12, 0xdd, 0xd5, 0x22, 0xfc, 0xaa, - 0xe6, 0x8c, 0x3e, 0x6e, 0xa9, 0xfd, 0xd9, 0x12, 0x8c, 0xcd, 0xf9, 0x9d, 0xe5, 0xb9, 0xe5, 0xce, - 0x9a, 0xe7, 0x36, 0xae, 0x91, 0x1d, 0x2a, 0x9a, 0x37, 0xc9, 0xce, 0xfc, 0xac, 0x58, 0x41, 0x6a, - 0xce, 0x5c, 0xa3, 0x8d, 0x98, 0xc3, 0xa8, 0x30, 0x5a, 0x77, 0xfd, 0x16, 0x89, 0xc2, 0xc8, 0x15, - 0x2e, 0x4f, 0x43, 0x18, 0x5d, 0xd6, 0x20, 0x6c, 0xe2, 0x51, 0xda, 0xc1, 0x1d, 0x9f, 0x44, 0x59, - 0x43, 0x76, 0x89, 0x36, 0x62, 0x0e, 0xa3, 0x48, 0x49, 0xd4, 0x89, 0x13, 0x31, 0x19, 0x15, 0xd2, - 0x2a, 0x6d, 0xc4, 0x1c, 0x46, 0x57, 0x7a, 0xdc, 0x59, 0x63, 0x81, 0x1b, 0x99, 0xc0, 0xfa, 0x15, - 0xde, 0x8c, 0x25, 0x9c, 0xa2, 0x6e, 0x92, 0x9d, 0x59, 0xba, 0xeb, 0xcd, 0xe4, 0xd7, 0x5c, 0xe3, - 0xcd, 0x58, 0xc2, 0x59, 0x29, 0xc2, 0xf4, 0x70, 0x7c, 0xd7, 0x95, 0x22, 0x4c, 0x77, 0xbf, 0xc7, - 0xfe, 0xf9, 0x97, 0x2c, 0x18, 0x31, 0xc3, 0xad, 0x50, 0x2b, 0x63, 0xe3, 0x2e, 0x75, 0x55, 0xb2, - 0xfd, 0xb1, 0xbc, 0xab, 0xbd, 0x5a, 0x6e, 0x12, 0x84, 0xf1, 0xb3, 0xc4, 0x6f, 0xb9, 0x3e, 0x61, - 0xa7, 0xe8, 0x3c, 0x4c, 0x2b, 0x15, 0xcb, 0x35, 0x13, 0x34, 0xc9, 0x21, 0x8c, 0x64, 0xfb, 0x16, - 0x9c, 0xea, 0x4a, 0xaa, 0xea, 0xc3, 0xb4, 0xd8, 0x37, 0xa5, 0xd5, 0xc6, 0x30, 0x4c, 0x09, 0xcb, - 0x72, 0x38, 0x33, 0x70, 0x8a, 0x2f, 0x24, 0xca, 0x69, 0xa5, 0xb1, 0x41, 0xda, 0x2a, 0x51, 0x8e, - 0xf9, 0xd7, 0x6f, 0x66, 0x81, 0xb8, 0x1b, 0xdf, 0xfe, 0x9c, 0x05, 0xa3, 0xa9, 0x3c, 0xb7, 0x82, - 0x8c, 0x20, 0xb6, 0xd2, 0x02, 0x16, 0xfd, 0xc7, 0x42, 0xa0, 0xcb, 0x4c, 0x99, 0xea, 0x95, 0xa6, - 0x41, 0xd8, 0xc4, 0xb3, 0xbf, 0x58, 0x82, 0xaa, 0x8c, 0xa0, 0xe8, 0xa3, 0x2b, 0x9f, 0xb1, 0x60, - 0x54, 0x9d, 0x69, 0x30, 0x67, 0x59, 0xa9, 0x88, 0xa4, 0x04, 0xda, 0x03, 0xb5, 0xdd, 0xf6, 0xd7, - 0x03, 0x6d, 0x91, 0x63, 0x93, 0x19, 0x4e, 0xf3, 0x46, 0x37, 0x01, 0xe2, 0x9d, 0x38, 0x21, 0x6d, - 0xc3, 0x6d, 0x67, 0x1b, 0x2b, 0x6e, 0xb2, 0x11, 0x44, 0x84, 0xae, 0xaf, 0xeb, 0x41, 0x93, 0xac, - 0x28, 0x4c, 0x6d, 0x42, 0xe9, 0x36, 0x6c, 0x50, 0xb2, 0xff, 0x41, 0x09, 0x4e, 0x66, 0xbb, 0x84, - 0x3e, 0x08, 0x23, 0x92, 0xbb, 0x71, 0x4d, 0x99, 0x0c, 0x1b, 0x19, 0xc1, 0x06, 0xec, 0xde, 0xee, - 0xc4, 0x44, 0xf7, 0x35, 0x71, 0x93, 0x26, 0x0a, 0x4e, 0x11, 0xe3, 0x07, 0x4b, 0xe2, 0x04, 0xb4, - 0xbe, 0x33, 0x1d, 0x86, 0xe2, 0x74, 0xc8, 0x38, 0x58, 0x32, 0xa1, 0x38, 0x83, 0x8d, 0x96, 0xe1, - 0x8c, 0xd1, 0x72, 0x9d, 0xb8, 0xad, 0x8d, 0xb5, 0x20, 0x92, 0x3b, 0xab, 0xc7, 0x74, 0x60, 0x57, - 0x37, 0x0e, 0xce, 0x7d, 0x92, 0x6a, 0xfb, 0x86, 0x13, 0x3a, 0x0d, 0x37, 0xd9, 0x11, 0x7e, 0x48, - 0x25, 0x9b, 0x66, 0x44, 0x3b, 0x56, 0x18, 0xf6, 0x22, 0x0c, 0xf4, 0x39, 0x83, 0xfa, 0xb2, 0xe8, - 0x5f, 0x86, 0x2a, 0x25, 0x27, 0xcd, 0xbb, 0x22, 0x48, 0x06, 0x50, 0x95, 0x37, 0x8d, 0x20, 0x1b, - 0xca, 0xae, 0x23, 0xcf, 0xee, 0xd4, 0x6b, 0xcd, 0xc7, 0x71, 0x87, 0x6d, 0x92, 0x29, 0x10, 0x3d, - 0x09, 0x65, 0xb2, 0x1d, 0x66, 0x0f, 0xe9, 0x2e, 0x6d, 0x87, 0x6e, 0x44, 0x62, 0x8a, 0x44, 0xb6, - 0x43, 0x74, 0x1e, 0x4a, 0x6e, 0x53, 0x28, 0x29, 0x10, 0x38, 0xa5, 0xf9, 0x59, 0x5c, 0x72, 0x9b, - 0xf6, 0x36, 0xd4, 0xd4, 0xd5, 0x26, 0x68, 0x53, 0xca, 0x6e, 0xab, 0x88, 0x90, 0x27, 0x49, 0xb7, - 0x87, 0xd4, 0xee, 0x00, 0xe8, 0x84, 0xbf, 0xa2, 0xe4, 0xcb, 0x05, 0x18, 0x68, 0x04, 0x22, 0x19, - 0xb9, 0xaa, 0xc9, 0x30, 0xa1, 0xcd, 0x20, 0xf6, 0x2d, 0x18, 0xbb, 0xe6, 0x07, 0x77, 0x58, 0x5d, - 0x76, 0x56, 0x86, 0x8c, 0x12, 0x5e, 0xa7, 0x3f, 0xb2, 0x26, 0x02, 0x83, 0x62, 0x0e, 0x53, 0xf5, - 0x99, 0x4a, 0xbd, 0xea, 0x33, 0xd9, 0x1f, 0xb7, 0x60, 0x44, 0x65, 0x0e, 0xcd, 0x6d, 0x6d, 0x52, - 0xba, 0xad, 0x28, 0xe8, 0x84, 0x59, 0xba, 0xec, 0xf2, 0x21, 0xcc, 0x61, 0x66, 0x4a, 0x5d, 0x69, - 0x9f, 0x94, 0xba, 0x0b, 0x30, 0xb0, 0xe9, 0xfa, 0xcd, 0xec, 0x6d, 0x1a, 0xd7, 0x5c, 0xbf, 0x89, - 0x19, 0x84, 0x76, 0xe1, 0xa4, 0xea, 0x82, 0x54, 0x08, 0x2f, 0xc2, 0xc8, 0x5a, 0xc7, 0xf5, 0x9a, - 0xb2, 0xbe, 0x5a, 0xc6, 0x53, 0x52, 0x37, 0x60, 0x38, 0x85, 0x49, 0xf7, 0x75, 0x6b, 0xae, 0xef, - 0x44, 0x3b, 0xcb, 0x5a, 0x03, 0x29, 0xa1, 0x54, 0x57, 0x10, 0x6c, 0x60, 0xd9, 0x6f, 0x96, 0x61, - 0x2c, 0x9d, 0x3f, 0xd5, 0xc7, 0xf6, 0xea, 0x49, 0xa8, 0xb0, 0x94, 0xaa, 0xec, 0xa7, 0xe5, 0x25, - 0xc9, 0x38, 0x0c, 0xc5, 0x30, 0xc8, 0x8b, 0x31, 0x14, 0x73, 0x13, 0x8d, 0xea, 0xa4, 0xf2, 0xaf, - 0xb0, 0x78, 0x32, 0x51, 0xff, 0x41, 0xb0, 0x42, 0x9f, 0xb2, 0x60, 0x28, 0x08, 0xcd, 0xba, 0x3e, - 0x1f, 0x28, 0x32, 0xb7, 0x4c, 0x24, 0xcb, 0x08, 0x8b, 0x58, 0x7d, 0x7a, 0xf9, 0x39, 0x24, 0xeb, - 0xf3, 0x3f, 0x02, 0x23, 0x26, 0xe6, 0x7e, 0x46, 0x71, 0xd5, 0x34, 0x8a, 0x3f, 0x63, 0x4e, 0x0a, - 0x91, 0x3d, 0xd7, 0xc7, 0x72, 0xbb, 0x01, 0x95, 0x86, 0x0a, 0x00, 0x38, 0x54, 0x55, 0x4e, 0x55, - 0x1d, 0x81, 0x1d, 0x02, 0x71, 0x6a, 0xf6, 0xb7, 0x2c, 0x63, 0x7e, 0x60, 0x12, 0xcf, 0x37, 0x51, - 0x04, 0xe5, 0xd6, 0xd6, 0xa6, 0x30, 0x45, 0xaf, 0x16, 0x34, 0xbc, 0x73, 0x5b, 0x9b, 0x7a, 0x8e, - 0x9b, 0xad, 0x98, 0x32, 0xeb, 0xc3, 0x09, 0x98, 0x4a, 0xb2, 0x2c, 0xef, 0x9f, 0x64, 0x69, 0xbf, - 0x55, 0x82, 0x53, 0x5d, 0x93, 0x0a, 0xbd, 0x01, 0x95, 0x88, 0xbe, 0xa5, 0x78, 0xbd, 0x85, 0xc2, - 0xd2, 0x22, 0xe3, 0xf9, 0xa6, 0xd6, 0xbb, 0xe9, 0x76, 0xcc, 0x59, 0xa2, 0xab, 0x80, 0x74, 0x98, - 0x8a, 0xf2, 0x40, 0xf2, 0x57, 0x3e, 0x2f, 0x1e, 0x45, 0xd3, 0x5d, 0x18, 0x38, 0xe7, 0x29, 0xf4, - 0x52, 0xd6, 0x91, 0x59, 0x4e, 0x9f, 0x5b, 0xee, 0xe5, 0x93, 0xb4, 0xff, 0x79, 0x09, 0x46, 0x53, - 0x65, 0x96, 0x90, 0x07, 0x55, 0xe2, 0x31, 0xa7, 0xbe, 0x54, 0x36, 0x47, 0xad, 0x5a, 0xac, 0x14, - 0xe4, 0x25, 0x41, 0x17, 0x2b, 0x0e, 0x0f, 0xc7, 0xe1, 0xfa, 0x8b, 0x30, 0x22, 0x3b, 0xf4, 0x01, - 0xa7, 0xed, 0x89, 0x01, 0x54, 0x73, 0xf4, 0x92, 0x01, 0xc3, 0x29, 0x4c, 0xfb, 0x77, 0xca, 0x30, - 0xce, 0x4f, 0x41, 0x9a, 0x6a, 0xe6, 0x2d, 0xca, 0xfd, 0xd6, 0x5f, 0xd1, 0xc5, 0xd0, 0xf8, 0x40, - 0xae, 0x1d, 0xf5, 0x92, 0x80, 0x7c, 0x46, 0x7d, 0x45, 0x66, 0x7d, 0x25, 0x13, 0x99, 0xc5, 0xcd, - 0xee, 0xd6, 0x31, 0xf5, 0xe8, 0xbb, 0x2b, 0x54, 0xeb, 0x57, 0x4a, 0x70, 0x22, 0x73, 0x03, 0x03, - 0x7a, 0x33, 0x5d, 0xb4, 0xd7, 0x2a, 0xc2, 0x57, 0xbe, 0x67, 0x51, 0xfe, 0x83, 0x95, 0xee, 0x7d, - 0x40, 0x4b, 0xc5, 0xfe, 0x83, 0x12, 0x8c, 0xa5, 0xaf, 0x8e, 0x78, 0x08, 0x47, 0xea, 0xdd, 0x50, - 0x63, 0xd5, 0xd1, 0xd9, 0x95, 0x98, 0xdc, 0x25, 0xcf, 0x0b, 0x51, 0xcb, 0x46, 0xac, 0xe1, 0x0f, - 0x45, 0x45, 0x64, 0xfb, 0xef, 0x59, 0x70, 0x96, 0xbf, 0x65, 0x76, 0x1e, 0xfe, 0xd5, 0xbc, 0xd1, - 0x7d, 0xb5, 0xd8, 0x0e, 0x66, 0x8a, 0xf8, 0xed, 0x37, 0xbe, 0xec, 0x2a, 0x3e, 0xd1, 0xdb, 0xf4, - 0x54, 0x78, 0x08, 0x3b, 0x7b, 0xa0, 0xc9, 0x60, 0xff, 0x41, 0x19, 0xf4, 0xed, 0x83, 0xc8, 0x15, - 0x39, 0x8e, 0x85, 0x14, 0x33, 0x5c, 0xd9, 0xf1, 0x1b, 0xfa, 0x9e, 0xc3, 0x6a, 0x26, 0xc5, 0xf1, - 0xe7, 0x2c, 0x18, 0x76, 0x7d, 0x37, 0x71, 0x1d, 0xb6, 0x8d, 0x2e, 0xe6, 0x66, 0x34, 0xc5, 0x6e, - 0x9e, 0x53, 0x0e, 0x22, 0xf3, 0x1c, 0x47, 0x31, 0xc3, 0x26, 0x67, 0xf4, 0x61, 0x11, 0x3c, 0x5d, - 0x2e, 0x2c, 0x3b, 0xb7, 0x9a, 0x89, 0x98, 0x0e, 0xa9, 0xe1, 0x95, 0x44, 0x05, 0x25, 0xb5, 0x63, - 0x4a, 0x4a, 0xd5, 0xc5, 0xd5, 0xf7, 0x40, 0xd3, 0x66, 0xcc, 0x19, 0xd9, 0x31, 0xa0, 0xee, 0xb1, - 0x38, 0x60, 0x60, 0xea, 0x14, 0xd4, 0x9c, 0x4e, 0x12, 0xb4, 0xe9, 0x30, 0x89, 0xa3, 0x26, 0x1d, - 0x7a, 0x2b, 0x01, 0x58, 0xe3, 0xd8, 0x6f, 0x56, 0x20, 0x93, 0x74, 0x88, 0xb6, 0xcd, 0x9b, 0x33, - 0xad, 0x62, 0x6f, 0xce, 0x54, 0x9d, 0xc9, 0xbb, 0x3d, 0x13, 0xb5, 0xa0, 0x12, 0x6e, 0x38, 0xb1, - 0x34, 0xab, 0x5f, 0x56, 0xfb, 0x38, 0xda, 0x78, 0x6f, 0x77, 0xe2, 0xc7, 0xfb, 0xf3, 0xba, 0xd2, - 0xb9, 0x3a, 0xc5, 0x8b, 0x8d, 0x68, 0xd6, 0x8c, 0x06, 0xe6, 0xf4, 0x0f, 0x72, 0x37, 0xdc, 0x27, - 0x44, 0x19, 0x78, 0x4c, 0xe2, 0x8e, 0x97, 0x88, 0xd9, 0xf0, 0x72, 0x81, 0xab, 0x8c, 0x13, 0xd6, - 0xe9, 0xf2, 0xfc, 0x3f, 0x36, 0x98, 0xa2, 0x0f, 0x42, 0x2d, 0x4e, 0x9c, 0x28, 0x39, 0x64, 0x82, - 0xab, 0x1a, 0xf4, 0x15, 0x49, 0x04, 0x6b, 0x7a, 0xe8, 0x15, 0x56, 0xdb, 0xd5, 0x8d, 0x37, 0x0e, - 0x99, 0xf3, 0x20, 0xeb, 0xc0, 0x0a, 0x0a, 0xd8, 0xa0, 0x86, 0x2e, 0x02, 0xb0, 0xb9, 0xcd, 0x03, - 0xfd, 0xaa, 0xcc, 0xcb, 0xa4, 0x44, 0x21, 0x56, 0x10, 0x6c, 0x60, 0xd9, 0x3f, 0x08, 0xe9, 0x7a, - 0x0f, 0x68, 0x42, 0x96, 0x97, 0xe0, 0x5e, 0x68, 0x96, 0xbb, 0x90, 0xaa, 0x04, 0xf1, 0xeb, 0x16, - 0x98, 0x45, 0x29, 0xd0, 0xeb, 0xbc, 0xfa, 0x85, 0x55, 0xc4, 0xc9, 0xa1, 0x41, 0x77, 0x72, 0xd1, - 0x09, 0x33, 0x47, 0xd8, 0xb2, 0x04, 0xc6, 0xf9, 0xf7, 0x40, 0x55, 0x42, 0x0f, 0x64, 0xd4, 0x7d, - 0x0c, 0x4e, 0x67, 0xef, 0x15, 0x17, 0xa7, 0x4e, 0xfb, 0xbb, 0x7e, 0xa4, 0x3f, 0xa7, 0xd4, 0xcb, - 0x9f, 0xd3, 0xc7, 0xfd, 0xa9, 0xbf, 0x61, 0xc1, 0x85, 0xfd, 0xae, 0x3f, 0x47, 0x8f, 0xc1, 0xc0, - 0x1d, 0x27, 0x92, 0x45, 0xb7, 0x99, 0xa0, 0xbc, 0xe5, 0x44, 0x3e, 0x66, 0xad, 0x68, 0x07, 0x06, - 0x79, 0x34, 0x98, 0xb0, 0xd6, 0x5f, 0x2e, 0xf6, 0x32, 0xf6, 0x6b, 0xc4, 0xd8, 0x2e, 0xf0, 0x48, - 0x34, 0x2c, 0x18, 0xda, 0xdf, 0xb6, 0x00, 0x2d, 0x6d, 0x91, 0x28, 0x72, 0x9b, 0x46, 0xfc, 0x1a, - 0xbb, 0x4e, 0xc5, 0xb8, 0x36, 0xc5, 0x4c, 0x71, 0xcd, 0x5c, 0xa7, 0x62, 0xfc, 0xcb, 0xbf, 0x4e, - 0xa5, 0x74, 0xb0, 0xeb, 0x54, 0xd0, 0x12, 0x9c, 0x6d, 0xf3, 0xed, 0x06, 0xbf, 0xa2, 0x80, 0xef, - 0x3d, 0x54, 0x42, 0xd9, 0xb9, 0xbb, 0xbb, 0x13, 0x67, 0x17, 0xf3, 0x10, 0x70, 0xfe, 0x73, 0xf6, - 0x7b, 0x00, 0xf1, 0xb0, 0xb5, 0x99, 0xbc, 0x18, 0xa4, 0x9e, 0xee, 0x17, 0xfb, 0xcb, 0x15, 0x38, - 0x91, 0x29, 0xc9, 0x4a, 0xb7, 0x7a, 0xdd, 0x41, 0x4f, 0x47, 0xd6, 0xdf, 0xdd, 0xdd, 0xeb, 0x2b, - 0x8c, 0xca, 0x87, 0x8a, 0xeb, 0x87, 0x9d, 0xa4, 0x98, 0x1c, 0x52, 0xde, 0x89, 0x79, 0x4a, 0xd0, - 0x70, 0x17, 0xd3, 0xbf, 0x98, 0xb3, 0x29, 0x32, 0x28, 0x2b, 0x65, 0x8c, 0x0f, 0x3c, 0x20, 0x77, - 0xc0, 0x27, 0x74, 0x88, 0x54, 0xa5, 0x08, 0xc7, 0x62, 0x66, 0xb2, 0x1c, 0xf7, 0x51, 0xfb, 0xaf, - 0x95, 0x60, 0xd8, 0xf8, 0x68, 0xe8, 0x17, 0xd3, 0x25, 0x9b, 0xac, 0xe2, 0x5e, 0x89, 0xd1, 0x9f, - 0xd4, 0x45, 0x99, 0xf8, 0x2b, 0x3d, 0xd5, 0x5d, 0xad, 0xe9, 0xde, 0xee, 0xc4, 0xc9, 0x4c, 0x3d, - 0xa6, 0x54, 0x05, 0xa7, 0xf3, 0x1f, 0x85, 0x13, 0x19, 0x32, 0x39, 0xaf, 0xbc, 0x9a, 0xbe, 0x36, - 0xfe, 0x88, 0x6e, 0x29, 0x73, 0xc8, 0xbe, 0x4e, 0x87, 0x4c, 0xa4, 0xd1, 0x05, 0x1e, 0xe9, 0xc3, - 0x07, 0x9b, 0xc9, 0x96, 0x2d, 0xf5, 0x99, 0x2d, 0xfb, 0x34, 0x54, 0xc3, 0xc0, 0x73, 0x1b, 0xae, - 0xaa, 0x42, 0xc8, 0xf2, 0x73, 0x97, 0x45, 0x1b, 0x56, 0x50, 0x74, 0x07, 0x6a, 0xea, 0x86, 0x7d, - 0xe1, 0xdf, 0x2e, 0xea, 0xd0, 0x47, 0x19, 0x2d, 0xfa, 0xe6, 0x7c, 0xcd, 0x0b, 0xd9, 0x30, 0xc8, - 0x94, 0xa0, 0x0c, 0xfd, 0x67, 0xbe, 0x77, 0xa6, 0x1d, 0x63, 0x2c, 0x20, 0xf6, 0xd7, 0x6a, 0x70, - 0x26, 0xaf, 0x2e, 0x36, 0xfa, 0x08, 0x0c, 0xf2, 0x3e, 0x16, 0x73, 0xf5, 0x42, 0x1e, 0x8f, 0x39, - 0x46, 0x50, 0x74, 0x8b, 0xfd, 0xc6, 0x82, 0xa7, 0xe0, 0xee, 0x39, 0x6b, 0x62, 0x86, 0x1c, 0x0f, - 0xf7, 0x05, 0x47, 0x73, 0x5f, 0x70, 0x38, 0x77, 0xcf, 0x59, 0x43, 0xdb, 0x50, 0x69, 0xb9, 0x09, - 0x71, 0x84, 0x13, 0xe1, 0xd6, 0xb1, 0x30, 0x27, 0x0e, 0xb7, 0xd2, 0xd8, 0x4f, 0xcc, 0x19, 0xa2, - 0xaf, 0x5a, 0x70, 0x62, 0x2d, 0x9d, 0x1a, 0x2f, 0x84, 0xa7, 0x73, 0x0c, 0xb5, 0xcf, 0xd3, 0x8c, - 0xf8, 0x7d, 0x42, 0x99, 0x46, 0x9c, 0xed, 0x0e, 0xfa, 0xa4, 0x05, 0x43, 0xeb, 0xae, 0x67, 0x94, - 0xc1, 0x3d, 0x86, 0x8f, 0x73, 0x99, 0x31, 0xd0, 0x3b, 0x0e, 0xfe, 0x3f, 0xc6, 0x92, 0x73, 0x2f, - 0x4d, 0x35, 0x78, 0x54, 0x4d, 0x35, 0xf4, 0x80, 0x34, 0xd5, 0xa7, 0x2d, 0xa8, 0xa9, 0x91, 0x16, - 0xe9, 0xce, 0x1f, 0x3c, 0xc6, 0x4f, 0xce, 0x3d, 0x27, 0xea, 0x2f, 0xd6, 0xcc, 0xd1, 0x17, 0x2c, - 0x18, 0x76, 0xde, 0xe8, 0x44, 0xa4, 0x49, 0xb6, 0x82, 0x30, 0x16, 0x97, 0x11, 0xbe, 0x5a, 0x7c, - 0x67, 0xa6, 0x29, 0x93, 0x59, 0xb2, 0xb5, 0x14, 0xc6, 0x22, 0x2d, 0x49, 0x37, 0x60, 0xb3, 0x0b, - 0xf6, 0x6e, 0x09, 0x26, 0xf6, 0xa1, 0x80, 0x5e, 0x84, 0x91, 0x20, 0x6a, 0x39, 0xbe, 0xfb, 0x86, - 0x59, 0xeb, 0x42, 0x59, 0x59, 0x4b, 0x06, 0x0c, 0xa7, 0x30, 0xcd, 0x84, 0xec, 0xd2, 0x3e, 0x09, - 0xd9, 0x17, 0x60, 0x20, 0x22, 0x61, 0x90, 0xdd, 0x2c, 0xb0, 0x94, 0x00, 0x06, 0x41, 0x8f, 0x43, - 0xd9, 0x09, 0x5d, 0x11, 0x88, 0xa6, 0xf6, 0x40, 0xd3, 0xcb, 0xf3, 0x98, 0xb6, 0xa7, 0xea, 0x43, - 0x54, 0xee, 0x4b, 0x7d, 0x08, 0xaa, 0x06, 0xc4, 0xd9, 0xc5, 0xa0, 0x56, 0x03, 0xe9, 0x33, 0x05, - 0xfb, 0xad, 0x32, 0x3c, 0xbe, 0xe7, 0x7c, 0xd1, 0x71, 0x78, 0xd6, 0x1e, 0x71, 0x78, 0x72, 0x78, - 0x4a, 0xfb, 0x0d, 0x4f, 0xb9, 0xc7, 0xf0, 0x7c, 0x92, 0x2e, 0x03, 0x59, 0x23, 0xa4, 0x98, 0xeb, - 0xe4, 0x7a, 0x95, 0x1c, 0x11, 0x2b, 0x40, 0x42, 0xb1, 0xe6, 0x4b, 0xf7, 0x00, 0xa9, 0x64, 0xe4, - 0x4a, 0x11, 0x6a, 0xa0, 0x67, 0xcd, 0x10, 0x3e, 0xf7, 0x7b, 0x65, 0x38, 0xdb, 0x3f, 0x5f, 0x82, - 0x27, 0xfb, 0x90, 0xde, 0xe6, 0x2c, 0xb6, 0xfa, 0x9c, 0xc5, 0xdf, 0xdd, 0x9f, 0xc9, 0xfe, 0x6b, - 0x16, 0x9c, 0xef, 0xad, 0x3c, 0xd0, 0x73, 0x30, 0xbc, 0x16, 0x39, 0x7e, 0x63, 0x83, 0x5d, 0x91, - 0x29, 0x07, 0x85, 0x8d, 0xb5, 0x6e, 0xc6, 0x26, 0x0e, 0xdd, 0xde, 0xf2, 0x98, 0x04, 0x03, 0x43, - 0x26, 0x8f, 0xd2, 0xed, 0xed, 0x6a, 0x16, 0x88, 0xbb, 0xf1, 0xed, 0x3f, 0x2b, 0xe5, 0x77, 0x8b, - 0x1b, 0x19, 0x07, 0xf9, 0x4e, 0xe2, 0x2b, 0x94, 0xfa, 0x90, 0x25, 0xe5, 0xfb, 0x2d, 0x4b, 0x06, - 0x7a, 0xc9, 0x12, 0x34, 0x0b, 0x27, 0x8d, 0x2b, 0x54, 0x78, 0x42, 0x30, 0x0f, 0xb8, 0x55, 0x55, - 0x32, 0x96, 0x33, 0x70, 0xdc, 0xf5, 0x04, 0x7a, 0x06, 0xaa, 0xae, 0x1f, 0x93, 0x46, 0x27, 0xe2, - 0x81, 0xde, 0x46, 0x12, 0xd6, 0xbc, 0x68, 0xc7, 0x0a, 0xc3, 0xfe, 0xa5, 0x12, 0x9c, 0xeb, 0x69, - 0x67, 0xdd, 0x27, 0xd9, 0x65, 0x7e, 0x8e, 0x81, 0xfb, 0xf3, 0x39, 0xcc, 0x41, 0xaa, 0xec, 0x3b, - 0x48, 0x7f, 0xd8, 0x7b, 0x62, 0x52, 0x9b, 0xfb, 0x7b, 0x76, 0x94, 0x5e, 0x82, 0x51, 0x27, 0x0c, - 0x39, 0x1e, 0x8b, 0xd7, 0xcc, 0x54, 0xc9, 0x99, 0x36, 0x81, 0x38, 0x8d, 0xdb, 0x97, 0xf6, 0xfc, - 0x63, 0x0b, 0x6a, 0x98, 0xac, 0x73, 0xe9, 0x80, 0x6e, 0x8b, 0x21, 0xb2, 0x8a, 0xa8, 0xa7, 0x49, - 0x07, 0x36, 0x76, 0x59, 0x9d, 0xc9, 0xbc, 0xc1, 0xee, 0xbe, 0x6a, 0xa7, 0x74, 0xa0, 0xab, 0x76, - 0xd4, 0x65, 0x2b, 0xe5, 0xde, 0x97, 0xad, 0xd8, 0x5f, 0x1f, 0xa2, 0xaf, 0x17, 0x06, 0x33, 0x11, - 0x69, 0xc6, 0xf4, 0xfb, 0x76, 0x22, 0x4f, 0x4c, 0x12, 0xf5, 0x7d, 0x6f, 0xe0, 0x05, 0x4c, 0xdb, - 0x53, 0x47, 0x31, 0xa5, 0x03, 0xd5, 0x08, 0x29, 0xef, 0x5b, 0x23, 0xe4, 0x25, 0x18, 0x8d, 0xe3, - 0x8d, 0xe5, 0xc8, 0xdd, 0x72, 0x12, 0x72, 0x8d, 0xec, 0x08, 0x2b, 0x4b, 0xe7, 0xf5, 0xaf, 0x5c, - 0xd1, 0x40, 0x9c, 0xc6, 0x45, 0x73, 0x70, 0x4a, 0x57, 0xea, 0x20, 0x51, 0xc2, 0xa2, 0xfb, 0xf9, - 0x4c, 0x50, 0x49, 0xbc, 0xba, 0xb6, 0x87, 0x40, 0xc0, 0xdd, 0xcf, 0x50, 0xf9, 0x96, 0x6a, 0xa4, - 0x1d, 0x19, 0x4c, 0xcb, 0xb7, 0x14, 0x1d, 0xda, 0x97, 0xae, 0x27, 0xd0, 0x22, 0x9c, 0xe6, 0x13, - 0x63, 0x3a, 0x0c, 0x8d, 0x37, 0x1a, 0x4a, 0xd7, 0x31, 0x9c, 0xeb, 0x46, 0xc1, 0x79, 0xcf, 0xa1, - 0x17, 0x60, 0x58, 0x35, 0xcf, 0xcf, 0x8a, 0x53, 0x04, 0xe5, 0xc5, 0x50, 0x64, 0xe6, 0x9b, 0xd8, - 0xc4, 0x43, 0x1f, 0x80, 0x47, 0xf5, 0x5f, 0x9e, 0x02, 0xc6, 0x8f, 0xd6, 0x66, 0x45, 0x11, 0x24, - 0x75, 0xb5, 0xc7, 0x5c, 0x2e, 0x5a, 0x13, 0xf7, 0x7a, 0x1e, 0xad, 0xc1, 0x79, 0x05, 0xba, 0xe4, - 0x27, 0x2c, 0x9f, 0x23, 0x26, 0x75, 0x27, 0x26, 0x37, 0x22, 0x4f, 0xdc, 0x8d, 0xaa, 0x6e, 0x5d, - 0x9c, 0x73, 0x93, 0x2b, 0x79, 0x98, 0x78, 0x01, 0xef, 0x41, 0x05, 0x4d, 0x41, 0x8d, 0xf8, 0xce, - 0x9a, 0x47, 0x96, 0x66, 0xe6, 0x59, 0x31, 0x25, 0xe3, 0x24, 0xef, 0x92, 0x04, 0x60, 0x8d, 0xa3, - 0x22, 0x4c, 0x47, 0x7a, 0xde, 0x00, 0xba, 0x0c, 0x67, 0x5a, 0x8d, 0x90, 0xda, 0x1e, 0x6e, 0x83, - 0x4c, 0x37, 0x58, 0x40, 0x1d, 0xfd, 0x30, 0xbc, 0xc0, 0xa4, 0x0a, 0x9f, 0x9e, 0x9b, 0x59, 0xee, - 0xc2, 0xc1, 0xb9, 0x4f, 0xb2, 0xc0, 0xcb, 0x28, 0xd8, 0xde, 0x19, 0x3f, 0x9d, 0x09, 0xbc, 0xa4, - 0x8d, 0x98, 0xc3, 0xd0, 0x55, 0x40, 0x2c, 0x16, 0xff, 0x4a, 0x92, 0x84, 0xca, 0xd8, 0x19, 0x3f, - 0xc3, 0x5e, 0x49, 0x85, 0x91, 0x5d, 0xee, 0xc2, 0xc0, 0x39, 0x4f, 0xd9, 0xff, 0xc1, 0x82, 0x51, - 0xb5, 0x5e, 0xef, 0x43, 0x36, 0x8a, 0x97, 0xce, 0x46, 0x99, 0x3b, 0xba, 0xc4, 0x63, 0x3d, 0xef, - 0x11, 0xd2, 0xfc, 0x33, 0xc3, 0x00, 0x5a, 0x2a, 0x2a, 0x85, 0x64, 0xf5, 0x54, 0x48, 0x0f, 0xad, - 0x44, 0xca, 0xab, 0x9c, 0x52, 0x79, 0xb0, 0x95, 0x53, 0x56, 0xe0, 0xac, 0x34, 0x17, 0xf8, 0x59, - 0xd1, 0x95, 0x20, 0x56, 0x02, 0xae, 0x5a, 0x7f, 0x5c, 0x10, 0x3a, 0x3b, 0x9f, 0x87, 0x84, 0xf3, - 0x9f, 0x4d, 0x59, 0x29, 0x43, 0xfb, 0x59, 0x29, 0x7a, 0x4d, 0x2f, 0xac, 0xcb, 0x3b, 0x3c, 0x32, - 0x6b, 0x7a, 0xe1, 0xf2, 0x0a, 0xd6, 0x38, 0xf9, 0x82, 0xbd, 0x56, 0x90, 0x60, 0x87, 0x03, 0x0b, - 0x76, 0x29, 0x62, 0x86, 0x7b, 0x8a, 0x18, 0xe9, 0x93, 0x1e, 0xe9, 0xe9, 0x93, 0x7e, 0x2f, 0x8c, - 0xb9, 0xfe, 0x06, 0x89, 0xdc, 0x84, 0x34, 0xd9, 0x5a, 0x60, 0xe2, 0xa7, 0xaa, 0xd5, 0xfa, 0x7c, - 0x0a, 0x8a, 0x33, 0xd8, 0x69, 0xb9, 0x38, 0xd6, 0x87, 0x5c, 0xec, 0xa1, 0x8d, 0x4e, 0x14, 0xa3, - 0x8d, 0x4e, 0x1e, 0x5d, 0x1b, 0x9d, 0x3a, 0x56, 0x6d, 0x84, 0x0a, 0xd1, 0x46, 0x7d, 0x09, 0x7a, - 0x63, 0xfb, 0x77, 0x66, 0x9f, 0xed, 0x5f, 0x2f, 0x55, 0x74, 0xf6, 0xd0, 0xaa, 0x28, 0x5f, 0xcb, - 0x3c, 0x72, 0x28, 0x2d, 0xf3, 0xe9, 0x12, 0x9c, 0xd5, 0x72, 0x98, 0xce, 0x7e, 0x77, 0x9d, 0x4a, - 0x22, 0x76, 0x0d, 0x14, 0x3f, 0xb7, 0x31, 0x92, 0xa3, 0x74, 0x9e, 0x95, 0x82, 0x60, 0x03, 0x8b, - 0xe5, 0x18, 0x91, 0x88, 0x95, 0xd1, 0xcd, 0x0a, 0xe9, 0x19, 0xd1, 0x8e, 0x15, 0x06, 0x9d, 0x5f, - 0xf4, 0xb7, 0xc8, 0xdb, 0xcc, 0x16, 0x8b, 0x9b, 0xd1, 0x20, 0x6c, 0xe2, 0xa1, 0xa7, 0x39, 0x13, - 0x26, 0x20, 0xa8, 0xa0, 0x1e, 0x11, 0xf7, 0xc2, 0x4a, 0x99, 0xa0, 0xa0, 0xb2, 0x3b, 0x2c, 0x99, - 0xac, 0xd2, 0xdd, 0x1d, 0x16, 0x02, 0xa5, 0x30, 0xec, 0xff, 0x69, 0xc1, 0xb9, 0xdc, 0xa1, 0xb8, - 0x0f, 0xca, 0x77, 0x3b, 0xad, 0x7c, 0x57, 0x8a, 0xda, 0x6e, 0x18, 0x6f, 0xd1, 0x43, 0x11, 0xff, - 0x3b, 0x0b, 0xc6, 0x34, 0xfe, 0x7d, 0x78, 0x55, 0x37, 0xfd, 0xaa, 0xc5, 0xed, 0xac, 0x6a, 0x5d, - 0xef, 0xf6, 0x3b, 0x25, 0x50, 0x05, 0x1c, 0xa7, 0x1b, 0xb2, 0x3c, 0xee, 0x3e, 0x27, 0x89, 0x3b, - 0x30, 0xc8, 0x0e, 0x42, 0xe3, 0x62, 0x82, 0x3c, 0xd2, 0xfc, 0xd9, 0xa1, 0xaa, 0x3e, 0x64, 0x66, - 0x7f, 0x63, 0x2c, 0x18, 0xb2, 0x22, 0xcf, 0x6e, 0x4c, 0xa5, 0x79, 0x53, 0xa4, 0x65, 0xe9, 0x22, - 0xcf, 0xa2, 0x1d, 0x2b, 0x0c, 0xaa, 0x1e, 0xdc, 0x46, 0xe0, 0xcf, 0x78, 0x4e, 0x2c, 0xef, 0x3e, - 0x54, 0xea, 0x61, 0x5e, 0x02, 0xb0, 0xc6, 0x61, 0x67, 0xa4, 0x6e, 0x1c, 0x7a, 0xce, 0x8e, 0xb1, - 0x7f, 0x36, 0xea, 0x13, 0x28, 0x10, 0x36, 0xf1, 0xec, 0x36, 0x8c, 0xa7, 0x5f, 0x62, 0x96, 0xac, - 0xb3, 0x00, 0xc5, 0xbe, 0x86, 0x73, 0x0a, 0x6a, 0x0e, 0x7b, 0x6a, 0xa1, 0xe3, 0x64, 0xaf, 0x2c, - 0x9f, 0x96, 0x00, 0xac, 0x71, 0xec, 0x5f, 0xb5, 0xe0, 0x74, 0xce, 0xa0, 0x15, 0x98, 0xf6, 0x96, - 0x68, 0x69, 0x93, 0xa7, 0xd8, 0x7f, 0x00, 0x86, 0x9a, 0x64, 0xdd, 0x91, 0x21, 0x70, 0x86, 0x6c, - 0x9f, 0xe5, 0xcd, 0x58, 0xc2, 0xed, 0xff, 0x6e, 0xc1, 0x89, 0x74, 0x5f, 0x63, 0x96, 0x4a, 0xc2, - 0x87, 0xc9, 0x8d, 0x1b, 0xc1, 0x16, 0x89, 0x76, 0xe8, 0x9b, 0x5b, 0x99, 0x54, 0x92, 0x2e, 0x0c, - 0x9c, 0xf3, 0x14, 0x2b, 0xdf, 0xda, 0x54, 0xa3, 0x2d, 0x67, 0xe4, 0xcd, 0x22, 0x67, 0xa4, 0xfe, - 0x98, 0xe6, 0x71, 0xb9, 0x62, 0x89, 0x4d, 0xfe, 0xf6, 0xb7, 0x07, 0x40, 0xe5, 0xc5, 0xb2, 0xf8, - 0xa3, 0x82, 0xa2, 0xb7, 0x0e, 0x9a, 0x41, 0xa4, 0x26, 0xc3, 0xc0, 0x5e, 0x01, 0x01, 0xdc, 0x4b, - 0x62, 0xba, 0x2e, 0xd5, 0x1b, 0xae, 0x6a, 0x10, 0x36, 0xf1, 0x68, 0x4f, 0x3c, 0x77, 0x8b, 0xf0, - 0x87, 0x06, 0xd3, 0x3d, 0x59, 0x90, 0x00, 0xac, 0x71, 0x68, 0x4f, 0x9a, 0xee, 0xfa, 0xba, 0xd8, - 0xf2, 0xab, 0x9e, 0xd0, 0xd1, 0xc1, 0x0c, 0xc2, 0x2b, 0x72, 0x07, 0x9b, 0xc2, 0x0a, 0x36, 0x2a, - 0x72, 0x07, 0x9b, 0x98, 0x41, 0xa8, 0xdd, 0xe6, 0x07, 0x51, 0x9b, 0x5d, 0x29, 0xdf, 0x54, 0x5c, - 0x84, 0xf5, 0xab, 0xec, 0xb6, 0xeb, 0xdd, 0x28, 0x38, 0xef, 0x39, 0x3a, 0x03, 0xc3, 0x88, 0x34, - 0xdd, 0x46, 0x62, 0x52, 0x83, 0xf4, 0x0c, 0x5c, 0xee, 0xc2, 0xc0, 0x39, 0x4f, 0xa1, 0x69, 0x38, - 0x21, 0xf3, 0x9a, 0x65, 0xd5, 0x9a, 0xe1, 0x74, 0x95, 0x0c, 0x9c, 0x06, 0xe3, 0x2c, 0x3e, 0x95, - 0x6a, 0x6d, 0x51, 0xb0, 0x8a, 0x19, 0xcb, 0x86, 0x54, 0x93, 0x85, 0xac, 0xb0, 0xc2, 0xb0, 0x3f, - 0x51, 0xa6, 0x5a, 0xb8, 0x47, 0xa1, 0xb6, 0xfb, 0x16, 0x2d, 0x98, 0x9e, 0x91, 0x03, 0x7d, 0xcc, - 0xc8, 0xe7, 0x61, 0xe4, 0x76, 0x1c, 0xf8, 0x2a, 0x12, 0xaf, 0xd2, 0x33, 0x12, 0xcf, 0xc0, 0xca, - 0x8f, 0xc4, 0x1b, 0x2c, 0x2a, 0x12, 0x6f, 0xe8, 0x90, 0x91, 0x78, 0xdf, 0xac, 0x80, 0xba, 0x1a, - 0xe4, 0x3a, 0x49, 0xee, 0x04, 0xd1, 0xa6, 0xeb, 0xb7, 0x58, 0x3e, 0xf8, 0x57, 0x2d, 0x18, 0xe1, - 0xeb, 0x65, 0xc1, 0xcc, 0xa4, 0x5a, 0x2f, 0xe8, 0xce, 0x89, 0x14, 0xb3, 0xc9, 0x55, 0x83, 0x51, - 0xe6, 0xea, 0x4d, 0x13, 0x84, 0x53, 0x3d, 0x42, 0x1f, 0x05, 0x90, 0xfe, 0xd1, 0x75, 0x29, 0x32, - 0xe7, 0x8b, 0xe9, 0x1f, 0x26, 0xeb, 0xda, 0x06, 0x5e, 0x55, 0x4c, 0xb0, 0xc1, 0x10, 0x7d, 0x5a, - 0x67, 0x99, 0xf1, 0x90, 0xfd, 0x0f, 0x1f, 0xcb, 0xd8, 0xf4, 0x93, 0x63, 0x86, 0x61, 0xc8, 0xf5, - 0x5b, 0x74, 0x9e, 0x88, 0x88, 0xa5, 0x77, 0xe5, 0xd5, 0x52, 0x58, 0x08, 0x9c, 0x66, 0xdd, 0xf1, - 0x1c, 0xbf, 0x41, 0xa2, 0x79, 0x8e, 0x6e, 0x5e, 0x38, 0xcd, 0x1a, 0xb0, 0x24, 0xd4, 0x75, 0xa9, - 0x4a, 0xa5, 0x9f, 0x4b, 0x55, 0xce, 0xbf, 0x0f, 0x4e, 0x75, 0x7d, 0xcc, 0x03, 0xa5, 0x94, 0x1d, - 0x3e, 0x1b, 0xcd, 0xfe, 0x17, 0x83, 0x5a, 0x69, 0x5d, 0x0f, 0x9a, 0xfc, 0x6a, 0x8f, 0x48, 0x7f, - 0x51, 0x61, 0xe3, 0x16, 0x38, 0x45, 0x8c, 0x4b, 0xab, 0x55, 0x23, 0x36, 0x59, 0xd2, 0x39, 0x1a, - 0x3a, 0x11, 0xf1, 0x8f, 0x7b, 0x8e, 0x2e, 0x2b, 0x26, 0xd8, 0x60, 0x88, 0x36, 0x52, 0x39, 0x25, - 0x97, 0x8f, 0x9e, 0x53, 0xc2, 0xaa, 0x4c, 0xe5, 0x55, 0xe3, 0xff, 0x82, 0x05, 0x63, 0x7e, 0x6a, - 0xe6, 0x16, 0x13, 0x46, 0x9a, 0xbf, 0x2a, 0xf8, 0xcd, 0x52, 0xe9, 0x36, 0x9c, 0xe1, 0x9f, 0xa7, - 0xd2, 0x2a, 0x07, 0x54, 0x69, 0xfa, 0x8e, 0xa0, 0xc1, 0x5e, 0x77, 0x04, 0x21, 0x5f, 0x5d, 0x92, - 0x36, 0x54, 0xf8, 0x25, 0x69, 0x90, 0x73, 0x41, 0xda, 0x2d, 0xa8, 0x35, 0x22, 0xe2, 0x24, 0x87, - 0xbc, 0x2f, 0x8b, 0x1d, 0xd0, 0xcf, 0x48, 0x02, 0x58, 0xd3, 0xb2, 0xff, 0xcf, 0x00, 0x9c, 0x94, - 0x23, 0x22, 0x43, 0xd0, 0xa9, 0x7e, 0xe4, 0x7c, 0xb5, 0x71, 0xab, 0xf4, 0xe3, 0x15, 0x09, 0xc0, - 0x1a, 0x87, 0xda, 0x63, 0x9d, 0x98, 0x2c, 0x85, 0xc4, 0x5f, 0x70, 0xd7, 0x62, 0x71, 0xce, 0xa9, - 0x16, 0xca, 0x0d, 0x0d, 0xc2, 0x26, 0x1e, 0x35, 0xc6, 0xb9, 0x5d, 0x1c, 0x67, 0xd3, 0x57, 0x84, - 0xbd, 0x8d, 0x25, 0x1c, 0xfd, 0x42, 0x6e, 0xe5, 0xd8, 0x62, 0x12, 0xb7, 0xba, 0x22, 0xef, 0x0f, - 0x78, 0xc5, 0xe2, 0xdf, 0xb1, 0xe0, 0x2c, 0x6f, 0x95, 0x23, 0x79, 0x23, 0x6c, 0x3a, 0x09, 0x89, - 0x8b, 0xa9, 0xe4, 0x9e, 0xd3, 0x3f, 0xed, 0xe4, 0xcd, 0x63, 0x8b, 0xf3, 0x7b, 0x83, 0xde, 0xb4, - 0xe0, 0xc4, 0x66, 0xaa, 0xe6, 0x87, 0x54, 0x1d, 0x47, 0x4d, 0xc7, 0x4f, 0x11, 0xd5, 0x4b, 0x2d, - 0xdd, 0x1e, 0xe3, 0x2c, 0x77, 0xfb, 0xcf, 0x2c, 0x30, 0xc5, 0xe8, 0xfd, 0x2f, 0x15, 0x72, 0x70, - 0x53, 0x50, 0x5a, 0x97, 0x95, 0x9e, 0xd6, 0xe5, 0xe3, 0x50, 0xee, 0xb8, 0x4d, 0xb1, 0xbf, 0xd0, - 0xa7, 0xaf, 0xf3, 0xb3, 0x98, 0xb6, 0xdb, 0xff, 0xb4, 0xa2, 0xfd, 0x16, 0x22, 0x2f, 0xea, 0x7b, - 0xe2, 0xb5, 0xd7, 0x55, 0xb1, 0x31, 0xfe, 0xe6, 0xd7, 0xbb, 0x8a, 0x8d, 0xfd, 0xe8, 0xc1, 0xd3, - 0xde, 0xf8, 0x00, 0xf5, 0xaa, 0x35, 0x36, 0xb4, 0x4f, 0xce, 0xdb, 0x6d, 0xa8, 0xd2, 0x2d, 0x18, - 0x73, 0x40, 0x56, 0x53, 0x9d, 0xaa, 0x5e, 0x11, 0xed, 0xf7, 0x76, 0x27, 0x7e, 0xe4, 0xe0, 0xdd, - 0x92, 0x4f, 0x63, 0x45, 0x1f, 0xc5, 0x50, 0xa3, 0xbf, 0x59, 0x7a, 0x9e, 0xd8, 0xdc, 0xdd, 0x50, - 0x32, 0x53, 0x02, 0x0a, 0xc9, 0xfd, 0xd3, 0x7c, 0x90, 0x0f, 0x35, 0x76, 0x1b, 0x2d, 0x63, 0xca, - 0xf7, 0x80, 0xcb, 0x2a, 0x49, 0x4e, 0x02, 0xee, 0xed, 0x4e, 0xbc, 0x74, 0x70, 0xa6, 0xea, 0x71, - 0xac, 0x59, 0xd8, 0x5f, 0x1c, 0xd0, 0x73, 0x57, 0xd4, 0x98, 0xfb, 0x9e, 0x98, 0xbb, 0x2f, 0x66, - 0xe6, 0xee, 0x85, 0xae, 0xb9, 0x3b, 0xa6, 0x6f, 0x4d, 0x4d, 0xcd, 0xc6, 0xfb, 0x6d, 0x08, 0xec, - 0xef, 0x6f, 0x60, 0x16, 0xd0, 0xeb, 0x1d, 0x37, 0x22, 0xf1, 0x72, 0xd4, 0xf1, 0x5d, 0xbf, 0xc5, - 0xa6, 0x63, 0xd5, 0xb4, 0x80, 0x52, 0x60, 0x9c, 0xc5, 0xa7, 0x9b, 0x7a, 0xfa, 0xcd, 0x6f, 0x39, - 0x5b, 0x7c, 0x56, 0x19, 0x65, 0xb7, 0x56, 0x44, 0x3b, 0x56, 0x18, 0xf6, 0xd7, 0xd9, 0x59, 0xb6, - 0x91, 0x17, 0x4c, 0xe7, 0x84, 0xc7, 0xae, 0xff, 0xe5, 0x35, 0xbb, 0xd4, 0x9c, 0xe0, 0x77, 0xfe, - 0x72, 0x18, 0xba, 0x03, 0x43, 0x6b, 0xfc, 0xfe, 0xbb, 0x62, 0xea, 0x93, 0x8b, 0xcb, 0xf4, 0xd8, - 0x2d, 0x27, 0xf2, 0x66, 0xbd, 0x7b, 0xfa, 0x27, 0x96, 0xdc, 0xec, 0xdf, 0xaf, 0xc0, 0x89, 0xcc, - 0x05, 0xb1, 0xa9, 0x6a, 0xa9, 0xa5, 0x7d, 0xab, 0xa5, 0x7e, 0x08, 0xa0, 0x49, 0x42, 0x2f, 0xd8, - 0x61, 0xe6, 0xd8, 0xc0, 0x81, 0xcd, 0x31, 0x65, 0xc1, 0xcf, 0x2a, 0x2a, 0xd8, 0xa0, 0x28, 0x0a, - 0x95, 0xf1, 0xe2, 0xab, 0x99, 0x42, 0x65, 0xc6, 0x2d, 0x06, 0x83, 0xf7, 0xf7, 0x16, 0x03, 0x17, - 0x4e, 0xf0, 0x2e, 0xaa, 0xec, 0xdb, 0x43, 0x24, 0xd9, 0xb2, 0xfc, 0x85, 0xd9, 0x34, 0x19, 0x9c, - 0xa5, 0xfb, 0x20, 0xef, 0x7f, 0x46, 0xef, 0x86, 0x9a, 0xfc, 0xce, 0xf1, 0x78, 0x4d, 0x57, 0x30, - 0x90, 0xd3, 0x80, 0xdd, 0xcb, 0x2c, 0x7e, 0x76, 0x15, 0x12, 0x80, 0x07, 0x55, 0x48, 0xc0, 0xfe, - 0x7c, 0x89, 0xda, 0xf1, 0xbc, 0x5f, 0xaa, 0x26, 0xce, 0x53, 0x30, 0xe8, 0x74, 0x92, 0x8d, 0xa0, - 0xeb, 0x36, 0xbf, 0x69, 0xd6, 0x8a, 0x05, 0x14, 0x2d, 0xc0, 0x40, 0x53, 0xd7, 0x39, 0x39, 0xc8, - 0xf7, 0xd4, 0x2e, 0x51, 0x27, 0x21, 0x98, 0x51, 0x41, 0x8f, 0xc1, 0x40, 0xe2, 0xb4, 0x64, 0xca, - 0x15, 0x4b, 0xb3, 0x5d, 0x75, 0x5a, 0x31, 0x66, 0xad, 0xa6, 0xfa, 0x1e, 0xd8, 0x47, 0x7d, 0xbf, - 0x04, 0xa3, 0xb1, 0xdb, 0xf2, 0x9d, 0xa4, 0x13, 0x11, 0xe3, 0x98, 0x4f, 0x47, 0x6e, 0x98, 0x40, - 0x9c, 0xc6, 0xb5, 0x7f, 0x73, 0x04, 0xce, 0xac, 0xcc, 0x2c, 0xca, 0xea, 0xdd, 0xc7, 0x96, 0x35, - 0x95, 0xc7, 0xe3, 0xfe, 0x65, 0x4d, 0xf5, 0xe0, 0xee, 0x19, 0x59, 0x53, 0x9e, 0x91, 0x35, 0x95, - 0x4e, 0x61, 0x29, 0x17, 0x91, 0xc2, 0x92, 0xd7, 0x83, 0x7e, 0x52, 0x58, 0x8e, 0x2d, 0x8d, 0x6a, - 0xcf, 0x0e, 0x1d, 0x28, 0x8d, 0x4a, 0xe5, 0x98, 0x15, 0x92, 0x5c, 0xd0, 0xe3, 0x53, 0xe5, 0xe6, - 0x98, 0xa9, 0xfc, 0x1e, 0x9e, 0x38, 0x23, 0x44, 0xfd, 0xab, 0xc5, 0x77, 0xa0, 0x8f, 0xfc, 0x1e, - 0x91, 0xbb, 0x63, 0xe6, 0x94, 0x0d, 0x15, 0x91, 0x53, 0x96, 0xd7, 0x9d, 0x7d, 0x73, 0xca, 0x5e, - 0x82, 0xd1, 0x86, 0x17, 0xf8, 0x64, 0x39, 0x0a, 0x92, 0xa0, 0x11, 0x78, 0xc2, 0xac, 0x57, 0x22, - 0x61, 0xc6, 0x04, 0xe2, 0x34, 0x6e, 0xaf, 0x84, 0xb4, 0xda, 0x51, 0x13, 0xd2, 0xe0, 0x01, 0x25, - 0xa4, 0xfd, 0xac, 0x4e, 0x9d, 0x1e, 0x66, 0x5f, 0xe4, 0x43, 0xc5, 0x7f, 0x91, 0x7e, 0xf2, 0xa7, - 0xd1, 0x5b, 0xfc, 0x3a, 0x3d, 0x6a, 0x18, 0xcf, 0x04, 0x6d, 0x6a, 0xf8, 0x8d, 0xb0, 0x21, 0x79, - 0xed, 0x18, 0x26, 0xec, 0xad, 0x15, 0xcd, 0x46, 0x5d, 0xb1, 0xa7, 0x9b, 0x70, 0xba, 0x23, 0x47, - 0x49, 0xed, 0xfe, 0x72, 0x09, 0xbe, 0x6f, 0xdf, 0x2e, 0xa0, 0x3b, 0x00, 0x89, 0xd3, 0x12, 0x13, - 0x55, 0x1c, 0x98, 0x1c, 0x31, 0xbc, 0x72, 0x55, 0xd2, 0xe3, 0x35, 0x49, 0xd4, 0x5f, 0x76, 0x14, - 0x21, 0x7f, 0xb3, 0xa8, 0xca, 0xc0, 0xeb, 0x2a, 0xdd, 0x88, 0x03, 0x8f, 0x60, 0x06, 0xa1, 0xea, - 0x3f, 0x22, 0x2d, 0x7d, 0xff, 0xb3, 0xfa, 0x7c, 0x98, 0xb5, 0x62, 0x01, 0x45, 0x2f, 0xc0, 0xb0, - 0xe3, 0x79, 0x3c, 0x3f, 0x86, 0xc4, 0xe2, 0x3e, 0x1d, 0x5d, 0x43, 0x4e, 0x83, 0xb0, 0x89, 0x67, - 0xff, 0x69, 0x09, 0x26, 0xf6, 0x91, 0x29, 0x5d, 0x19, 0x7f, 0x95, 0xbe, 0x33, 0xfe, 0x44, 0x8e, - 0xc2, 0x60, 0x8f, 0x1c, 0x85, 0x17, 0x60, 0x38, 0x21, 0x4e, 0x5b, 0x04, 0x64, 0x09, 0x4f, 0x80, - 0x3e, 0x01, 0xd6, 0x20, 0x6c, 0xe2, 0x51, 0x29, 0x36, 0xe6, 0x34, 0x1a, 0x24, 0x8e, 0x65, 0x12, - 0x82, 0xf0, 0xa6, 0x16, 0x96, 0xe1, 0xc0, 0x9c, 0xd4, 0xd3, 0x29, 0x16, 0x38, 0xc3, 0x32, 0x3b, - 0xe0, 0xb5, 0x3e, 0x07, 0xfc, 0x6b, 0x25, 0x78, 0x7c, 0x4f, 0xed, 0xd6, 0x77, 0x7e, 0x48, 0x27, - 0x26, 0x51, 0x76, 0xe2, 0xdc, 0x88, 0x49, 0x84, 0x19, 0x84, 0x8f, 0x52, 0x18, 0x1a, 0xf7, 0x6b, - 0x17, 0x9d, 0xbc, 0xc4, 0x47, 0x29, 0xc5, 0x02, 0x67, 0x58, 0x1e, 0x76, 0x5a, 0xfe, 0xfd, 0x12, - 0x3c, 0xd9, 0x87, 0x0d, 0x50, 0x60, 0x92, 0x57, 0x3a, 0xd5, 0xae, 0xfc, 0x80, 0x32, 0x22, 0x0f, - 0x39, 0x5c, 0x5f, 0x2f, 0xc1, 0xf9, 0xde, 0xaa, 0x18, 0xfd, 0x18, 0x9c, 0x88, 0x54, 0x14, 0x96, - 0x99, 0xa5, 0x77, 0x9a, 0x7b, 0x12, 0x52, 0x20, 0x9c, 0xc5, 0x45, 0x93, 0x00, 0xa1, 0x93, 0x6c, - 0xc4, 0x97, 0xb6, 0xdd, 0x38, 0x11, 0x55, 0x68, 0xc6, 0xf8, 0xd9, 0x95, 0x6c, 0xc5, 0x06, 0x06, - 0x65, 0xc7, 0xfe, 0xcd, 0x06, 0xd7, 0x83, 0x84, 0x3f, 0xc4, 0xb7, 0x11, 0xa7, 0xe5, 0x9d, 0x1d, - 0x06, 0x08, 0x67, 0x71, 0x29, 0x3b, 0x76, 0x3a, 0xca, 0x3b, 0xca, 0xf7, 0x17, 0x8c, 0xdd, 0x82, - 0x6a, 0xc5, 0x06, 0x46, 0x36, 0xff, 0xb0, 0xb2, 0x7f, 0xfe, 0xa1, 0xfd, 0x4f, 0x4a, 0x70, 0xae, - 0xa7, 0x29, 0xd7, 0xdf, 0x02, 0x7c, 0xf8, 0x72, 0x06, 0x0f, 0x37, 0x77, 0x0e, 0x98, 0xdb, 0xf6, - 0xc7, 0x3d, 0x66, 0x9a, 0xc8, 0x6d, 0x3b, 0x7c, 0x72, 0xf8, 0xc3, 0x37, 0x9e, 0x5d, 0xe9, 0x6c, - 0x03, 0x07, 0x48, 0x67, 0xcb, 0x7c, 0x8c, 0x4a, 0x9f, 0x0b, 0xf9, 0xcf, 0xcb, 0x3d, 0x87, 0x97, - 0x6e, 0xfd, 0xfa, 0xf2, 0xd3, 0xce, 0xc2, 0x49, 0xd7, 0x67, 0xf7, 0x37, 0xad, 0x74, 0xd6, 0x44, - 0x61, 0x92, 0x52, 0xfa, 0xf6, 0xf4, 0xf9, 0x0c, 0x1c, 0x77, 0x3d, 0xf1, 0x10, 0xa6, 0x17, 0x1e, - 0x6e, 0x48, 0x0f, 0x96, 0xe0, 0x8a, 0x96, 0xe0, 0xac, 0x1c, 0x8a, 0x0d, 0x27, 0x22, 0x4d, 0xa1, - 0x46, 0x62, 0x91, 0x50, 0x71, 0x8e, 0x27, 0x65, 0xe4, 0x20, 0xe0, 0xfc, 0xe7, 0xd8, 0x95, 0x39, - 0x41, 0xe8, 0x36, 0xc4, 0x26, 0x47, 0x5f, 0x99, 0x43, 0x1b, 0x31, 0x87, 0xd9, 0x1f, 0x82, 0x9a, - 0x7a, 0x7f, 0x1e, 0xd6, 0xad, 0x26, 0x5d, 0x57, 0x58, 0xb7, 0x9a, 0x71, 0x06, 0x16, 0xfd, 0x5a, - 0xd4, 0x24, 0xce, 0xac, 0x9e, 0x6b, 0x64, 0x87, 0xd9, 0xc7, 0xf6, 0x0f, 0xc1, 0x88, 0xf2, 0xb3, - 0xf4, 0x7b, 0x91, 0x90, 0xfd, 0xc5, 0x41, 0x18, 0x4d, 0x15, 0x07, 0x4c, 0x39, 0x58, 0xad, 0x7d, - 0x1d, 0xac, 0x2c, 0x4c, 0xbf, 0xe3, 0xcb, 0x5b, 0xc6, 0x8c, 0x30, 0xfd, 0x8e, 0x4f, 0x30, 0x87, - 0x51, 0xf3, 0xb6, 0x19, 0xed, 0xe0, 0x8e, 0x2f, 0xc2, 0x69, 0x95, 0x79, 0x3b, 0xcb, 0x5a, 0xb1, - 0x80, 0xa2, 0x8f, 0x5b, 0x30, 0x12, 0x33, 0xef, 0x3d, 0x77, 0x4f, 0x8b, 0x49, 0x77, 0xf5, 0xe8, - 0xb5, 0x0f, 0x55, 0x21, 0x4c, 0x16, 0x21, 0x63, 0xb6, 0xe0, 0x14, 0x47, 0xf4, 0x29, 0x0b, 0x6a, - 0xea, 0x32, 0x14, 0x71, 0x15, 0xe0, 0x4a, 0xb1, 0xb5, 0x17, 0xb9, 0x5f, 0x53, 0x1d, 0x84, 0xa8, - 0x22, 0x78, 0x58, 0x33, 0x46, 0xb1, 0xf2, 0x1d, 0x0f, 0x1d, 0x8f, 0xef, 0x18, 0x72, 0xfc, 0xc6, - 0xef, 0x86, 0x5a, 0xdb, 0xf1, 0xdd, 0x75, 0x12, 0x27, 0xdc, 0x9d, 0x2b, 0x4b, 0xc2, 0xca, 0x46, - 0xac, 0xe1, 0x54, 0x21, 0xc7, 0xec, 0xc5, 0x12, 0xc3, 0xff, 0xca, 0x14, 0xf2, 0x8a, 0x6e, 0xc6, - 0x26, 0x8e, 0xe9, 0x2c, 0x86, 0x07, 0xea, 0x2c, 0x1e, 0xde, 0xdb, 0x59, 0x6c, 0xff, 0x43, 0x0b, - 0xce, 0xe6, 0x7e, 0xb5, 0x87, 0x37, 0xf0, 0xd1, 0xfe, 0x52, 0x05, 0x4e, 0xe7, 0x54, 0xf9, 0x44, - 0x3b, 0xe6, 0x7c, 0xb6, 0x8a, 0x88, 0x21, 0x48, 0x1f, 0x89, 0xcb, 0x61, 0xcc, 0x99, 0xc4, 0x07, - 0x3b, 0xaa, 0xd1, 0xc7, 0x25, 0xe5, 0xfb, 0x7b, 0x5c, 0x62, 0x4c, 0xcb, 0x81, 0x07, 0x3a, 0x2d, - 0x2b, 0xfb, 0x9c, 0x61, 0xfc, 0x9a, 0x05, 0xe3, 0xed, 0x1e, 0xa5, 0xe5, 0x85, 0xe3, 0xf1, 0xe6, - 0xf1, 0x14, 0xae, 0xaf, 0x3f, 0x76, 0x77, 0x77, 0xa2, 0x67, 0x45, 0x7f, 0xdc, 0xb3, 0x57, 0xf6, - 0xb7, 0xcb, 0xc0, 0x4a, 0xcc, 0xb2, 0x4a, 0x6e, 0x3b, 0xe8, 0x63, 0x66, 0xb1, 0x60, 0xab, 0xa8, - 0xc2, 0xb6, 0x9c, 0xb8, 0x2a, 0x36, 0xcc, 0x47, 0x30, 0xaf, 0xf6, 0x70, 0x56, 0x68, 0x95, 0xfa, - 0x10, 0x5a, 0x9e, 0xac, 0xca, 0x5c, 0x2e, 0xbe, 0x2a, 0x73, 0x2d, 0x5b, 0x91, 0x79, 0xef, 0x4f, - 0x3c, 0xf0, 0x50, 0x7e, 0xe2, 0xbf, 0x65, 0x71, 0xc1, 0x93, 0xf9, 0x0a, 0xda, 0x32, 0xb0, 0xf6, - 0xb0, 0x0c, 0x9e, 0x81, 0x6a, 0x4c, 0xbc, 0xf5, 0x2b, 0xc4, 0xf1, 0x84, 0x05, 0xa1, 0xcf, 0xaf, - 0x45, 0x3b, 0x56, 0x18, 0xec, 0xda, 0x56, 0xcf, 0x0b, 0xee, 0x5c, 0x6a, 0x87, 0xc9, 0x8e, 0xb0, - 0x25, 0xf4, 0xb5, 0xad, 0x0a, 0x82, 0x0d, 0x2c, 0xfb, 0x6f, 0x97, 0xf8, 0x0c, 0x14, 0x41, 0x10, - 0x2f, 0x66, 0x2e, 0xda, 0xeb, 0x3f, 0x7e, 0xe0, 0x23, 0x00, 0x0d, 0x75, 0x45, 0xbd, 0x38, 0x13, - 0xba, 0x72, 0xe4, 0xfb, 0xb3, 0x05, 0x3d, 0xfd, 0x1a, 0xba, 0x0d, 0x1b, 0xfc, 0x52, 0xb2, 0xb4, - 0xbc, 0xaf, 0x2c, 0x4d, 0x89, 0x95, 0x81, 0x7d, 0xb4, 0xdd, 0x9f, 0x5a, 0x90, 0xb2, 0x88, 0x50, - 0x08, 0x15, 0xda, 0xdd, 0x9d, 0x62, 0x6e, 0xdf, 0x37, 0x49, 0x53, 0xd1, 0x28, 0xa6, 0x3d, 0xfb, - 0x89, 0x39, 0x23, 0xe4, 0x89, 0x58, 0x09, 0x3e, 0xaa, 0xd7, 0x8b, 0x63, 0x78, 0x25, 0x08, 0x36, - 0xf9, 0xc1, 0xa6, 0x8e, 0xbb, 0xb0, 0x5f, 0x84, 0x53, 0x5d, 0x9d, 0x62, 0x77, 0x6a, 0x05, 0x54, - 0xfb, 0x64, 0xa6, 0x2b, 0x4b, 0xe0, 0xc4, 0x1c, 0x66, 0x7f, 0xdd, 0x82, 0x93, 0x59, 0xf2, 0xe8, - 0x2d, 0x0b, 0x4e, 0xc5, 0x59, 0x7a, 0xc7, 0x35, 0x76, 0x2a, 0xde, 0xb1, 0x0b, 0x84, 0xbb, 0x3b, - 0x61, 0xff, 0x5f, 0x31, 0xf9, 0x6f, 0xb9, 0x7e, 0x33, 0xb8, 0xa3, 0x0c, 0x13, 0xab, 0xa7, 0x61, - 0x42, 0xd7, 0x63, 0x63, 0x83, 0x34, 0x3b, 0x5e, 0x57, 0xe6, 0xe8, 0x8a, 0x68, 0xc7, 0x0a, 0x83, - 0x25, 0xca, 0x75, 0x44, 0xd9, 0xf6, 0xcc, 0xa4, 0x9c, 0x15, 0xed, 0x58, 0x61, 0xa0, 0xe7, 0x61, - 0xc4, 0x78, 0x49, 0x39, 0x2f, 0x99, 0x41, 0x6e, 0xa8, 0xcc, 0x18, 0xa7, 0xb0, 0xd0, 0x24, 0x80, - 0x32, 0x72, 0xa4, 0x8a, 0x64, 0x8e, 0x22, 0x25, 0x89, 0x62, 0x6c, 0x60, 0xb0, 0xb4, 0x54, 0xaf, - 0x13, 0x33, 0x1f, 0xff, 0xa0, 0x2e, 0x25, 0x3a, 0x23, 0xda, 0xb0, 0x82, 0x52, 0x69, 0xd2, 0x76, - 0xfc, 0x8e, 0xe3, 0xd1, 0x11, 0x12, 0x5b, 0x3f, 0xb5, 0x0c, 0x17, 0x15, 0x04, 0x1b, 0x58, 0xf4, - 0x8d, 0x13, 0xb7, 0x4d, 0x5e, 0x09, 0x7c, 0x19, 0xa7, 0xa6, 0x8f, 0x7d, 0x44, 0x3b, 0x56, 0x18, - 0xf6, 0x7f, 0xb5, 0xe0, 0x84, 0x4e, 0x72, 0xe7, 0xb7, 0x67, 0x9b, 0x3b, 0x55, 0x6b, 0xdf, 0x9d, - 0x6a, 0x3a, 0xfb, 0xb7, 0xd4, 0x57, 0xf6, 0xaf, 0x99, 0x98, 0x5b, 0xde, 0x33, 0x31, 0xf7, 0xfb, - 0xf5, 0xcd, 0xac, 0x3c, 0x83, 0x77, 0x38, 0xef, 0x56, 0x56, 0x64, 0xc3, 0x60, 0xc3, 0x51, 0x15, - 0x5e, 0x46, 0xf8, 0xde, 0x61, 0x66, 0x9a, 0x21, 0x09, 0x88, 0xbd, 0x04, 0x35, 0x75, 0xfa, 0x21, - 0x37, 0xaa, 0x56, 0xfe, 0x46, 0xb5, 0xaf, 0x04, 0xc1, 0xfa, 0xda, 0x37, 0xbe, 0xf3, 0xc4, 0x3b, - 0x7e, 0xef, 0x3b, 0x4f, 0xbc, 0xe3, 0x8f, 0xbe, 0xf3, 0xc4, 0x3b, 0x3e, 0x7e, 0xf7, 0x09, 0xeb, - 0x1b, 0x77, 0x9f, 0xb0, 0x7e, 0xef, 0xee, 0x13, 0xd6, 0x1f, 0xdd, 0x7d, 0xc2, 0xfa, 0xf6, 0xdd, - 0x27, 0xac, 0x2f, 0xfc, 0xa7, 0x27, 0xde, 0xf1, 0x4a, 0x6e, 0xa0, 0x22, 0xfd, 0xf1, 0x6c, 0xa3, - 0x39, 0xb5, 0x75, 0x91, 0xc5, 0xca, 0xd1, 0xe5, 0x35, 0x65, 0xcc, 0xa9, 0x29, 0xb9, 0xbc, 0xfe, - 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2b, 0x8e, 0xba, 0x30, 0x2f, 0xe1, 0x00, 0x00, + 0x51, 0xf6, 0x21, 0xfb, 0x17, 0x2d, 0x18, 0x9d, 0xbe, 0xb5, 0x32, 0xdd, 0x49, 0x36, 0x66, 0x02, + 0x7f, 0xdd, 0x6d, 0xa1, 0x17, 0x60, 0xb8, 0xe1, 0x75, 0xe2, 0x84, 0x44, 0xd7, 0x9d, 0x36, 0x19, + 0xb7, 0x2e, 0x58, 0x4f, 0xd7, 0xea, 0xa7, 0xbf, 0xb1, 0x3b, 0xf1, 0x8e, 0xbb, 0xbb, 0x13, 0xc3, + 0x33, 0x1a, 0x84, 0x4d, 0x3c, 0xf4, 0x03, 0x30, 0x14, 0x05, 0x1e, 0x99, 0xc6, 0xd7, 0xc7, 0x4b, + 0xec, 0x91, 0x13, 0xe2, 0x91, 0x21, 0xcc, 0x9b, 0xb1, 0x84, 0x53, 0xd4, 0x30, 0x0a, 0xd6, 0x5d, + 0x8f, 0x8c, 0x97, 0xd3, 0xa8, 0xcb, 0xbc, 0x19, 0x4b, 0xb8, 0xfd, 0x87, 0x25, 0x80, 0xe9, 0x30, + 0x5c, 0x8e, 0x82, 0xdb, 0xa4, 0x91, 0xa0, 0x0f, 0x43, 0x95, 0x0e, 0x73, 0xd3, 0x49, 0x1c, 0xd6, + 0xb1, 0xe1, 0x8b, 0x3f, 0x38, 0xc9, 0xdf, 0x7a, 0xd2, 0x7c, 0x6b, 0x3d, 0xc9, 0x28, 0xf6, 0xe4, + 0xd6, 0x73, 0x93, 0x4b, 0x6b, 0xf4, 0xf9, 0x45, 0x92, 0x38, 0x75, 0x24, 0x98, 0x81, 0x6e, 0xc3, + 0x8a, 0x2a, 0xf2, 0x61, 0x20, 0x0e, 0x49, 0x83, 0xbd, 0xc3, 0xf0, 0xc5, 0x85, 0xc9, 0xa3, 0xcc, + 0xe6, 0x49, 0xdd, 0xf3, 0x95, 0x90, 0x34, 0xea, 0x23, 0x82, 0xf3, 0x00, 0xfd, 0x87, 0x19, 0x1f, + 0xb4, 0x05, 0x83, 0x71, 0xe2, 0x24, 0x9d, 0x98, 0x0d, 0xc5, 0xf0, 0xc5, 0xeb, 0x85, 0x71, 0x64, + 0x54, 0xeb, 0x63, 0x82, 0xe7, 0x20, 0xff, 0x8f, 0x05, 0x37, 0xfb, 0x4f, 0x2c, 0x18, 0xd3, 0xc8, + 0x0b, 0x6e, 0x9c, 0xa0, 0x9f, 0xe8, 0x1a, 0xdc, 0xc9, 0xfe, 0x06, 0x97, 0x3e, 0xcd, 0x86, 0xf6, + 0xa4, 0x60, 0x56, 0x95, 0x2d, 0xc6, 0xc0, 0xb6, 0xa1, 0xe2, 0x26, 0xa4, 0x1d, 0x8f, 0x97, 0x2e, + 0x94, 0x9f, 0x1e, 0xbe, 0x78, 0xa5, 0xa8, 0xf7, 0xac, 0x8f, 0x0a, 0xa6, 0x95, 0x79, 0x4a, 0x1e, + 0x73, 0x2e, 0xf6, 0xaf, 0x8e, 0x98, 0xef, 0x47, 0x07, 0x1c, 0x3d, 0x07, 0xc3, 0x71, 0xd0, 0x89, + 0x1a, 0x04, 0x93, 0x30, 0x88, 0xc7, 0xad, 0x0b, 0x65, 0x3a, 0xf5, 0xe8, 0xa4, 0x5e, 0xd1, 0xcd, + 0xd8, 0xc4, 0x41, 0x9f, 0xb7, 0x60, 0xa4, 0x49, 0xe2, 0xc4, 0xf5, 0x19, 0x7f, 0xd9, 0xf9, 0xd5, + 0x23, 0x77, 0x5e, 0x36, 0xce, 0x6a, 0xe2, 0xf5, 0x33, 0xe2, 0x45, 0x46, 0x8c, 0xc6, 0x18, 0xa7, + 0xf8, 0xd3, 0xc5, 0xd9, 0x24, 0x71, 0x23, 0x72, 0x43, 0xfa, 0x5f, 0x2c, 0x1f, 0xb5, 0x38, 0x67, + 0x35, 0x08, 0x9b, 0x78, 0xc8, 0x87, 0x0a, 0x5d, 0x7c, 0xf1, 0xf8, 0x00, 0xeb, 0xff, 0xfc, 0xd1, + 0xfa, 0x2f, 0x06, 0x95, 0xae, 0x6b, 0x3d, 0xfa, 0xf4, 0x5f, 0x8c, 0x39, 0x1b, 0xf4, 0x39, 0x0b, + 0xc6, 0x85, 0x70, 0xc0, 0x84, 0x0f, 0xe8, 0xad, 0x0d, 0x37, 0x21, 0x9e, 0x1b, 0x27, 0xe3, 0x15, + 0xd6, 0x87, 0xa9, 0xfe, 0xe6, 0xd6, 0x5c, 0x14, 0x74, 0xc2, 0x6b, 0xae, 0xdf, 0xac, 0x5f, 0x10, + 0x9c, 0xc6, 0x67, 0x7a, 0x10, 0xc6, 0x3d, 0x59, 0xa2, 0x2f, 0x59, 0x70, 0xde, 0x77, 0xda, 0x24, + 0x0e, 0x1d, 0xfa, 0x69, 0x39, 0xb8, 0xee, 0x39, 0x8d, 0x4d, 0xd6, 0xa3, 0xc1, 0xc3, 0xf5, 0xc8, + 0x16, 0x3d, 0x3a, 0x7f, 0xbd, 0x27, 0x69, 0xbc, 0x07, 0x5b, 0xf4, 0x35, 0x0b, 0x4e, 0x05, 0x51, + 0xb8, 0xe1, 0xf8, 0xa4, 0x29, 0xa1, 0xf1, 0xf8, 0x10, 0x5b, 0x7a, 0x1f, 0x3a, 0xda, 0x27, 0x5a, + 0xca, 0x92, 0x5d, 0x0c, 0x7c, 0x37, 0x09, 0xa2, 0x15, 0x92, 0x24, 0xae, 0xdf, 0x8a, 0xeb, 0x67, + 0xef, 0xee, 0x4e, 0x9c, 0xea, 0xc2, 0xc2, 0xdd, 0xfd, 0x41, 0x3f, 0x09, 0xc3, 0xf1, 0x8e, 0xdf, + 0xb8, 0xe5, 0xfa, 0xcd, 0xe0, 0x4e, 0x3c, 0x5e, 0x2d, 0x62, 0xf9, 0xae, 0x28, 0x82, 0x62, 0x01, + 0x6a, 0x06, 0xd8, 0xe4, 0x96, 0xff, 0xe1, 0xf4, 0x54, 0xaa, 0x15, 0xfd, 0xe1, 0xf4, 0x64, 0xda, + 0x83, 0x2d, 0xfa, 0x39, 0x0b, 0x46, 0x63, 0xb7, 0xe5, 0x3b, 0x49, 0x27, 0x22, 0xd7, 0xc8, 0x4e, + 0x3c, 0x0e, 0xac, 0x23, 0x57, 0x8f, 0x38, 0x2a, 0x06, 0xc9, 0xfa, 0x59, 0xd1, 0xc7, 0x51, 0xb3, + 0x35, 0xc6, 0x69, 0xbe, 0x79, 0x0b, 0x4d, 0x4f, 0xeb, 0xe1, 0x62, 0x17, 0x9a, 0x9e, 0xd4, 0x3d, + 0x59, 0xa2, 0x1f, 0x87, 0x93, 0xbc, 0x49, 0x8d, 0x6c, 0x3c, 0x3e, 0xc2, 0x04, 0xed, 0x99, 0xbb, + 0xbb, 0x13, 0x27, 0x57, 0x32, 0x30, 0xdc, 0x85, 0x8d, 0x5e, 0x87, 0x89, 0x90, 0x44, 0x6d, 0x37, + 0x59, 0xf2, 0xbd, 0x1d, 0x29, 0xbe, 0x1b, 0x41, 0x48, 0x9a, 0xa2, 0x3b, 0xf1, 0xf8, 0xe8, 0x05, + 0xeb, 0xe9, 0x6a, 0xfd, 0x5d, 0xa2, 0x9b, 0x13, 0xcb, 0x7b, 0xa3, 0xe3, 0xfd, 0xe8, 0xd9, 0xff, + 0xba, 0x04, 0x27, 0xb3, 0x8a, 0x13, 0xfd, 0x1d, 0x0b, 0x4e, 0xdc, 0xbe, 0x93, 0xac, 0x06, 0x9b, + 0xc4, 0x8f, 0xeb, 0x3b, 0x54, 0xbc, 0x31, 0x95, 0x31, 0x7c, 0xb1, 0x51, 0xac, 0x8a, 0x9e, 0xbc, + 0x9a, 0xe6, 0x72, 0xc9, 0x4f, 0xa2, 0x9d, 0xfa, 0xa3, 0xe2, 0xed, 0x4e, 0x5c, 0xbd, 0xb5, 0x6a, + 0x42, 0x71, 0xb6, 0x53, 0xe7, 0x3f, 0x63, 0xc1, 0x99, 0x3c, 0x12, 0xe8, 0x24, 0x94, 0x37, 0xc9, + 0x0e, 0x37, 0xe0, 0x30, 0xfd, 0x89, 0x5e, 0x85, 0xca, 0x96, 0xe3, 0x75, 0x88, 0xb0, 0x6e, 0xe6, + 0x8e, 0xf6, 0x22, 0xaa, 0x67, 0x98, 0x53, 0xfd, 0x91, 0xd2, 0x8b, 0x96, 0xfd, 0xbb, 0x65, 0x18, + 0x36, 0xf4, 0xdb, 0x7d, 0xb0, 0xd8, 0x82, 0x94, 0xc5, 0xb6, 0x58, 0x98, 0x6a, 0xee, 0x69, 0xb2, + 0xdd, 0xc9, 0x98, 0x6c, 0x4b, 0xc5, 0xb1, 0xdc, 0xd3, 0x66, 0x43, 0x09, 0xd4, 0x82, 0x90, 0x5a, + 0xef, 0x54, 0xf5, 0x0f, 0x14, 0xf1, 0x09, 0x97, 0x24, 0xb9, 0xfa, 0xe8, 0xdd, 0xdd, 0x89, 0x9a, + 0xfa, 0x8b, 0x35, 0x23, 0xfb, 0x5b, 0x16, 0x9c, 0x31, 0xfa, 0x38, 0x13, 0xf8, 0x4d, 0x97, 0x7d, + 0xda, 0x0b, 0x30, 0x90, 0xec, 0x84, 0x72, 0x87, 0xa0, 0x46, 0x6a, 0x75, 0x27, 0x24, 0x98, 0x41, + 0xa8, 0xa1, 0xdf, 0x26, 0x71, 0xec, 0xb4, 0x48, 0x76, 0x4f, 0xb0, 0xc8, 0x9b, 0xb1, 0x84, 0xa3, + 0x08, 0x90, 0xe7, 0xc4, 0xc9, 0x6a, 0xe4, 0xf8, 0x31, 0x23, 0xbf, 0xea, 0xb6, 0x89, 0x18, 0xe0, + 0xbf, 0xd8, 0xdf, 0x8c, 0xa1, 0x4f, 0xd4, 0x1f, 0xb9, 0xbb, 0x3b, 0x81, 0x16, 0xba, 0x28, 0xe1, + 0x1c, 0xea, 0xf6, 0x97, 0x2c, 0x78, 0x24, 0xdf, 0x16, 0x43, 0x4f, 0xc1, 0x20, 0xdf, 0x1e, 0x8a, + 0xb7, 0xd3, 0x9f, 0x84, 0xb5, 0x62, 0x01, 0x45, 0x53, 0x50, 0x53, 0x7a, 0x42, 0xbc, 0xe3, 0x29, + 0x81, 0x5a, 0xd3, 0xca, 0x45, 0xe3, 0xd0, 0x41, 0xa3, 0x7f, 0x84, 0xe5, 0xa6, 0x06, 0x8d, 0xed, + 0xa7, 0x18, 0xc4, 0xfe, 0x8f, 0x16, 0x9c, 0x30, 0x7a, 0x75, 0x1f, 0x4c, 0x73, 0x3f, 0x6d, 0x9a, + 0xcf, 0x17, 0x36, 0x9f, 0x7b, 0xd8, 0xe6, 0x9f, 0xb3, 0xe0, 0xbc, 0x81, 0xb5, 0xe8, 0x24, 0x8d, + 0x8d, 0x4b, 0xdb, 0x61, 0x44, 0x62, 0xba, 0xf5, 0x46, 0x8f, 0x1b, 0x72, 0xab, 0x3e, 0x2c, 0x28, + 0x94, 0xaf, 0x91, 0x1d, 0x2e, 0xc4, 0x9e, 0x81, 0x2a, 0x9f, 0x9c, 0x41, 0x24, 0x46, 0x5c, 0xbd, + 0xdb, 0x92, 0x68, 0xc7, 0x0a, 0x03, 0xd9, 0x30, 0xc8, 0x84, 0x13, 0x5d, 0xac, 0x54, 0x0d, 0x01, + 0xfd, 0x88, 0x37, 0x59, 0x0b, 0x16, 0x10, 0x3b, 0x4e, 0x75, 0x67, 0x39, 0x22, 0xec, 0xe3, 0x36, + 0x2f, 0xbb, 0xc4, 0x6b, 0xc6, 0x74, 0xdb, 0xe0, 0xf8, 0x7e, 0x90, 0x88, 0x1d, 0x80, 0xb1, 0x6d, + 0x98, 0xd6, 0xcd, 0xd8, 0xc4, 0xa1, 0x4c, 0x3d, 0x67, 0x8d, 0x78, 0x7c, 0x44, 0x05, 0xd3, 0x05, + 0xd6, 0x82, 0x05, 0xc4, 0xbe, 0x5b, 0x62, 0x1b, 0x14, 0xb5, 0xf4, 0xc9, 0xfd, 0xd8, 0xdd, 0x46, + 0x29, 0x59, 0xb9, 0x5c, 0x9c, 0xe0, 0x22, 0xbd, 0x77, 0xb8, 0x6f, 0x64, 0xc4, 0x25, 0x2e, 0x94, + 0xeb, 0xde, 0xbb, 0xdc, 0xdf, 0x2a, 0xc1, 0x44, 0xfa, 0x81, 0x2e, 0x69, 0x4b, 0xb7, 0x54, 0x06, + 0xa3, 0xac, 0xbf, 0xc3, 0xc0, 0xc7, 0x26, 0x5e, 0x0f, 0x81, 0x55, 0x3a, 0x4e, 0x81, 0x65, 0xca, + 0xd3, 0xf2, 0x3e, 0xf2, 0xf4, 0x29, 0x35, 0xea, 0x03, 0x19, 0x01, 0x96, 0xd6, 0x29, 0x17, 0x60, + 0x20, 0x4e, 0x48, 0x38, 0x5e, 0x49, 0xcb, 0xa3, 0x95, 0x84, 0x84, 0x98, 0x41, 0xec, 0xff, 0x56, + 0x82, 0x47, 0xd3, 0x63, 0xa8, 0x55, 0xc0, 0xfb, 0x52, 0x2a, 0xe0, 0xdd, 0xa6, 0x0a, 0xb8, 0xb7, + 0x3b, 0xf1, 0xce, 0x1e, 0x8f, 0x7d, 0xd7, 0x68, 0x08, 0x34, 0x97, 0x19, 0xc5, 0xa9, 0xf4, 0x28, + 0xde, 0xdb, 0x9d, 0x78, 0xbc, 0xc7, 0x3b, 0x66, 0x86, 0xf9, 0x29, 0x18, 0x8c, 0x88, 0x13, 0x07, + 0xbe, 0x18, 0x68, 0xf5, 0x39, 0x30, 0x6b, 0xc5, 0x02, 0x6a, 0xff, 0x7e, 0x2d, 0x3b, 0xd8, 0x73, + 0xdc, 0x61, 0x17, 0x44, 0xc8, 0x85, 0x01, 0x66, 0xd6, 0x73, 0xd1, 0x70, 0xed, 0x68, 0xcb, 0x88, + 0xaa, 0x01, 0x45, 0xba, 0x5e, 0xa5, 0x5f, 0x8d, 0x36, 0x61, 0xc6, 0x02, 0x6d, 0x43, 0xb5, 0x21, + 0xad, 0xed, 0x52, 0x11, 0x7e, 0x29, 0x61, 0x6b, 0x6b, 0x8e, 0x23, 0x54, 0x5e, 0x2b, 0x13, 0x5d, + 0x71, 0x43, 0x04, 0xca, 0x2d, 0x37, 0x11, 0x9f, 0xf5, 0x88, 0xfb, 0xa9, 0x39, 0xd7, 0x78, 0xc5, + 0x21, 0xaa, 0x44, 0xe6, 0xdc, 0x04, 0x53, 0xfa, 0xe8, 0x67, 0x2c, 0x18, 0x8e, 0x1b, 0xed, 0xe5, + 0x28, 0xd8, 0x72, 0x9b, 0x24, 0x12, 0xd6, 0xd4, 0x11, 0x45, 0xd3, 0xca, 0xcc, 0xa2, 0x24, 0xa8, + 0xf9, 0xf2, 0xfd, 0xad, 0x86, 0x60, 0x93, 0x2f, 0xdd, 0x65, 0x3c, 0x2a, 0xde, 0x7d, 0x96, 0x34, + 0x5c, 0xaa, 0xff, 0xe4, 0xa6, 0x8a, 0xcd, 0x94, 0x23, 0x5b, 0x97, 0xb3, 0x9d, 0xc6, 0x26, 0x5d, + 0x6f, 0xba, 0x43, 0xef, 0xbc, 0xbb, 0x3b, 0xf1, 0xe8, 0x4c, 0x3e, 0x4f, 0xdc, 0xab, 0x33, 0x6c, + 0xc0, 0xc2, 0x8e, 0xe7, 0x61, 0xf2, 0x7a, 0x87, 0x30, 0x97, 0x49, 0x01, 0x03, 0xb6, 0xac, 0x09, + 0x66, 0x06, 0xcc, 0x80, 0x60, 0x93, 0x2f, 0x7a, 0x1d, 0x06, 0xdb, 0x4e, 0x12, 0xb9, 0xdb, 0xc2, + 0x4f, 0x72, 0x44, 0x7b, 0x7f, 0x91, 0xd1, 0xd2, 0xcc, 0x99, 0xa6, 0xe6, 0x8d, 0x58, 0x30, 0x42, + 0x6d, 0xa8, 0xb4, 0x49, 0xd4, 0x22, 0xe3, 0xd5, 0x22, 0x7c, 0xc2, 0x8b, 0x94, 0x94, 0x66, 0x58, + 0xa3, 0xd6, 0x11, 0x6b, 0xc3, 0x9c, 0x0b, 0x7a, 0x15, 0xaa, 0x31, 0xf1, 0x48, 0x83, 0xda, 0x37, + 0x35, 0xc6, 0xf1, 0x87, 0xfa, 0xb4, 0xf5, 0xa8, 0x61, 0xb1, 0x22, 0x1e, 0xe5, 0x0b, 0x4c, 0xfe, + 0xc3, 0x8a, 0x24, 0x1d, 0xc0, 0xd0, 0xeb, 0xb4, 0x5c, 0x7f, 0x1c, 0x8a, 0x18, 0xc0, 0x65, 0x46, + 0x2b, 0x33, 0x80, 0xbc, 0x11, 0x0b, 0x46, 0xf6, 0x7f, 0xb6, 0x00, 0xa5, 0x85, 0xda, 0x7d, 0x30, + 0x6a, 0x5f, 0x4f, 0x1b, 0xb5, 0x0b, 0x45, 0x5a, 0x1d, 0x3d, 0xec, 0xda, 0xdf, 0xa8, 0x41, 0x46, + 0x1d, 0x5c, 0x27, 0x71, 0x42, 0x9a, 0x6f, 0x8b, 0xf0, 0xb7, 0x45, 0xf8, 0xdb, 0x22, 0x5c, 0x89, + 0xf0, 0xb5, 0x8c, 0x08, 0x7f, 0xaf, 0xb1, 0xea, 0xf5, 0x01, 0xec, 0x6b, 0xea, 0x84, 0xd6, 0xec, + 0x81, 0x81, 0x40, 0x25, 0xc1, 0xd5, 0x95, 0xa5, 0xeb, 0xb9, 0x32, 0xfb, 0xb5, 0xb4, 0xcc, 0x3e, + 0x2a, 0x8b, 0xff, 0x1f, 0xa4, 0xf4, 0xbf, 0xb2, 0xe0, 0x5d, 0x69, 0xe9, 0x25, 0x67, 0xce, 0x7c, + 0xcb, 0x0f, 0x22, 0x32, 0xeb, 0xae, 0xaf, 0x93, 0x88, 0xf8, 0x0d, 0x12, 0x2b, 0x2f, 0x86, 0xd5, + 0xcb, 0x8b, 0x81, 0x9e, 0x87, 0x91, 0xdb, 0x71, 0xe0, 0x2f, 0x07, 0xae, 0x2f, 0x44, 0x10, 0xdd, + 0x08, 0x9f, 0xbc, 0xbb, 0x3b, 0x31, 0x42, 0x47, 0x54, 0xb6, 0xe3, 0x14, 0x16, 0x9a, 0x81, 0x53, + 0xb7, 0x5f, 0x5f, 0x76, 0x12, 0xc3, 0x1d, 0x20, 0x37, 0xee, 0xec, 0xc0, 0xe2, 0xea, 0xcb, 0x19, + 0x20, 0xee, 0xc6, 0xb7, 0xff, 0x66, 0x09, 0xce, 0x65, 0x5e, 0x24, 0xf0, 0xbc, 0xa0, 0x93, 0xd0, + 0x4d, 0x0d, 0xfa, 0x8a, 0x05, 0x27, 0xdb, 0x69, 0x8f, 0x43, 0x2c, 0x1c, 0xbb, 0xef, 0x2f, 0x4c, + 0x47, 0x64, 0x5c, 0x1a, 0xf5, 0x71, 0x31, 0x42, 0x27, 0x33, 0x80, 0x18, 0x77, 0xf5, 0x05, 0xbd, + 0x0a, 0xb5, 0xb6, 0xb3, 0x7d, 0x23, 0x6c, 0x3a, 0x89, 0xdc, 0x4f, 0xf6, 0x76, 0x03, 0x74, 0x12, + 0xd7, 0x9b, 0xe4, 0x47, 0xfb, 0x93, 0xf3, 0x7e, 0xb2, 0x14, 0xad, 0x24, 0x91, 0xeb, 0xb7, 0xb8, + 0x3b, 0x6f, 0x51, 0x92, 0xc1, 0x9a, 0xa2, 0xfd, 0x65, 0x2b, 0xab, 0xa4, 0xd4, 0xe8, 0x44, 0x4e, + 0x42, 0x5a, 0x3b, 0xe8, 0x23, 0x50, 0xa1, 0x1b, 0x3f, 0x39, 0x2a, 0xb7, 0x8a, 0xd4, 0x9c, 0xc6, + 0x97, 0xd0, 0x4a, 0x94, 0xfe, 0x8b, 0x31, 0x67, 0x6a, 0x7f, 0xa5, 0x96, 0x35, 0x16, 0xd8, 0xe1, + 0xed, 0x45, 0x80, 0x56, 0xb0, 0x4a, 0xda, 0xa1, 0x47, 0x87, 0xc5, 0x62, 0x27, 0x00, 0xca, 0xd7, + 0x31, 0xa7, 0x20, 0xd8, 0xc0, 0x42, 0x7f, 0xd9, 0x02, 0x68, 0xc9, 0x39, 0x2f, 0x0d, 0x81, 0x1b, + 0x45, 0xbe, 0x8e, 0x5e, 0x51, 0xba, 0x2f, 0x8a, 0x21, 0x36, 0x98, 0xa3, 0x9f, 0xb6, 0xa0, 0x9a, + 0xc8, 0xee, 0x73, 0xd5, 0xb8, 0x5a, 0x64, 0x4f, 0xe4, 0x4b, 0x6b, 0x9b, 0x48, 0x0d, 0x89, 0xe2, + 0x8b, 0x7e, 0xd6, 0x02, 0x88, 0x77, 0xfc, 0xc6, 0x72, 0xe0, 0xb9, 0x8d, 0x1d, 0xa1, 0x31, 0x6f, + 0x16, 0xea, 0x8f, 0x51, 0xd4, 0xeb, 0x63, 0x74, 0x34, 0xf4, 0x7f, 0x6c, 0x70, 0x46, 0x1f, 0x83, + 0x6a, 0x2c, 0xa6, 0x9b, 0xd0, 0x91, 0xab, 0xc5, 0x7a, 0x85, 0x38, 0x6d, 0x21, 0x5e, 0xc5, 0x3f, + 0xac, 0x78, 0xa2, 0x9f, 0xb7, 0xe0, 0x44, 0x98, 0xf6, 0xf3, 0x09, 0x75, 0x58, 0x9c, 0x0c, 0xc8, + 0xf8, 0x11, 0xeb, 0xa7, 0xef, 0xee, 0x4e, 0x9c, 0xc8, 0x34, 0xe2, 0x6c, 0x2f, 0xa8, 0x04, 0xd4, + 0x33, 0x78, 0x29, 0xe4, 0x3e, 0xc7, 0x21, 0x2d, 0x01, 0xe7, 0xb2, 0x40, 0xdc, 0x8d, 0x8f, 0x96, + 0xe1, 0x0c, 0xed, 0xdd, 0x0e, 0x37, 0x3f, 0xa5, 0x7a, 0x89, 0x99, 0x32, 0xac, 0xd6, 0x1f, 0x13, + 0x33, 0x84, 0x79, 0xf5, 0xb3, 0x38, 0x38, 0xf7, 0x49, 0xf4, 0xbb, 0x16, 0x3c, 0xe6, 0x32, 0x35, + 0x60, 0x3a, 0xcc, 0xb5, 0x46, 0x10, 0x27, 0xb1, 0xa4, 0x50, 0x59, 0xd1, 0x4b, 0xfd, 0xd4, 0xff, + 0x82, 0x78, 0x83, 0xc7, 0xe6, 0xf7, 0xe8, 0x12, 0xde, 0xb3, 0xc3, 0xe8, 0x87, 0x61, 0x54, 0xae, + 0x8b, 0x65, 0x2a, 0x82, 0x99, 0xa2, 0xad, 0xd5, 0x4f, 0xdd, 0xdd, 0x9d, 0x18, 0x5d, 0x35, 0x01, + 0x38, 0x8d, 0x67, 0x7f, 0xb3, 0x94, 0x3a, 0x0f, 0x51, 0x4e, 0x48, 0x26, 0x6e, 0x1a, 0xd2, 0xff, + 0x23, 0xa5, 0x67, 0xa1, 0xe2, 0x46, 0x79, 0x97, 0xb4, 0xb8, 0x51, 0x4d, 0x31, 0x36, 0x98, 0x53, + 0xa3, 0xf4, 0x94, 0x93, 0x75, 0x75, 0x0a, 0x09, 0xf8, 0x6a, 0x91, 0x5d, 0xea, 0x3e, 0xbd, 0x3a, + 0x27, 0xba, 0x76, 0xaa, 0x0b, 0x84, 0xbb, 0xbb, 0x64, 0x7f, 0x33, 0x7d, 0x06, 0x63, 0x2c, 0xde, + 0x3e, 0xce, 0x97, 0x3e, 0x6f, 0xc1, 0x70, 0x14, 0x78, 0x9e, 0xeb, 0xb7, 0xa8, 0xa0, 0x11, 0xda, + 0xf2, 0x83, 0xc7, 0xa2, 0xb0, 0x84, 0x44, 0x61, 0xa6, 0x2d, 0xd6, 0x3c, 0xb1, 0xd9, 0x01, 0xfb, + 0x4f, 0x2c, 0x18, 0xef, 0x25, 0x10, 0x11, 0x81, 0x77, 0xca, 0xd5, 0xae, 0xa2, 0x2b, 0x96, 0xfc, + 0x59, 0xe2, 0x11, 0xe5, 0x78, 0xae, 0xd6, 0x9f, 0x14, 0xaf, 0xf9, 0xce, 0xe5, 0xde, 0xa8, 0x78, + 0x2f, 0x3a, 0xe8, 0x15, 0x38, 0x69, 0xbc, 0x57, 0xac, 0x06, 0xa6, 0x56, 0x9f, 0xa4, 0x16, 0xc8, + 0x74, 0x06, 0x76, 0x6f, 0x77, 0xe2, 0x91, 0x6c, 0x9b, 0x90, 0xd8, 0x5d, 0x74, 0xec, 0x5f, 0x29, + 0x65, 0xbf, 0x96, 0x52, 0xb6, 0x6f, 0x59, 0x5d, 0xdb, 0xf9, 0xf7, 0x1f, 0x87, 0x82, 0x63, 0x1b, + 0x7f, 0x15, 0xc0, 0xd1, 0x1b, 0xe7, 0x01, 0x9e, 0x10, 0xdb, 0xff, 0x66, 0x00, 0xf6, 0xe8, 0x59, + 0x1f, 0xd6, 0xf3, 0x81, 0x8f, 0x15, 0x3f, 0x6b, 0xa9, 0x23, 0xa7, 0x32, 0x5b, 0xe4, 0xcd, 0xe3, + 0x1a, 0x7b, 0xbe, 0x81, 0x89, 0x79, 0x94, 0x82, 0x72, 0x63, 0xa7, 0x0f, 0xb7, 0xd0, 0x57, 0xad, + 0xf4, 0xa1, 0x19, 0x0f, 0x3b, 0x73, 0x8f, 0xad, 0x4f, 0xc6, 0x49, 0x1c, 0xef, 0x98, 0x3e, 0xbf, + 0xe9, 0x75, 0x46, 0x37, 0x09, 0xb0, 0xee, 0xfa, 0x8e, 0xe7, 0xbe, 0x41, 0xb7, 0x27, 0x15, 0xa6, + 0x61, 0x99, 0xc9, 0x72, 0x59, 0xb5, 0x62, 0x03, 0xe3, 0xfc, 0x5f, 0x82, 0x61, 0xe3, 0xcd, 0x73, + 0x82, 0x2b, 0xce, 0x98, 0xc1, 0x15, 0x35, 0x23, 0x26, 0xe2, 0xfc, 0x7b, 0xe1, 0x64, 0xb6, 0x83, + 0x07, 0x79, 0xde, 0xfe, 0x5f, 0x43, 0xd9, 0x53, 0xac, 0x55, 0x12, 0xb5, 0x69, 0xd7, 0xde, 0xf6, + 0x2c, 0xbd, 0xed, 0x59, 0x7a, 0xdb, 0xb3, 0x64, 0x1e, 0x0e, 0x08, 0xaf, 0xc9, 0xd0, 0x7d, 0xf2, + 0x9a, 0xa4, 0xfc, 0x40, 0xd5, 0xc2, 0xfd, 0x40, 0xf6, 0xdd, 0x0a, 0xa4, 0xec, 0x28, 0x3e, 0xde, + 0x3f, 0x00, 0x43, 0x11, 0x09, 0x83, 0x1b, 0x78, 0x41, 0xe8, 0x10, 0x1d, 0x6b, 0xcf, 0x9b, 0xb1, + 0x84, 0x53, 0x5d, 0x13, 0x3a, 0xc9, 0x86, 0x50, 0x22, 0x4a, 0xd7, 0x2c, 0x3b, 0xc9, 0x06, 0x66, + 0x10, 0xf4, 0x5e, 0x18, 0x4b, 0x9c, 0xa8, 0x45, 0xed, 0xed, 0x2d, 0xf6, 0x59, 0xc5, 0x59, 0xe7, + 0x23, 0x02, 0x77, 0x6c, 0x35, 0x05, 0xc5, 0x19, 0x6c, 0xf4, 0x3a, 0x0c, 0x6c, 0x10, 0xaf, 0x2d, + 0x86, 0x7c, 0xa5, 0x38, 0x19, 0xcf, 0xde, 0xf5, 0x0a, 0xf1, 0xda, 0x5c, 0x02, 0xd1, 0x5f, 0x98, + 0xb1, 0xa2, 0xf3, 0xad, 0xb6, 0xd9, 0x89, 0x93, 0xa0, 0xed, 0xbe, 0x21, 0x5d, 0x7c, 0xef, 0x2f, + 0x98, 0xf1, 0x35, 0x49, 0x9f, 0xfb, 0x52, 0xd4, 0x5f, 0xac, 0x39, 0xb3, 0x7e, 0x34, 0xdd, 0x88, + 0x7d, 0xaa, 0x1d, 0xe1, 0xa9, 0x2b, 0xba, 0x1f, 0xb3, 0x92, 0x3e, 0xef, 0x87, 0xfa, 0x8b, 0x35, + 0x67, 0xb4, 0xa3, 0xe6, 0xfd, 0x30, 0xeb, 0xc3, 0x8d, 0x82, 0xfb, 0xc0, 0xe7, 0x7c, 0xee, 0xfc, + 0x7f, 0x12, 0x2a, 0x8d, 0x0d, 0x27, 0x4a, 0xc6, 0x47, 0xd8, 0xa4, 0x51, 0x3e, 0x9d, 0x19, 0xda, + 0x88, 0x39, 0x0c, 0x3d, 0x0e, 0xe5, 0x88, 0xac, 0xb3, 0xb8, 0x4d, 0x23, 0xa2, 0x07, 0x93, 0x75, + 0x4c, 0xdb, 0xed, 0x5f, 0x2a, 0xa5, 0xcd, 0xa5, 0xf4, 0x7b, 0xf3, 0xd9, 0xde, 0xe8, 0x44, 0xb1, + 0xf4, 0xfb, 0x18, 0xb3, 0x9d, 0x35, 0x63, 0x09, 0x47, 0x9f, 0xb0, 0x60, 0xe8, 0x76, 0x1c, 0xf8, + 0x3e, 0x49, 0x84, 0x6a, 0xba, 0x59, 0xf0, 0x50, 0x5c, 0xe5, 0xd4, 0x75, 0x1f, 0x44, 0x03, 0x96, + 0x7c, 0x69, 0x77, 0xc9, 0x76, 0xc3, 0xeb, 0x34, 0xbb, 0x82, 0x34, 0x2e, 0xf1, 0x66, 0x2c, 0xe1, + 0x14, 0xd5, 0xf5, 0x39, 0xea, 0x40, 0x1a, 0x75, 0xde, 0x17, 0xa8, 0x02, 0x6e, 0xff, 0xf5, 0x41, + 0x38, 0x9b, 0xbb, 0x38, 0xa8, 0x21, 0xc3, 0x4c, 0x85, 0xcb, 0xae, 0x47, 0x64, 0x78, 0x12, 0x33, + 0x64, 0x6e, 0xaa, 0x56, 0x6c, 0x60, 0xa0, 0x9f, 0x02, 0x08, 0x9d, 0xc8, 0x69, 0x13, 0xe5, 0x97, + 0x3d, 0xb2, 0xbd, 0x40, 0xfb, 0xb1, 0x2c, 0x69, 0xea, 0xbd, 0xa9, 0x6a, 0x8a, 0xb1, 0xc1, 0x12, + 0xbd, 0x00, 0xc3, 0x11, 0xf1, 0x88, 0x13, 0xb3, 0xb0, 0xdf, 0x6c, 0x0e, 0x03, 0xd6, 0x20, 0x6c, + 0xe2, 0xa1, 0xa7, 0x54, 0x24, 0x57, 0x26, 0xa2, 0x25, 0x1d, 0xcd, 0x85, 0xde, 0xb4, 0x60, 0x6c, + 0xdd, 0xf5, 0x88, 0xe6, 0x2e, 0x32, 0x0e, 0x96, 0x8e, 0xfe, 0x92, 0x97, 0x4d, 0xba, 0x5a, 0x42, + 0xa6, 0x9a, 0x63, 0x9c, 0x61, 0x4f, 0x3f, 0xf3, 0x16, 0x89, 0x98, 0x68, 0x1d, 0x4c, 0x7f, 0xe6, + 0x9b, 0xbc, 0x19, 0x4b, 0x38, 0x9a, 0x86, 0x13, 0xa1, 0x13, 0xc7, 0x33, 0x11, 0x69, 0x12, 0x3f, + 0x71, 0x1d, 0x8f, 0xe7, 0x03, 0x54, 0x75, 0x3c, 0xf0, 0x72, 0x1a, 0x8c, 0xb3, 0xf8, 0xe8, 0x03, + 0xf0, 0x28, 0x77, 0x7c, 0x2c, 0xba, 0x71, 0xec, 0xfa, 0x2d, 0x3d, 0x0d, 0x84, 0xff, 0x67, 0x42, + 0x90, 0x7a, 0x74, 0x3e, 0x1f, 0x0d, 0xf7, 0x7a, 0x1e, 0x3d, 0x03, 0xd5, 0x78, 0xd3, 0x0d, 0x67, + 0xa2, 0x66, 0xcc, 0x0e, 0x3d, 0xaa, 0xda, 0xdb, 0xb8, 0x22, 0xda, 0xb1, 0xc2, 0x40, 0x0d, 0x18, + 0xe1, 0x9f, 0x84, 0x87, 0xa2, 0x09, 0xf9, 0xf8, 0x6c, 0x4f, 0xf5, 0x28, 0xd2, 0xdb, 0x26, 0xb1, + 0x73, 0xe7, 0x92, 0x3c, 0x82, 0xe1, 0x27, 0x06, 0x37, 0x0d, 0x32, 0x38, 0x45, 0xd4, 0xfe, 0x85, + 0x52, 0x7a, 0xc7, 0x6d, 0x2e, 0x52, 0x14, 0xd3, 0xa5, 0x98, 0xdc, 0x74, 0x22, 0xe9, 0x8d, 0x39, + 0x62, 0xda, 0x82, 0xa0, 0x7b, 0xd3, 0x89, 0xcc, 0x45, 0xcd, 0x18, 0x60, 0xc9, 0x09, 0xdd, 0x86, + 0x81, 0xc4, 0x73, 0x0a, 0xca, 0x73, 0x32, 0x38, 0x6a, 0x07, 0xc8, 0xc2, 0x74, 0x8c, 0x19, 0x0f, + 0xf4, 0x18, 0xb5, 0xfa, 0xd7, 0xe4, 0x11, 0x89, 0x30, 0xd4, 0xd7, 0x62, 0xcc, 0x5a, 0xed, 0x7b, + 0x90, 0x23, 0x57, 0x95, 0x22, 0x43, 0x17, 0x01, 0xe8, 0x06, 0x72, 0x39, 0x22, 0xeb, 0xee, 0xb6, + 0x30, 0x24, 0xd4, 0xda, 0xbd, 0xae, 0x20, 0xd8, 0xc0, 0x92, 0xcf, 0xac, 0x74, 0xd6, 0xe9, 0x33, + 0xa5, 0xee, 0x67, 0x38, 0x04, 0x1b, 0x58, 0xe8, 0x79, 0x18, 0x74, 0xdb, 0x4e, 0x4b, 0x85, 0x60, + 0x3e, 0x46, 0x17, 0xed, 0x3c, 0x6b, 0xb9, 0xb7, 0x3b, 0x31, 0xa6, 0x3a, 0xc4, 0x9a, 0xb0, 0xc0, + 0x45, 0xbf, 0x62, 0xc1, 0x48, 0x23, 0x68, 0xb7, 0x03, 0x9f, 0x6f, 0xbb, 0xc4, 0x1e, 0xf2, 0xf6, + 0x71, 0xa9, 0xf9, 0xc9, 0x19, 0x83, 0x19, 0xdf, 0x44, 0xaa, 0x84, 0x2c, 0x13, 0x84, 0x53, 0xbd, + 0x32, 0xd7, 0x76, 0x65, 0x9f, 0xb5, 0xfd, 0xeb, 0x16, 0x9c, 0xe2, 0xcf, 0x1a, 0xbb, 0x41, 0x91, + 0x7b, 0x14, 0x1c, 0xf3, 0x6b, 0x75, 0x6d, 0x90, 0x95, 0x97, 0xae, 0x0b, 0x8e, 0xbb, 0x3b, 0x89, + 0xe6, 0xe0, 0xd4, 0x7a, 0x10, 0x35, 0x88, 0x39, 0x10, 0x42, 0x30, 0x29, 0x42, 0x97, 0xb3, 0x08, + 0xb8, 0xfb, 0x19, 0x74, 0x13, 0x1e, 0x31, 0x1a, 0xcd, 0x71, 0xe0, 0xb2, 0xe9, 0x09, 0x41, 0xed, + 0x91, 0xcb, 0xb9, 0x58, 0xb8, 0xc7, 0xd3, 0x69, 0x87, 0x49, 0xad, 0x0f, 0x87, 0xc9, 0x6b, 0x70, + 0xae, 0xd1, 0x3d, 0x32, 0x5b, 0x71, 0x67, 0x2d, 0xe6, 0x92, 0xaa, 0x5a, 0xff, 0x3e, 0x41, 0xe0, + 0xdc, 0x4c, 0x2f, 0x44, 0xdc, 0x9b, 0x06, 0xfa, 0x08, 0x54, 0x23, 0xc2, 0xbe, 0x4a, 0x2c, 0x12, + 0x71, 0x8e, 0xb8, 0x4b, 0xd6, 0x16, 0x28, 0x27, 0xab, 0x65, 0xaf, 0x68, 0x88, 0xb1, 0xe2, 0x88, + 0xee, 0xc0, 0x50, 0xe8, 0x24, 0x8d, 0x0d, 0x91, 0x7e, 0x73, 0xe4, 0xf8, 0x17, 0xc5, 0x9c, 0xf9, + 0xc0, 0x8d, 0x84, 0x5d, 0xce, 0x04, 0x4b, 0x6e, 0xd4, 0x1a, 0x69, 0x04, 0xed, 0x30, 0xf0, 0x89, + 0x9f, 0xc4, 0xe3, 0xa3, 0xda, 0x1a, 0x99, 0x51, 0xad, 0xd8, 0xc0, 0x40, 0xcb, 0x70, 0x86, 0xf9, + 0x8c, 0x6e, 0xb9, 0xc9, 0x46, 0xd0, 0x49, 0xe4, 0x16, 0x68, 0x7c, 0x2c, 0x7d, 0x54, 0xb1, 0x90, + 0x83, 0x83, 0x73, 0x9f, 0x3c, 0xff, 0x3e, 0x38, 0xd5, 0xb5, 0x94, 0x0f, 0xe4, 0xae, 0x99, 0x85, + 0x47, 0xf2, 0x17, 0xcd, 0x81, 0x9c, 0x36, 0xff, 0x38, 0x13, 0x36, 0x6b, 0x18, 0xd2, 0x7d, 0x38, + 0x00, 0x1d, 0x28, 0x13, 0x7f, 0x4b, 0xe8, 0x90, 0xcb, 0x47, 0xfb, 0x76, 0x97, 0xfc, 0x2d, 0xbe, + 0xe6, 0x99, 0x97, 0xe3, 0x92, 0xbf, 0x85, 0x29, 0x6d, 0xf4, 0x45, 0x2b, 0x65, 0x08, 0x72, 0xb7, + 0xe1, 0x87, 0x8e, 0x65, 0xe7, 0xd0, 0xb7, 0x6d, 0x68, 0xff, 0xdb, 0x12, 0x5c, 0xd8, 0x8f, 0x48, + 0x1f, 0xc3, 0xf7, 0x24, 0x0c, 0xc6, 0xec, 0x20, 0x5c, 0x08, 0xe5, 0x61, 0x3a, 0x57, 0xf9, 0xd1, + 0xf8, 0x6b, 0x58, 0x80, 0x90, 0x07, 0xe5, 0xb6, 0x13, 0x0a, 0x6f, 0xd2, 0xfc, 0x51, 0x13, 0x69, + 0xe8, 0x7f, 0xc7, 0x5b, 0x74, 0x42, 0xee, 0xa3, 0x30, 0x1a, 0x30, 0x65, 0x83, 0x12, 0xa8, 0x38, + 0x51, 0xe4, 0xc8, 0x53, 0xd7, 0x6b, 0xc5, 0xf0, 0x9b, 0xa6, 0x24, 0xf9, 0xa1, 0x55, 0xaa, 0x09, + 0x73, 0x66, 0xf6, 0x67, 0x87, 0x52, 0xc9, 0x24, 0xec, 0x28, 0x3d, 0x86, 0x41, 0xe1, 0x44, 0xb2, + 0x8a, 0xce, 0x5f, 0xe2, 0xd9, 0x80, 0x6c, 0x9f, 0x28, 0x72, 0xaa, 0x05, 0x2b, 0xf4, 0x19, 0x8b, + 0x65, 0x2e, 0xcb, 0x04, 0x1b, 0xb1, 0x3b, 0x3b, 0x9e, 0x44, 0x6a, 0x33, 0x1f, 0x5a, 0x36, 0x62, + 0x93, 0xbb, 0xa8, 0x40, 0xc0, 0xac, 0xd2, 0xee, 0x0a, 0x04, 0xcc, 0xca, 0x94, 0x70, 0xb4, 0x9d, + 0x73, 0x64, 0x5e, 0x40, 0xf6, 0x6b, 0x1f, 0x87, 0xe4, 0x5f, 0xb5, 0xe0, 0x94, 0x9b, 0x3d, 0xfb, + 0x14, 0x7b, 0x99, 0x23, 0x06, 0x65, 0xf4, 0x3e, 0x5a, 0x55, 0xea, 0xbc, 0x0b, 0x84, 0xbb, 0x3b, + 0x83, 0x9a, 0x30, 0xe0, 0xfa, 0xeb, 0x81, 0x30, 0x62, 0xea, 0x47, 0xeb, 0xd4, 0xbc, 0xbf, 0x1e, + 0xe8, 0xd5, 0x4c, 0xff, 0x61, 0x46, 0x1d, 0x2d, 0xc0, 0x99, 0x48, 0x78, 0x9b, 0xae, 0xb8, 0x71, + 0x12, 0x44, 0x3b, 0x0b, 0x6e, 0xdb, 0x4d, 0x98, 0x01, 0x52, 0xae, 0x8f, 0x53, 0xfd, 0x80, 0x73, + 0xe0, 0x38, 0xf7, 0x29, 0xf4, 0x06, 0x0c, 0xc9, 0x54, 0xeb, 0x6a, 0x11, 0xfb, 0xc2, 0xee, 0xf9, + 0xaf, 0x26, 0xd3, 0x8a, 0xc8, 0xaa, 0x96, 0x0c, 0xed, 0x37, 0x87, 0xa1, 0xfb, 0x58, 0x14, 0x7d, + 0x14, 0x6a, 0x91, 0x4a, 0xff, 0xb6, 0x8a, 0x50, 0xd7, 0xf2, 0xfb, 0x8a, 0x23, 0x59, 0x65, 0x0a, + 0xe9, 0x44, 0x6f, 0xcd, 0x91, 0x6e, 0x58, 0x62, 0x7d, 0x7a, 0x5a, 0xc0, 0xdc, 0x16, 0x5c, 0xf5, + 0xc9, 0xd8, 0x8e, 0xdf, 0xc0, 0x8c, 0x07, 0x8a, 0x60, 0x70, 0x83, 0x38, 0x5e, 0xb2, 0x51, 0x8c, + 0x13, 0xff, 0x0a, 0xa3, 0x95, 0x4d, 0x02, 0xe2, 0xad, 0x58, 0x70, 0x42, 0xdb, 0x30, 0xb4, 0xc1, + 0x27, 0x80, 0xd8, 0x43, 0x2c, 0x1e, 0x75, 0x70, 0x53, 0xb3, 0x4a, 0x7f, 0x6e, 0xd1, 0x80, 0x25, + 0x3b, 0x16, 0x6f, 0x63, 0x44, 0x04, 0xf0, 0xa5, 0x5b, 0x5c, 0xfe, 0x53, 0xff, 0xe1, 0x00, 0x1f, + 0x86, 0x91, 0x88, 0x34, 0x02, 0xbf, 0xe1, 0x7a, 0xa4, 0x39, 0x2d, 0x1d, 0xf4, 0x07, 0xc9, 0x9a, + 0x61, 0xfb, 0x70, 0x6c, 0xd0, 0xc0, 0x29, 0x8a, 0xe8, 0xd3, 0x16, 0x8c, 0xa9, 0x9c, 0x51, 0xfa, + 0x41, 0x88, 0x70, 0x08, 0x2f, 0x14, 0x94, 0xa1, 0xca, 0x68, 0xd6, 0xd1, 0xdd, 0xdd, 0x89, 0xb1, + 0x74, 0x1b, 0xce, 0xf0, 0x45, 0xaf, 0x00, 0x04, 0x6b, 0x3c, 0xa8, 0x66, 0x3a, 0x11, 0xde, 0xe1, + 0x83, 0xbc, 0xea, 0x18, 0x4f, 0x9f, 0x93, 0x14, 0xb0, 0x41, 0x0d, 0x5d, 0x03, 0xe0, 0xcb, 0x66, + 0x75, 0x27, 0x94, 0x1b, 0x0d, 0x99, 0xf6, 0x04, 0x2b, 0x0a, 0x72, 0x6f, 0x77, 0xa2, 0xdb, 0x5b, + 0xc7, 0x02, 0x17, 0x8c, 0xc7, 0xd1, 0x4f, 0xc2, 0x50, 0xdc, 0x69, 0xb7, 0x1d, 0xe5, 0x3b, 0x2e, + 0x30, 0x21, 0x8f, 0xd3, 0x35, 0x44, 0x11, 0x6f, 0xc0, 0x92, 0x23, 0xba, 0x4d, 0x85, 0x6a, 0x2c, + 0xdc, 0x88, 0x6c, 0x15, 0x71, 0x9b, 0x60, 0x98, 0xbd, 0xd3, 0x7b, 0xa4, 0xe1, 0x8d, 0x73, 0x70, + 0xee, 0xed, 0x4e, 0x3c, 0x92, 0x6e, 0x5f, 0x08, 0x44, 0x8a, 0x5c, 0x2e, 0x4d, 0x74, 0x55, 0x56, + 0x5e, 0xa1, 0xaf, 0x2d, 0x0b, 0x02, 0x3c, 0xad, 0x2b, 0xaf, 0xb0, 0xe6, 0xde, 0x63, 0x66, 0x3e, + 0x8c, 0x16, 0xe1, 0x74, 0x23, 0xf0, 0x93, 0x28, 0xf0, 0x3c, 0x5e, 0x79, 0x88, 0xef, 0xf9, 0xb8, + 0x6f, 0xf9, 0x9d, 0xa2, 0xdb, 0xa7, 0x67, 0xba, 0x51, 0x70, 0xde, 0x73, 0xb6, 0x9f, 0x8e, 0x36, + 0x14, 0x83, 0xf3, 0x3c, 0x8c, 0x90, 0xed, 0x84, 0x44, 0xbe, 0xe3, 0xdd, 0xc0, 0x0b, 0xd2, 0xab, + 0xca, 0xd6, 0xc0, 0x25, 0xa3, 0x1d, 0xa7, 0xb0, 0x90, 0xad, 0x1c, 0x1d, 0x46, 0xda, 0x27, 0x77, + 0x74, 0x48, 0xb7, 0x86, 0xfd, 0xbf, 0x4b, 0x29, 0x83, 0x6c, 0x35, 0x22, 0x04, 0x05, 0x50, 0xf1, + 0x83, 0xa6, 0x92, 0xfd, 0x57, 0x8b, 0x91, 0xfd, 0xd7, 0x83, 0xa6, 0x51, 0x9e, 0x85, 0xfe, 0x8b, + 0x31, 0xe7, 0xc3, 0xea, 0x57, 0xc8, 0x42, 0x1f, 0x0c, 0x20, 0x36, 0x1a, 0x45, 0x72, 0x56, 0xf5, + 0x2b, 0x96, 0x4c, 0x46, 0x38, 0xcd, 0x17, 0x6d, 0x42, 0x65, 0x23, 0x88, 0x13, 0xb9, 0xfd, 0x38, + 0xe2, 0x4e, 0xe7, 0x4a, 0x10, 0x27, 0xcc, 0x8a, 0x50, 0xaf, 0x4d, 0x5b, 0x62, 0xcc, 0x79, 0xd8, + 0xff, 0xc5, 0x4a, 0xf9, 0xd0, 0x6f, 0xb1, 0xc8, 0xdb, 0x2d, 0xe2, 0xd3, 0x65, 0x6d, 0x86, 0x1a, + 0xfd, 0x70, 0x26, 0x8f, 0xf1, 0x5d, 0xbd, 0x0a, 0x6b, 0xdd, 0xa1, 0x14, 0x26, 0x19, 0x09, 0x23, + 0x2a, 0xe9, 0xe3, 0x56, 0x3a, 0xa3, 0xb4, 0x54, 0xc4, 0x06, 0xc3, 0xcc, 0xaa, 0xde, 0x37, 0x39, + 0xd5, 0xfe, 0xa2, 0x05, 0x43, 0x75, 0xa7, 0xb1, 0x19, 0xac, 0xaf, 0xa3, 0x67, 0xa0, 0xda, 0xec, + 0x44, 0x66, 0x72, 0xab, 0x72, 0x1c, 0xcc, 0x8a, 0x76, 0xac, 0x30, 0xe8, 0x1c, 0x5e, 0x77, 0x1a, + 0x32, 0xb7, 0xba, 0xcc, 0xe7, 0xf0, 0x65, 0xd6, 0x82, 0x05, 0x04, 0xbd, 0x00, 0xc3, 0x6d, 0x67, + 0x5b, 0x3e, 0x9c, 0x75, 0xe0, 0x2f, 0x6a, 0x10, 0x36, 0xf1, 0xec, 0x7f, 0x69, 0xc1, 0x78, 0xdd, + 0x89, 0xdd, 0xc6, 0x74, 0x27, 0xd9, 0xa8, 0xbb, 0xc9, 0x5a, 0xa7, 0xb1, 0x49, 0x12, 0x9e, 0x50, + 0x4f, 0x7b, 0xd9, 0x89, 0xe9, 0x52, 0x52, 0xfb, 0x3a, 0xd5, 0xcb, 0x1b, 0xa2, 0x1d, 0x2b, 0x0c, + 0xf4, 0x06, 0x0c, 0x87, 0x4e, 0x1c, 0xdf, 0x09, 0xa2, 0x26, 0x26, 0xeb, 0xc5, 0x94, 0xb3, 0x58, + 0x21, 0x8d, 0x88, 0x24, 0x98, 0xac, 0x8b, 0x43, 0x66, 0x4d, 0x1f, 0x9b, 0xcc, 0xec, 0xcf, 0x5b, + 0x70, 0xae, 0x4e, 0x9c, 0x88, 0x44, 0xac, 0xfa, 0x85, 0x7a, 0x91, 0x19, 0x2f, 0xe8, 0x34, 0xd1, + 0xeb, 0x50, 0x4d, 0x68, 0x33, 0xed, 0x96, 0x55, 0x6c, 0xb7, 0xd8, 0x19, 0xf1, 0xaa, 0x20, 0x8e, + 0x15, 0x1b, 0xfb, 0x6f, 0x58, 0x30, 0xc2, 0x8e, 0xdb, 0x66, 0x49, 0xe2, 0xb8, 0x5e, 0x57, 0x91, + 0x28, 0xab, 0xcf, 0x22, 0x51, 0x17, 0x60, 0x60, 0x23, 0x68, 0x93, 0xec, 0x51, 0xf1, 0x95, 0x80, + 0x6e, 0xab, 0x29, 0x04, 0x3d, 0x47, 0x3f, 0xbc, 0xeb, 0x27, 0x0e, 0x5d, 0x02, 0xd2, 0x9d, 0x7b, + 0x82, 0x7f, 0x74, 0xd5, 0x8c, 0x4d, 0x1c, 0xfb, 0xb7, 0x6a, 0x30, 0x24, 0xe2, 0x09, 0xfa, 0x2e, + 0xaa, 0x20, 0xf7, 0xf7, 0xa5, 0x9e, 0xfb, 0xfb, 0x18, 0x06, 0x1b, 0xac, 0x5a, 0x9d, 0x30, 0x23, + 0xaf, 0x15, 0x12, 0x80, 0xc2, 0x0b, 0xe0, 0xe9, 0x6e, 0xf1, 0xff, 0x58, 0xb0, 0x42, 0x5f, 0xb0, + 0xe0, 0x44, 0x23, 0xf0, 0x7d, 0xd2, 0xd0, 0x36, 0xce, 0x40, 0x11, 0x71, 0x06, 0x33, 0x69, 0xa2, + 0xfa, 0xac, 0x27, 0x03, 0xc0, 0x59, 0xf6, 0xe8, 0x25, 0x18, 0xe5, 0x63, 0x76, 0x33, 0xe5, 0x83, + 0xd6, 0xb5, 0x83, 0x4c, 0x20, 0x4e, 0xe3, 0xa2, 0x49, 0xee, 0xcb, 0x17, 0x55, 0x7a, 0x06, 0xb5, + 0xab, 0xce, 0xa8, 0xcf, 0x63, 0x60, 0xa0, 0x08, 0x50, 0x44, 0xd6, 0x23, 0x12, 0x6f, 0x88, 0x78, + 0x0b, 0x66, 0x5f, 0x0d, 0x1d, 0x2e, 0x01, 0x1b, 0x77, 0x51, 0xc2, 0x39, 0xd4, 0xd1, 0xa6, 0xd8, + 0x60, 0x56, 0x8b, 0x90, 0xa1, 0xe2, 0x33, 0xf7, 0xdc, 0x67, 0x4e, 0x40, 0x25, 0xde, 0x70, 0xa2, + 0x26, 0xb3, 0xeb, 0xca, 0x3c, 0xe9, 0x67, 0x85, 0x36, 0x60, 0xde, 0x8e, 0x66, 0xe1, 0x64, 0xa6, + 0xf2, 0x51, 0x2c, 0x7c, 0xc5, 0x2a, 0xc1, 0x23, 0x53, 0x33, 0x29, 0xc6, 0x5d, 0x4f, 0x98, 0xce, + 0x87, 0xe1, 0x7d, 0x9c, 0x0f, 0x3b, 0x2a, 0xaa, 0x8f, 0x7b, 0x71, 0x5f, 0x2e, 0x64, 0x00, 0xfa, + 0x0a, 0xe1, 0xfb, 0x5c, 0x26, 0x84, 0x6f, 0x94, 0x75, 0xe0, 0x66, 0x31, 0x1d, 0x38, 0x78, 0xbc, + 0xde, 0x83, 0x8c, 0xbf, 0xfb, 0x73, 0x0b, 0xe4, 0x77, 0x9d, 0x71, 0x1a, 0x1b, 0x84, 0x4e, 0x19, + 0xf4, 0x5e, 0x18, 0x53, 0x5b, 0xe8, 0x99, 0xa0, 0xe3, 0xf3, 0xd0, 0xbb, 0xb2, 0x3e, 0x14, 0xc6, + 0x29, 0x28, 0xce, 0x60, 0xa3, 0x29, 0xa8, 0xd1, 0x71, 0xe2, 0x8f, 0x72, 0x5d, 0xab, 0xb6, 0xe9, + 0xd3, 0xcb, 0xf3, 0xe2, 0x29, 0x8d, 0x83, 0x02, 0x38, 0xe5, 0x39, 0x71, 0xc2, 0x7a, 0x40, 0x77, + 0xd4, 0x87, 0x2c, 0x7f, 0xc0, 0xb2, 0x08, 0x16, 0xb2, 0x84, 0x70, 0x37, 0x6d, 0xfb, 0x5b, 0x03, + 0x30, 0x9a, 0x92, 0x8c, 0x07, 0x54, 0xd2, 0xcf, 0x40, 0x55, 0xea, 0xcd, 0x6c, 0xa1, 0x16, 0xa5, + 0x5c, 0x15, 0x06, 0x55, 0x5a, 0x6b, 0x5a, 0xab, 0x66, 0x8d, 0x0a, 0x43, 0xe1, 0x62, 0x13, 0x8f, + 0x09, 0xe5, 0xc4, 0x8b, 0x67, 0x3c, 0x97, 0xf8, 0x09, 0xef, 0x66, 0x31, 0x42, 0x79, 0x75, 0x61, + 0xc5, 0x24, 0xaa, 0x85, 0x72, 0x06, 0x80, 0xb3, 0xec, 0xd1, 0xa7, 0x2c, 0x18, 0x75, 0xee, 0xc4, + 0xba, 0xa4, 0xaa, 0x08, 0xd6, 0x3b, 0xa2, 0x92, 0x4a, 0x55, 0x69, 0xe5, 0x2e, 0xdf, 0x54, 0x13, + 0x4e, 0x33, 0x45, 0x6f, 0x59, 0x80, 0xc8, 0x36, 0x69, 0xc8, 0x70, 0x42, 0xd1, 0x97, 0xc1, 0x22, + 0x76, 0x9a, 0x97, 0xba, 0xe8, 0x72, 0xa9, 0xde, 0xdd, 0x8e, 0x73, 0xfa, 0x60, 0xff, 0xb3, 0xb2, + 0x5a, 0x50, 0x3a, 0x82, 0xd5, 0x31, 0x22, 0xe9, 0xac, 0xc3, 0x47, 0xd2, 0xe9, 0x88, 0x84, 0xee, + 0xac, 0xca, 0x54, 0x12, 0x56, 0xe9, 0x01, 0x25, 0x61, 0xfd, 0xb4, 0x95, 0x2a, 0x49, 0x34, 0x7c, + 0xf1, 0x95, 0x62, 0xa3, 0x67, 0x27, 0x79, 0xb4, 0x44, 0x46, 0xba, 0xa7, 0x83, 0x64, 0xa8, 0x34, + 0x35, 0xd0, 0x0e, 0x24, 0x0d, 0xff, 0x7d, 0x19, 0x86, 0x0d, 0x4d, 0x9a, 0x6b, 0x16, 0x59, 0x0f, + 0x99, 0x59, 0x54, 0x3a, 0x80, 0x59, 0xf4, 0x53, 0x50, 0x6b, 0x48, 0x29, 0x5f, 0x4c, 0x51, 0xde, + 0xac, 0xee, 0xd0, 0x82, 0x5e, 0x35, 0x61, 0xcd, 0x13, 0xcd, 0xa5, 0x52, 0x77, 0x84, 0x86, 0x18, + 0x60, 0x1a, 0x22, 0x2f, 0xb7, 0x46, 0x68, 0x8a, 0xee, 0x67, 0x58, 0xe5, 0xaa, 0xd0, 0x15, 0xef, + 0x25, 0x63, 0xdc, 0x79, 0xe5, 0xaa, 0xe5, 0x79, 0xd9, 0x8c, 0x4d, 0x1c, 0xfb, 0x5b, 0x96, 0xfa, + 0xb8, 0xf7, 0xa1, 0x46, 0xc3, 0xed, 0x74, 0x8d, 0x86, 0x4b, 0x85, 0x0c, 0x73, 0x8f, 0xe2, 0x0c, + 0xd7, 0x61, 0x68, 0x26, 0x68, 0xb7, 0x1d, 0xbf, 0x89, 0xbe, 0x1f, 0x86, 0x1a, 0xfc, 0xa7, 0x70, + 0xec, 0xb0, 0xe3, 0x41, 0x01, 0xc5, 0x12, 0x86, 0x1e, 0x83, 0x01, 0x27, 0x6a, 0x49, 0x67, 0x0e, + 0x0b, 0xae, 0x99, 0x8e, 0x5a, 0x31, 0x66, 0xad, 0xf6, 0x3f, 0x1a, 0x00, 0x76, 0xa6, 0xed, 0x44, + 0xa4, 0xb9, 0x1a, 0xb0, 0xa2, 0x80, 0xc7, 0x7a, 0xa8, 0xa6, 0x37, 0x4b, 0x0f, 0xf3, 0xc1, 0x9a, + 0x71, 0xb8, 0x52, 0xbe, 0xcf, 0x87, 0x2b, 0x3d, 0xce, 0xcb, 0x06, 0x1e, 0xa2, 0xf3, 0x32, 0xfb, + 0xb3, 0x16, 0x20, 0x15, 0x08, 0xa1, 0x0f, 0xb4, 0xa7, 0xa0, 0xa6, 0x42, 0x22, 0x84, 0x61, 0xa5, + 0x45, 0x84, 0x04, 0x60, 0x8d, 0xd3, 0xc7, 0x0e, 0xf9, 0x49, 0x29, 0xbf, 0xcb, 0xe9, 0xb8, 0x5c, + 0x26, 0xf5, 0x85, 0x38, 0xb7, 0x7f, 0xbb, 0x04, 0x8f, 0x70, 0x95, 0xbc, 0xe8, 0xf8, 0x4e, 0x8b, + 0xb4, 0x69, 0xaf, 0xfa, 0x0d, 0x51, 0x68, 0xd0, 0xad, 0x99, 0x2b, 0xe3, 0x6c, 0x8f, 0xba, 0x76, + 0xf9, 0x9a, 0xe3, 0xab, 0x6c, 0xde, 0x77, 0x13, 0xcc, 0x88, 0xa3, 0x18, 0xaa, 0xb2, 0x62, 0xbd, + 0x90, 0xc5, 0x05, 0x31, 0x52, 0x62, 0x49, 0xe8, 0x4d, 0x82, 0x15, 0x23, 0x6a, 0xb8, 0x7a, 0x41, + 0x63, 0x13, 0x93, 0x30, 0x60, 0x72, 0xd7, 0x08, 0x73, 0x5c, 0x10, 0xed, 0x58, 0x61, 0xd8, 0xbf, + 0x6d, 0x41, 0x56, 0x23, 0x19, 0xd5, 0xd7, 0xac, 0x3d, 0xab, 0xaf, 0x1d, 0xa0, 0xfc, 0xd9, 0x4f, + 0xc0, 0xb0, 0x93, 0x50, 0x23, 0x82, 0x6f, 0xbb, 0xcb, 0x87, 0x3b, 0xd6, 0x58, 0x0c, 0x9a, 0xee, + 0xba, 0xcb, 0xb6, 0xdb, 0x26, 0x39, 0xfb, 0x7f, 0x0c, 0xc0, 0xa9, 0xae, 0x6c, 0x10, 0xf4, 0x22, + 0x8c, 0x34, 0xc4, 0xf4, 0x08, 0xa5, 0x43, 0xab, 0x66, 0x86, 0xc5, 0x69, 0x18, 0x4e, 0x61, 0xf6, + 0x31, 0x41, 0xe7, 0xe1, 0x74, 0x44, 0x37, 0xfa, 0x1d, 0x32, 0xbd, 0x9e, 0x90, 0x68, 0x85, 0x34, + 0x02, 0xbf, 0xc9, 0x6b, 0x04, 0x96, 0xeb, 0x8f, 0xde, 0xdd, 0x9d, 0x38, 0x8d, 0xbb, 0xc1, 0x38, + 0xef, 0x19, 0x14, 0xc2, 0xa8, 0x67, 0xda, 0x80, 0x62, 0x03, 0x70, 0x28, 0xf3, 0x51, 0xd9, 0x08, + 0xa9, 0x66, 0x9c, 0x66, 0x90, 0x36, 0x24, 0x2b, 0x0f, 0xc8, 0x90, 0xfc, 0xa4, 0x36, 0x24, 0xf9, + 0xf9, 0xfb, 0x07, 0x0b, 0xce, 0x06, 0x3a, 0x6e, 0x4b, 0xf2, 0x65, 0xa8, 0xca, 0xd8, 0xa4, 0xbe, + 0x62, 0x7a, 0x4c, 0x3a, 0x3d, 0x24, 0xda, 0xbd, 0x12, 0xe4, 0x6c, 0x42, 0xe8, 0x3a, 0xd3, 0x1a, + 0x3f, 0xb5, 0xce, 0x0e, 0xa6, 0xf5, 0xd1, 0x36, 0x8f, 0xcb, 0xe2, 0xba, 0xed, 0x03, 0x45, 0x6f, + 0xa2, 0x74, 0xa8, 0x96, 0x4a, 0x92, 0x50, 0xe1, 0x5a, 0x17, 0x01, 0xb4, 0xa1, 0x26, 0x42, 0xe0, + 0xd5, 0xb1, 0xaf, 0xb6, 0xe7, 0xb0, 0x81, 0x45, 0xf7, 0xd4, 0xae, 0x1f, 0x27, 0x8e, 0xe7, 0x5d, + 0x71, 0xfd, 0x44, 0x38, 0x07, 0x95, 0x12, 0x9f, 0xd7, 0x20, 0x6c, 0xe2, 0x9d, 0x7f, 0x8f, 0xf1, + 0x5d, 0x0e, 0xf2, 0x3d, 0x37, 0xe0, 0xdc, 0x9c, 0x9b, 0xa8, 0xc4, 0x0d, 0x35, 0x8f, 0xa8, 0x1d, + 0xa6, 0x12, 0x91, 0xac, 0x9e, 0x89, 0x48, 0x46, 0xe2, 0x44, 0x29, 0x9d, 0xe7, 0x91, 0x4d, 0x9c, + 0xb0, 0x5f, 0x84, 0x33, 0x73, 0x6e, 0x72, 0xd9, 0xf5, 0xc8, 0x01, 0x99, 0xd8, 0xbf, 0x39, 0x08, + 0x23, 0x66, 0xea, 0xdf, 0x41, 0x72, 0xa9, 0x3e, 0x4f, 0x4d, 0x2d, 0xf1, 0x76, 0xae, 0x3a, 0x34, + 0xbb, 0x75, 0xe4, 0x3c, 0xc4, 0xfc, 0x11, 0x33, 0xac, 0x2d, 0xcd, 0x13, 0x9b, 0x1d, 0x40, 0x77, + 0xa0, 0xb2, 0xce, 0x02, 0xfb, 0xcb, 0x45, 0x44, 0x16, 0xe4, 0x8d, 0xa8, 0x5e, 0x66, 0x3c, 0x35, + 0x80, 0xf3, 0xa3, 0x1a, 0x32, 0x4a, 0x67, 0x8b, 0x19, 0xc1, 0xa8, 0x22, 0x4f, 0x4c, 0x61, 0xf4, + 0x12, 0xf5, 0x95, 0x43, 0x88, 0xfa, 0x94, 0xe0, 0x1d, 0x7c, 0x40, 0x82, 0x97, 0x25, 0x69, 0x24, + 0x1b, 0xcc, 0x7e, 0x13, 0xd1, 0xf3, 0x43, 0x6c, 0x10, 0x8c, 0x24, 0x8d, 0x14, 0x18, 0x67, 0xf1, + 0xd1, 0xc7, 0x94, 0xe8, 0xae, 0x16, 0xe1, 0x57, 0x35, 0x67, 0xf4, 0x71, 0x4b, 0xed, 0xcf, 0x96, + 0x60, 0x6c, 0xce, 0xef, 0x2c, 0xcf, 0x2d, 0x77, 0xd6, 0x3c, 0xb7, 0x71, 0x8d, 0xec, 0x50, 0xd1, + 0xbc, 0x49, 0x76, 0xe6, 0x67, 0xc5, 0x0a, 0x52, 0x73, 0xe6, 0x1a, 0x6d, 0xc4, 0x1c, 0x46, 0x85, + 0xd1, 0xba, 0xeb, 0xb7, 0x48, 0x14, 0x46, 0xae, 0x70, 0x79, 0x1a, 0xc2, 0xe8, 0xb2, 0x06, 0x61, + 0x13, 0x8f, 0xd2, 0x0e, 0xee, 0xf8, 0x24, 0xca, 0x1a, 0xb2, 0x4b, 0xb4, 0x11, 0x73, 0x18, 0x45, + 0x4a, 0xa2, 0x4e, 0x9c, 0x88, 0xc9, 0xa8, 0x90, 0x56, 0x69, 0x23, 0xe6, 0x30, 0xba, 0xd2, 0xe3, + 0xce, 0x1a, 0x0b, 0xdc, 0xc8, 0x84, 0xea, 0xaf, 0xf0, 0x66, 0x2c, 0xe1, 0x14, 0x75, 0x93, 0xec, + 0xcc, 0xd2, 0x5d, 0x6f, 0x26, 0x63, 0xe7, 0x1a, 0x6f, 0xc6, 0x12, 0xce, 0x8a, 0x1b, 0xa6, 0x87, + 0xe3, 0xbb, 0xae, 0xb8, 0x61, 0xba, 0xfb, 0x3d, 0xf6, 0xcf, 0xbf, 0x6c, 0xc1, 0x88, 0x19, 0x6e, + 0x85, 0x5a, 0x19, 0x1b, 0x77, 0xa9, 0xab, 0x36, 0xee, 0x8f, 0xe5, 0x5d, 0x2c, 0xd6, 0x72, 0x93, + 0x20, 0x8c, 0x9f, 0x25, 0x7e, 0xcb, 0xf5, 0x09, 0x3b, 0x45, 0xe7, 0x61, 0x5a, 0xa9, 0x58, 0xae, + 0x99, 0xa0, 0x49, 0x0e, 0x61, 0x24, 0xdb, 0xb7, 0xe0, 0x54, 0x57, 0x9a, 0x56, 0x1f, 0xa6, 0xc5, + 0xbe, 0x49, 0xb2, 0x36, 0x86, 0x61, 0x4a, 0x58, 0x16, 0xd8, 0x99, 0x81, 0x53, 0x7c, 0x21, 0x51, + 0x4e, 0x2b, 0x8d, 0x0d, 0xd2, 0x56, 0xa9, 0x77, 0xcc, 0xbf, 0x7e, 0x33, 0x0b, 0xc4, 0xdd, 0xf8, + 0xf6, 0xe7, 0x2c, 0x18, 0x4d, 0x65, 0xce, 0x15, 0x64, 0x04, 0xb1, 0x95, 0x16, 0xb0, 0xe8, 0x3f, + 0x16, 0x02, 0x5d, 0x66, 0xca, 0x54, 0xaf, 0x34, 0x0d, 0xc2, 0x26, 0x9e, 0xfd, 0xc5, 0x12, 0x54, + 0x65, 0x04, 0x45, 0x1f, 0x5d, 0xf9, 0x8c, 0x05, 0xa3, 0xea, 0x4c, 0x83, 0x39, 0xcb, 0x4a, 0x45, + 0xa4, 0x39, 0xd0, 0x1e, 0xa8, 0xed, 0xb6, 0xbf, 0x1e, 0x68, 0x8b, 0x1c, 0x9b, 0xcc, 0x70, 0x9a, + 0x37, 0xba, 0x09, 0x10, 0xef, 0xc4, 0x09, 0x69, 0x1b, 0x6e, 0x3b, 0xdb, 0x58, 0x71, 0x93, 0x8d, + 0x20, 0x22, 0x74, 0x7d, 0x5d, 0x0f, 0x9a, 0x64, 0x45, 0x61, 0x6a, 0x13, 0x4a, 0xb7, 0x61, 0x83, + 0x92, 0xfd, 0x0f, 0x4a, 0x70, 0x32, 0xdb, 0x25, 0xf4, 0x41, 0x18, 0x91, 0xdc, 0x8d, 0x3b, 0xd2, + 0x64, 0xd8, 0xc8, 0x08, 0x36, 0x60, 0xf7, 0x76, 0x27, 0x26, 0xba, 0x2f, 0xa9, 0x9b, 0x34, 0x51, + 0x70, 0x8a, 0x18, 0x3f, 0x58, 0x12, 0x27, 0xa0, 0xf5, 0x9d, 0xe9, 0x30, 0x14, 0xa7, 0x43, 0xc6, + 0xc1, 0x92, 0x09, 0xc5, 0x19, 0x6c, 0xb4, 0x0c, 0x67, 0x8c, 0x96, 0xeb, 0xc4, 0x6d, 0x6d, 0xac, + 0x05, 0x91, 0xdc, 0x59, 0x3d, 0xa6, 0x03, 0xbb, 0xba, 0x71, 0x70, 0xee, 0x93, 0x54, 0xdb, 0x37, + 0x9c, 0xd0, 0x69, 0xb8, 0xc9, 0x8e, 0xf0, 0x43, 0x2a, 0xd9, 0x34, 0x23, 0xda, 0xb1, 0xc2, 0xb0, + 0x17, 0x61, 0xa0, 0xcf, 0x19, 0xd4, 0x97, 0x45, 0xff, 0x32, 0x54, 0x29, 0x39, 0x69, 0xde, 0x15, + 0x41, 0x32, 0x80, 0xaa, 0xbc, 0xbb, 0x04, 0xd9, 0x50, 0x76, 0x1d, 0x79, 0x76, 0xa7, 0x5e, 0x6b, + 0x3e, 0x8e, 0x3b, 0x6c, 0x93, 0x4c, 0x81, 0xe8, 0x49, 0x28, 0x93, 0xed, 0x30, 0x7b, 0x48, 0x77, + 0x69, 0x3b, 0x74, 0x23, 0x12, 0x53, 0x24, 0xb2, 0x1d, 0xa2, 0xf3, 0x50, 0x72, 0x9b, 0x42, 0x49, + 0x81, 0xc0, 0x29, 0xcd, 0xcf, 0xe2, 0x92, 0xdb, 0xb4, 0xb7, 0xa1, 0xa6, 0x2e, 0x4b, 0x41, 0x9b, + 0x52, 0x76, 0x5b, 0x45, 0x84, 0x3c, 0x49, 0xba, 0x3d, 0xa4, 0x76, 0x07, 0x40, 0xa7, 0x10, 0x16, + 0x25, 0x5f, 0x2e, 0xc0, 0x40, 0x23, 0x10, 0xe9, 0xcd, 0x55, 0x4d, 0x86, 0x09, 0x6d, 0x06, 0xb1, + 0x6f, 0xc1, 0xd8, 0x35, 0x3f, 0xb8, 0xc3, 0x2a, 0xbd, 0xb3, 0xc2, 0x66, 0x94, 0xf0, 0x3a, 0xfd, + 0x91, 0x35, 0x11, 0x18, 0x14, 0x73, 0x98, 0xaa, 0xf8, 0x54, 0xea, 0x55, 0xf1, 0xc9, 0xfe, 0xb8, + 0x05, 0x23, 0x2a, 0x17, 0x69, 0x6e, 0x6b, 0x93, 0xd2, 0x6d, 0x45, 0x41, 0x27, 0xcc, 0xd2, 0x65, + 0xd7, 0x19, 0x61, 0x0e, 0x33, 0x93, 0xf4, 0x4a, 0xfb, 0x24, 0xe9, 0x5d, 0x80, 0x81, 0x4d, 0xd7, + 0x6f, 0x66, 0xef, 0xe7, 0xb8, 0xe6, 0xfa, 0x4d, 0xcc, 0x20, 0xb4, 0x0b, 0x27, 0x55, 0x17, 0xa4, + 0x42, 0x78, 0x11, 0x46, 0xd6, 0x3a, 0xae, 0xd7, 0x94, 0x15, 0xdb, 0x32, 0x9e, 0x92, 0xba, 0x01, + 0xc3, 0x29, 0x4c, 0xba, 0xaf, 0x5b, 0x73, 0x7d, 0x27, 0xda, 0x59, 0xd6, 0x1a, 0x48, 0x09, 0xa5, + 0xba, 0x82, 0x60, 0x03, 0xcb, 0x7e, 0xb3, 0x0c, 0x63, 0xe9, 0x8c, 0xac, 0x3e, 0xb6, 0x57, 0x4f, + 0x42, 0x85, 0x25, 0x69, 0x65, 0x3f, 0x2d, 0x2f, 0x72, 0xc6, 0x61, 0x28, 0x86, 0x41, 0x5e, 0xde, + 0xa1, 0x98, 0xbb, 0x6d, 0x54, 0x27, 0x95, 0x7f, 0x85, 0xc5, 0x93, 0x89, 0x8a, 0x12, 0x82, 0x15, + 0xfa, 0x94, 0x05, 0x43, 0x41, 0x68, 0x56, 0x0a, 0xfa, 0x40, 0x91, 0xd9, 0x6a, 0x22, 0x59, 0x46, + 0x58, 0xc4, 0xea, 0xd3, 0xcb, 0xcf, 0x21, 0x59, 0x9f, 0xff, 0x11, 0x18, 0x31, 0x31, 0xf7, 0x33, + 0x8a, 0xab, 0xa6, 0x51, 0xfc, 0x19, 0x73, 0x52, 0x88, 0x7c, 0xbc, 0x3e, 0x96, 0xdb, 0x0d, 0xa8, + 0x34, 0x54, 0x00, 0xc0, 0xa1, 0xea, 0x7c, 0xaa, 0x7a, 0x0b, 0xec, 0x10, 0x88, 0x53, 0xb3, 0xbf, + 0x65, 0x19, 0xf3, 0x03, 0x93, 0x78, 0xbe, 0x89, 0x22, 0x28, 0xb7, 0xb6, 0x36, 0x85, 0x29, 0x7a, + 0xb5, 0xa0, 0xe1, 0x9d, 0xdb, 0xda, 0xd4, 0x73, 0xdc, 0x6c, 0xc5, 0x94, 0x59, 0x1f, 0x4e, 0xc0, + 0x54, 0xda, 0x66, 0x79, 0xff, 0xb4, 0x4d, 0xfb, 0xad, 0x12, 0x9c, 0xea, 0x9a, 0x54, 0xe8, 0x0d, + 0xa8, 0x44, 0xf4, 0x2d, 0xc5, 0xeb, 0x2d, 0x14, 0x96, 0x68, 0x19, 0xcf, 0x37, 0xb5, 0xde, 0x4d, + 0xb7, 0x63, 0xce, 0x12, 0x5d, 0x05, 0xa4, 0xc3, 0x54, 0x94, 0x07, 0x92, 0xbf, 0xf2, 0x79, 0xf1, + 0x28, 0x9a, 0xee, 0xc2, 0xc0, 0x39, 0x4f, 0xa1, 0x97, 0xb2, 0x8e, 0xcc, 0x72, 0xfa, 0xdc, 0x72, + 0x2f, 0x9f, 0xa4, 0xfd, 0xcf, 0x4b, 0x30, 0x9a, 0x2a, 0xdc, 0x84, 0x3c, 0xa8, 0x12, 0x8f, 0x39, + 0xf5, 0xa5, 0xb2, 0x39, 0x6a, 0x1d, 0x64, 0xa5, 0x20, 0x2f, 0x09, 0xba, 0x58, 0x71, 0x78, 0x38, + 0x0e, 0xd7, 0x5f, 0x84, 0x11, 0xd9, 0xa1, 0x0f, 0x38, 0x6d, 0x4f, 0x0c, 0xa0, 0x9a, 0xa3, 0x97, + 0x0c, 0x18, 0x4e, 0x61, 0xda, 0xbf, 0x53, 0x86, 0x71, 0x7e, 0x0a, 0xd2, 0x54, 0x33, 0x6f, 0x51, + 0xee, 0xb7, 0xfe, 0x8a, 0x2e, 0xaf, 0xc6, 0x07, 0x72, 0xed, 0xa8, 0xd7, 0x0e, 0xe4, 0x33, 0xea, + 0x2b, 0x32, 0xeb, 0x2b, 0x99, 0xc8, 0x2c, 0x6e, 0x76, 0xb7, 0x8e, 0xa9, 0x47, 0xdf, 0x5d, 0xa1, + 0x5a, 0x7f, 0xb7, 0x04, 0x27, 0x32, 0x77, 0x3a, 0xa0, 0x37, 0xd3, 0x65, 0x80, 0xad, 0x22, 0x7c, + 0xe5, 0x7b, 0x96, 0xf9, 0x3f, 0x58, 0x31, 0xe0, 0x07, 0xb4, 0x54, 0xec, 0x3f, 0x28, 0xc1, 0x58, + 0xfa, 0x32, 0x8a, 0x87, 0x70, 0xa4, 0xde, 0x0d, 0x35, 0x56, 0x6f, 0x9d, 0x5d, 0xb2, 0xc9, 0x5d, + 0xf2, 0xbc, 0xb4, 0xb5, 0x6c, 0xc4, 0x1a, 0xfe, 0x50, 0xd4, 0x58, 0xb6, 0xff, 0x9e, 0x05, 0x67, + 0xf9, 0x5b, 0x66, 0xe7, 0xe1, 0x5f, 0xcd, 0x1b, 0xdd, 0x57, 0x8b, 0xed, 0x60, 0xa6, 0x2c, 0xe0, + 0x7e, 0xe3, 0xcb, 0x2e, 0xf7, 0x13, 0xbd, 0x4d, 0x4f, 0x85, 0x87, 0xb0, 0xb3, 0x07, 0x9a, 0x0c, + 0xf6, 0x1f, 0x94, 0x41, 0xdf, 0x67, 0x88, 0x5c, 0x91, 0xe3, 0x58, 0x48, 0x79, 0xc4, 0x95, 0x1d, + 0xbf, 0xa1, 0x6f, 0x4e, 0xac, 0x66, 0x52, 0x1c, 0x7f, 0xce, 0x82, 0x61, 0xd7, 0x77, 0x13, 0xd7, + 0x61, 0xdb, 0xe8, 0x62, 0xee, 0x5a, 0x53, 0xec, 0xe6, 0x39, 0xe5, 0x20, 0x32, 0xcf, 0x71, 0x14, + 0x33, 0x6c, 0x72, 0x46, 0x1f, 0x16, 0xc1, 0xd3, 0xe5, 0xc2, 0xb2, 0x73, 0xab, 0x99, 0x88, 0xe9, + 0x90, 0x1a, 0x5e, 0x49, 0x54, 0x50, 0x52, 0x3b, 0xa6, 0xa4, 0x54, 0xa5, 0x5d, 0x7d, 0xb3, 0x34, + 0x6d, 0xc6, 0x9c, 0x91, 0x1d, 0x03, 0xea, 0x1e, 0x8b, 0x03, 0x06, 0xa6, 0x4e, 0x41, 0xcd, 0xe9, + 0x24, 0x41, 0x9b, 0x0e, 0x93, 0x38, 0x6a, 0xd2, 0xa1, 0xb7, 0x12, 0x80, 0x35, 0x8e, 0xfd, 0x66, + 0x05, 0x32, 0x49, 0x87, 0x68, 0xdb, 0xbc, 0x8b, 0xd3, 0x2a, 0xf6, 0x2e, 0x4e, 0xd5, 0x99, 0xbc, + 0xfb, 0x38, 0x51, 0x0b, 0x2a, 0xe1, 0x86, 0x13, 0x4b, 0xb3, 0xfa, 0x65, 0xb5, 0x8f, 0xa3, 0x8d, + 0xf7, 0x76, 0x27, 0x7e, 0xbc, 0x3f, 0xaf, 0x2b, 0x9d, 0xab, 0x53, 0xbc, 0x7c, 0x89, 0x66, 0xcd, + 0x68, 0x60, 0x4e, 0xff, 0x20, 0xb7, 0xcd, 0x7d, 0x42, 0x14, 0x96, 0xc7, 0x24, 0xee, 0x78, 0x89, + 0x98, 0x0d, 0x2f, 0x17, 0xb8, 0xca, 0x38, 0x61, 0x9d, 0x2e, 0xcf, 0xff, 0x63, 0x83, 0x29, 0xfa, + 0x20, 0xd4, 0xe2, 0xc4, 0x89, 0x92, 0x43, 0x26, 0xb8, 0xaa, 0x41, 0x5f, 0x91, 0x44, 0xb0, 0xa6, + 0x87, 0x5e, 0x61, 0xd5, 0x62, 0xdd, 0x78, 0xe3, 0x90, 0x39, 0x0f, 0xb2, 0xb2, 0xac, 0xa0, 0x80, + 0x0d, 0x6a, 0xe8, 0x22, 0x00, 0x9b, 0xdb, 0x3c, 0xd0, 0xaf, 0xca, 0xbc, 0x4c, 0x4a, 0x14, 0x62, + 0x05, 0xc1, 0x06, 0x96, 0xfd, 0x83, 0x90, 0xae, 0xf7, 0x80, 0x26, 0x64, 0x79, 0x09, 0xee, 0x85, + 0x66, 0xb9, 0x0b, 0xa9, 0x4a, 0x10, 0xbf, 0x6e, 0x81, 0x59, 0x94, 0x02, 0xbd, 0xce, 0xab, 0x5f, + 0x58, 0x45, 0x9c, 0x1c, 0x1a, 0x74, 0x27, 0x17, 0x9d, 0x30, 0x73, 0x84, 0x2d, 0x4b, 0x60, 0x9c, + 0x7f, 0x0f, 0x54, 0x25, 0xf4, 0x40, 0x46, 0xdd, 0xc7, 0xe0, 0x74, 0xf6, 0xa6, 0x72, 0x71, 0xea, + 0xb4, 0xbf, 0xeb, 0x47, 0xfa, 0x73, 0x4a, 0xbd, 0xfc, 0x39, 0x7d, 0xdc, 0xc8, 0xfa, 0x1b, 0x16, + 0x5c, 0xd8, 0xef, 0x42, 0x75, 0xf4, 0x18, 0x0c, 0xdc, 0x71, 0x22, 0x59, 0xc6, 0x9b, 0x09, 0xca, + 0x5b, 0x4e, 0xe4, 0x63, 0xd6, 0x8a, 0x76, 0x60, 0x90, 0x47, 0x83, 0x09, 0x6b, 0xfd, 0xe5, 0x62, + 0xaf, 0x77, 0xbf, 0x46, 0x8c, 0xed, 0x02, 0x8f, 0x44, 0xc3, 0x82, 0xa1, 0xfd, 0x6d, 0x0b, 0xd0, + 0xd2, 0x16, 0x89, 0x22, 0xb7, 0x69, 0xc4, 0xaf, 0xb1, 0x0b, 0x5a, 0x8c, 0x8b, 0x58, 0xcc, 0x14, + 0xd7, 0xcc, 0x05, 0x2d, 0xc6, 0xbf, 0xfc, 0x0b, 0x5a, 0x4a, 0x07, 0xbb, 0xa0, 0x05, 0x2d, 0xc1, + 0xd9, 0x36, 0xdf, 0x6e, 0xf0, 0x4b, 0x0f, 0xf8, 0xde, 0x43, 0x25, 0x94, 0x9d, 0xbb, 0xbb, 0x3b, + 0x71, 0x76, 0x31, 0x0f, 0x01, 0xe7, 0x3f, 0x67, 0xbf, 0x07, 0x10, 0x0f, 0x5b, 0x9b, 0xc9, 0x8b, + 0x41, 0xea, 0xe9, 0x7e, 0xb1, 0xbf, 0x5c, 0x81, 0x13, 0x99, 0x22, 0xaf, 0x74, 0xab, 0xd7, 0x1d, + 0xf4, 0x74, 0x64, 0xfd, 0xdd, 0xdd, 0xbd, 0xbe, 0xc2, 0xa8, 0x7c, 0xa8, 0xb8, 0x7e, 0xd8, 0x49, + 0x8a, 0xc9, 0x21, 0xe5, 0x9d, 0x98, 0xa7, 0x04, 0x0d, 0x77, 0x31, 0xfd, 0x8b, 0x39, 0x9b, 0x22, + 0x83, 0xb2, 0x52, 0xc6, 0xf8, 0xc0, 0x03, 0x72, 0x07, 0x7c, 0x42, 0x87, 0x48, 0x55, 0x8a, 0x70, + 0x2c, 0x66, 0x26, 0xcb, 0x71, 0x1f, 0xb5, 0xff, 0x5a, 0x09, 0x86, 0x8d, 0x8f, 0x86, 0x7e, 0x29, + 0x5d, 0xb2, 0xc9, 0x2a, 0xee, 0x95, 0x18, 0xfd, 0x49, 0x5d, 0x94, 0x89, 0xbf, 0xd2, 0x53, 0xdd, + 0xd5, 0x9a, 0xee, 0xed, 0x4e, 0x9c, 0xcc, 0xd4, 0x63, 0x4a, 0x55, 0x70, 0x3a, 0xff, 0x51, 0x38, + 0x91, 0x21, 0x93, 0xf3, 0xca, 0xab, 0xe9, 0x8b, 0xe8, 0x8f, 0xe8, 0x96, 0x32, 0x87, 0xec, 0xeb, + 0x74, 0xc8, 0x44, 0x1a, 0x5d, 0xe0, 0x91, 0x3e, 0x7c, 0xb0, 0x99, 0x6c, 0xd9, 0x52, 0x9f, 0xd9, + 0xb2, 0x4f, 0x43, 0x35, 0x0c, 0x3c, 0xb7, 0xe1, 0xaa, 0xba, 0x86, 0x2c, 0x3f, 0x77, 0x59, 0xb4, + 0x61, 0x05, 0x45, 0x77, 0xa0, 0xa6, 0xee, 0xec, 0x17, 0xfe, 0xed, 0xa2, 0x0e, 0x7d, 0x94, 0xd1, + 0xa2, 0xef, 0xe2, 0xd7, 0xbc, 0x90, 0x0d, 0x83, 0x4c, 0x09, 0xca, 0xd0, 0x7f, 0xe6, 0x7b, 0x67, + 0xda, 0x31, 0xc6, 0x02, 0x62, 0x7f, 0xad, 0x06, 0x67, 0xf2, 0x2a, 0x6d, 0xa3, 0x8f, 0xc0, 0x20, + 0xef, 0x63, 0x31, 0x97, 0x39, 0xe4, 0xf1, 0x98, 0x63, 0x04, 0x45, 0xb7, 0xd8, 0x6f, 0x2c, 0x78, + 0x0a, 0xee, 0x9e, 0xb3, 0x26, 0x66, 0xc8, 0xf1, 0x70, 0x5f, 0x70, 0x34, 0xf7, 0x05, 0x87, 0x73, + 0xf7, 0x9c, 0x35, 0xb4, 0x0d, 0x95, 0x96, 0x9b, 0x10, 0x47, 0x38, 0x11, 0x6e, 0x1d, 0x0b, 0x73, + 0xe2, 0x70, 0x2b, 0x8d, 0xfd, 0xc4, 0x9c, 0x21, 0xfa, 0xaa, 0x05, 0x27, 0xd6, 0xd2, 0xa9, 0xf1, + 0x42, 0x78, 0x3a, 0xc7, 0x50, 0x4d, 0x3d, 0xcd, 0x88, 0xdf, 0x50, 0x94, 0x69, 0xc4, 0xd9, 0xee, + 0xa0, 0x4f, 0x5a, 0x30, 0xb4, 0xee, 0x7a, 0x46, 0x61, 0xdd, 0x63, 0xf8, 0x38, 0x97, 0x19, 0x03, + 0xbd, 0xe3, 0xe0, 0xff, 0x63, 0x2c, 0x39, 0xf7, 0xd2, 0x54, 0x83, 0x47, 0xd5, 0x54, 0x43, 0x0f, + 0x48, 0x53, 0x7d, 0xda, 0x82, 0x9a, 0x1a, 0x69, 0x91, 0xee, 0xfc, 0xc1, 0x63, 0xfc, 0xe4, 0xdc, + 0x73, 0xa2, 0xfe, 0x62, 0xcd, 0x1c, 0x7d, 0xc1, 0x82, 0x61, 0xe7, 0x8d, 0x4e, 0x44, 0x9a, 0x64, + 0x2b, 0x08, 0x63, 0x71, 0xbd, 0xe1, 0xab, 0xc5, 0x77, 0x66, 0x9a, 0x32, 0x99, 0x25, 0x5b, 0x4b, + 0x61, 0x2c, 0xd2, 0x92, 0x74, 0x03, 0x36, 0xbb, 0x60, 0xef, 0x96, 0x60, 0x62, 0x1f, 0x0a, 0xe8, + 0x45, 0x18, 0x09, 0xa2, 0x96, 0xe3, 0xbb, 0x6f, 0x98, 0xb5, 0x2e, 0x94, 0x95, 0xb5, 0x64, 0xc0, + 0x70, 0x0a, 0xd3, 0x4c, 0xc8, 0x2e, 0xed, 0x93, 0x90, 0x7d, 0x01, 0x06, 0x22, 0x12, 0x06, 0xd9, + 0xcd, 0x02, 0x4b, 0x09, 0x60, 0x10, 0xf4, 0x38, 0x94, 0x9d, 0xd0, 0x15, 0x81, 0x68, 0x6a, 0x0f, + 0x34, 0xbd, 0x3c, 0x8f, 0x69, 0x7b, 0xaa, 0x3e, 0x44, 0xe5, 0xbe, 0xd4, 0x87, 0xa0, 0x6a, 0x40, + 0x9c, 0x5d, 0x0c, 0x6a, 0x35, 0x90, 0x3e, 0x53, 0xb0, 0xdf, 0x2a, 0xc3, 0xe3, 0x7b, 0xce, 0x17, + 0x1d, 0x87, 0x67, 0xed, 0x11, 0x87, 0x27, 0x87, 0xa7, 0xb4, 0xdf, 0xf0, 0x94, 0x7b, 0x0c, 0xcf, + 0x27, 0xe9, 0x32, 0x90, 0x35, 0x42, 0x8a, 0xb9, 0xa0, 0xae, 0x57, 0xc9, 0x11, 0xb1, 0x02, 0x24, + 0x14, 0x6b, 0xbe, 0x74, 0x0f, 0x90, 0x4a, 0x46, 0xae, 0x14, 0xa1, 0x06, 0x7a, 0xd6, 0x0c, 0xe1, + 0x73, 0xbf, 0x57, 0x86, 0xb3, 0xfd, 0xf3, 0x25, 0x78, 0xb2, 0x0f, 0xe9, 0x6d, 0xce, 0x62, 0xab, + 0xcf, 0x59, 0xfc, 0xdd, 0xfd, 0x99, 0xec, 0xbf, 0x66, 0xc1, 0xf9, 0xde, 0xca, 0x03, 0x3d, 0x07, + 0xc3, 0x6b, 0x91, 0xe3, 0x37, 0x36, 0xd8, 0xa5, 0x9b, 0x72, 0x50, 0xd8, 0x58, 0xeb, 0x66, 0x6c, + 0xe2, 0xd0, 0xed, 0x2d, 0x8f, 0x49, 0x30, 0x30, 0x64, 0xf2, 0x28, 0xdd, 0xde, 0xae, 0x66, 0x81, + 0xb8, 0x1b, 0xdf, 0xfe, 0xb3, 0x52, 0x7e, 0xb7, 0xb8, 0x91, 0x71, 0x90, 0xef, 0x24, 0xbe, 0x42, + 0xa9, 0x0f, 0x59, 0x52, 0xbe, 0xdf, 0xb2, 0x64, 0xa0, 0x97, 0x2c, 0x41, 0xb3, 0x70, 0xd2, 0xb8, + 0x94, 0x85, 0x27, 0x04, 0xf3, 0x80, 0x5b, 0x55, 0x25, 0x63, 0x39, 0x03, 0xc7, 0x5d, 0x4f, 0xa0, + 0x67, 0xa0, 0xea, 0xfa, 0x31, 0x69, 0x74, 0x22, 0x1e, 0xe8, 0x6d, 0x24, 0x61, 0xcd, 0x8b, 0x76, + 0xac, 0x30, 0xec, 0x5f, 0x2e, 0xc1, 0xb9, 0x9e, 0x76, 0xd6, 0x7d, 0x92, 0x5d, 0xe6, 0xe7, 0x18, + 0xb8, 0x3f, 0x9f, 0xc3, 0x1c, 0xa4, 0xca, 0xbe, 0x83, 0xf4, 0x87, 0xbd, 0x27, 0x26, 0xb5, 0xb9, + 0xbf, 0x67, 0x47, 0xe9, 0x25, 0x18, 0x75, 0xc2, 0x90, 0xe3, 0xb1, 0x78, 0xcd, 0x4c, 0x95, 0x9c, + 0x69, 0x13, 0x88, 0xd3, 0xb8, 0x7d, 0x69, 0xcf, 0x3f, 0xb6, 0xa0, 0x86, 0xc9, 0x3a, 0x97, 0x0e, + 0xe8, 0xb6, 0x18, 0x22, 0xab, 0x88, 0x7a, 0x9a, 0x74, 0x60, 0x63, 0x97, 0xd5, 0x99, 0xcc, 0x1b, + 0xec, 0xee, 0xcb, 0x7b, 0x4a, 0x07, 0xba, 0xbc, 0x47, 0x5d, 0xdf, 0x52, 0xee, 0x7d, 0x7d, 0x8b, + 0xfd, 0xf5, 0x21, 0xfa, 0x7a, 0x61, 0x30, 0x13, 0x91, 0x66, 0x4c, 0xbf, 0x6f, 0x27, 0xf2, 0xc4, + 0x24, 0x51, 0xdf, 0xf7, 0x06, 0x5e, 0xc0, 0xb4, 0x3d, 0x75, 0x14, 0x53, 0x3a, 0x50, 0x8d, 0x90, + 0xf2, 0xbe, 0x35, 0x42, 0x5e, 0x82, 0xd1, 0x38, 0xde, 0x58, 0x8e, 0xdc, 0x2d, 0x27, 0x21, 0xd7, + 0xc8, 0x8e, 0xb0, 0xb2, 0x74, 0x5e, 0xff, 0xca, 0x15, 0x0d, 0xc4, 0x69, 0x5c, 0x34, 0x07, 0xa7, + 0x74, 0xa5, 0x0e, 0x12, 0x25, 0x2c, 0xba, 0x9f, 0xcf, 0x04, 0x95, 0xc4, 0xab, 0x6b, 0x7b, 0x08, + 0x04, 0xdc, 0xfd, 0x0c, 0x95, 0x6f, 0xa9, 0x46, 0xda, 0x91, 0xc1, 0xb4, 0x7c, 0x4b, 0xd1, 0xa1, + 0x7d, 0xe9, 0x7a, 0x02, 0x2d, 0xc2, 0x69, 0x3e, 0x31, 0xa6, 0xc3, 0xd0, 0x78, 0xa3, 0xa1, 0x74, + 0x1d, 0xc3, 0xb9, 0x6e, 0x14, 0x9c, 0xf7, 0x1c, 0x7a, 0x01, 0x86, 0x55, 0xf3, 0xfc, 0xac, 0x38, + 0x45, 0x50, 0x5e, 0x0c, 0x45, 0x66, 0xbe, 0x89, 0x4d, 0x3c, 0xf4, 0x01, 0x78, 0x54, 0xff, 0xe5, + 0x29, 0x60, 0xfc, 0x68, 0x6d, 0x56, 0x14, 0x41, 0x52, 0x97, 0x85, 0xcc, 0xe5, 0xa2, 0x35, 0x71, + 0xaf, 0xe7, 0xd1, 0x1a, 0x9c, 0x57, 0xa0, 0x4b, 0x7e, 0xc2, 0xf2, 0x39, 0x62, 0x52, 0x77, 0x62, + 0x72, 0x23, 0xf2, 0xc4, 0x6d, 0xab, 0xea, 0x1e, 0xc7, 0x39, 0x37, 0xb9, 0x92, 0x87, 0x89, 0x17, + 0xf0, 0x1e, 0x54, 0xd0, 0x14, 0xd4, 0x88, 0xef, 0xac, 0x79, 0x64, 0x69, 0x66, 0x9e, 0x15, 0x53, + 0x32, 0x4e, 0xf2, 0x2e, 0x49, 0x00, 0xd6, 0x38, 0x2a, 0xc2, 0x74, 0xa4, 0xe7, 0x9d, 0xa2, 0xcb, + 0x70, 0xa6, 0xd5, 0x08, 0xa9, 0xed, 0xe1, 0x36, 0xc8, 0x74, 0x83, 0x05, 0xd4, 0xd1, 0x0f, 0xc3, + 0x0b, 0x4c, 0xaa, 0xf0, 0xe9, 0xb9, 0x99, 0xe5, 0x2e, 0x1c, 0x9c, 0xfb, 0x24, 0x0b, 0xbc, 0x8c, + 0x82, 0xed, 0x9d, 0xf1, 0xd3, 0x99, 0xc0, 0x4b, 0xda, 0x88, 0x39, 0x0c, 0x5d, 0x05, 0xc4, 0x62, + 0xf1, 0xaf, 0x24, 0x49, 0xa8, 0x8c, 0x9d, 0xf1, 0x33, 0xec, 0x95, 0x54, 0x18, 0xd9, 0xe5, 0x2e, + 0x0c, 0x9c, 0xf3, 0x94, 0xfd, 0x1f, 0x2c, 0x18, 0x55, 0xeb, 0xf5, 0x3e, 0x64, 0xa3, 0x78, 0xe9, + 0x6c, 0x94, 0xb9, 0xa3, 0x4b, 0x3c, 0xd6, 0xf3, 0x1e, 0x21, 0xcd, 0x3f, 0x33, 0x0c, 0xa0, 0xa5, + 0xa2, 0x52, 0x48, 0x56, 0x4f, 0x85, 0xf4, 0xd0, 0x4a, 0xa4, 0xbc, 0xca, 0x29, 0x95, 0x07, 0x5b, + 0x39, 0x65, 0x05, 0xce, 0x4a, 0x73, 0x81, 0x9f, 0x15, 0x5d, 0x09, 0x62, 0x25, 0xe0, 0xaa, 0xf5, + 0xc7, 0x05, 0xa1, 0xb3, 0xf3, 0x79, 0x48, 0x38, 0xff, 0xd9, 0x94, 0x95, 0x32, 0xb4, 0x9f, 0x95, + 0xa2, 0xd7, 0xf4, 0xc2, 0xba, 0xbc, 0x15, 0x24, 0xb3, 0xa6, 0x17, 0x2e, 0xaf, 0x60, 0x8d, 0x93, + 0x2f, 0xd8, 0x6b, 0x05, 0x09, 0x76, 0x38, 0xb0, 0x60, 0x97, 0x22, 0x66, 0xb8, 0xa7, 0x88, 0x91, + 0x3e, 0xe9, 0x91, 0x9e, 0x3e, 0xe9, 0xf7, 0xc2, 0x98, 0xeb, 0x6f, 0x90, 0xc8, 0x4d, 0x48, 0x93, + 0xad, 0x05, 0x26, 0x7e, 0xaa, 0x5a, 0xad, 0xcf, 0xa7, 0xa0, 0x38, 0x83, 0x9d, 0x96, 0x8b, 0x63, + 0x7d, 0xc8, 0xc5, 0x1e, 0xda, 0xe8, 0x44, 0x31, 0xda, 0xe8, 0xe4, 0xd1, 0xb5, 0xd1, 0xa9, 0x63, + 0xd5, 0x46, 0xa8, 0x10, 0x6d, 0xd4, 0x97, 0xa0, 0x37, 0xb6, 0x7f, 0x67, 0xf6, 0xd9, 0xfe, 0xf5, + 0x52, 0x45, 0x67, 0x0f, 0xad, 0x8a, 0xf2, 0xb5, 0xcc, 0x23, 0x87, 0xd2, 0x32, 0x9f, 0x2e, 0xc1, + 0x59, 0x2d, 0x87, 0xe9, 0xec, 0x77, 0xd7, 0xa9, 0x24, 0x62, 0x17, 0x4b, 0xf1, 0x73, 0x1b, 0x23, + 0x39, 0x4a, 0xe7, 0x59, 0x29, 0x08, 0x36, 0xb0, 0x58, 0x8e, 0x11, 0x89, 0x58, 0x19, 0xdd, 0xac, + 0x90, 0x9e, 0x11, 0xed, 0x58, 0x61, 0xd0, 0xf9, 0x45, 0x7f, 0x8b, 0xbc, 0xcd, 0x6c, 0xb1, 0xb8, + 0x19, 0x0d, 0xc2, 0x26, 0x1e, 0x7a, 0x9a, 0x33, 0x61, 0x02, 0x82, 0x0a, 0xea, 0x11, 0x71, 0xd3, + 0xac, 0x94, 0x09, 0x0a, 0x2a, 0xbb, 0xc3, 0x92, 0xc9, 0x2a, 0xdd, 0xdd, 0x61, 0x21, 0x50, 0x0a, + 0xc3, 0xfe, 0x9f, 0x16, 0x9c, 0xcb, 0x1d, 0x8a, 0xfb, 0xa0, 0x7c, 0xb7, 0xd3, 0xca, 0x77, 0xa5, + 0xa8, 0xed, 0x86, 0xf1, 0x16, 0x3d, 0x14, 0xf1, 0xbf, 0xb3, 0x60, 0x4c, 0xe3, 0xdf, 0x87, 0x57, + 0x75, 0xd3, 0xaf, 0x5a, 0xdc, 0xce, 0xaa, 0xd6, 0xf5, 0x6e, 0xbf, 0x53, 0x02, 0x55, 0xc0, 0x71, + 0xba, 0x21, 0xcb, 0xe3, 0xee, 0x73, 0x92, 0xb8, 0x03, 0x83, 0xec, 0x20, 0x34, 0x2e, 0x26, 0xc8, + 0x23, 0xcd, 0x9f, 0x1d, 0xaa, 0xea, 0x43, 0x66, 0xf6, 0x37, 0xc6, 0x82, 0x21, 0x2b, 0xf2, 0xec, + 0xc6, 0x54, 0x9a, 0x37, 0x45, 0x5a, 0x96, 0x2e, 0xf2, 0x2c, 0xda, 0xb1, 0xc2, 0xa0, 0xea, 0xc1, + 0x6d, 0x04, 0xfe, 0x8c, 0xe7, 0xc4, 0xf2, 0x36, 0x45, 0xa5, 0x1e, 0xe6, 0x25, 0x00, 0x6b, 0x1c, + 0x76, 0x46, 0xea, 0xc6, 0xa1, 0xe7, 0xec, 0x18, 0xfb, 0x67, 0xa3, 0x3e, 0x81, 0x02, 0x61, 0x13, + 0xcf, 0x6e, 0xc3, 0x78, 0xfa, 0x25, 0x66, 0xc9, 0x3a, 0x0b, 0x50, 0xec, 0x6b, 0x38, 0xa7, 0xa0, + 0xe6, 0xb0, 0xa7, 0x16, 0x3a, 0x4e, 0xf6, 0x12, 0xf4, 0x69, 0x09, 0xc0, 0x1a, 0xc7, 0xfe, 0x55, + 0x0b, 0x4e, 0xe7, 0x0c, 0x5a, 0x81, 0x69, 0x6f, 0x89, 0x96, 0x36, 0x79, 0x8a, 0xfd, 0x07, 0x60, + 0xa8, 0x49, 0xd6, 0x1d, 0x19, 0x02, 0x67, 0xc8, 0xf6, 0x59, 0xde, 0x8c, 0x25, 0xdc, 0xfe, 0xef, + 0x16, 0x9c, 0x48, 0xf7, 0x35, 0x66, 0xa9, 0x24, 0x7c, 0x98, 0xdc, 0xb8, 0x11, 0x6c, 0x91, 0x68, + 0x87, 0xbe, 0xb9, 0x95, 0x49, 0x25, 0xe9, 0xc2, 0xc0, 0x39, 0x4f, 0xb1, 0xf2, 0xad, 0x4d, 0x35, + 0xda, 0x72, 0x46, 0xde, 0x2c, 0x72, 0x46, 0xea, 0x8f, 0x69, 0x1e, 0x97, 0x2b, 0x96, 0xd8, 0xe4, + 0x6f, 0x7f, 0x7b, 0x00, 0x54, 0x5e, 0x2c, 0x8b, 0x3f, 0x2a, 0x28, 0x7a, 0xeb, 0xa0, 0x19, 0x44, + 0x6a, 0x32, 0x0c, 0xec, 0x15, 0x10, 0xc0, 0xbd, 0x24, 0xa6, 0xeb, 0x52, 0xbd, 0xe1, 0xaa, 0x06, + 0x61, 0x13, 0x8f, 0xf6, 0xc4, 0x73, 0xb7, 0x08, 0x7f, 0x68, 0x30, 0xdd, 0x93, 0x05, 0x09, 0xc0, + 0x1a, 0x87, 0xf6, 0xa4, 0xe9, 0xae, 0xaf, 0x8b, 0x2d, 0xbf, 0xea, 0x09, 0x1d, 0x1d, 0xcc, 0x20, + 0xbc, 0x22, 0x77, 0xb0, 0x29, 0xac, 0x60, 0xa3, 0x22, 0x77, 0xb0, 0x89, 0x19, 0x84, 0xda, 0x6d, + 0x7e, 0x10, 0xb5, 0xd9, 0x25, 0xf5, 0x4d, 0xc5, 0x45, 0x58, 0xbf, 0xca, 0x6e, 0xbb, 0xde, 0x8d, + 0x82, 0xf3, 0x9e, 0xa3, 0x33, 0x30, 0x8c, 0x48, 0xd3, 0x6d, 0x24, 0x26, 0x35, 0x48, 0xcf, 0xc0, + 0xe5, 0x2e, 0x0c, 0x9c, 0xf3, 0x14, 0x9a, 0x86, 0x13, 0x32, 0xaf, 0x59, 0x56, 0xad, 0x19, 0x4e, + 0x57, 0xc9, 0xc0, 0x69, 0x30, 0xce, 0xe2, 0x53, 0xa9, 0xd6, 0x16, 0x05, 0xab, 0x98, 0xb1, 0x6c, + 0x48, 0x35, 0x59, 0xc8, 0x0a, 0x2b, 0x0c, 0xfb, 0x13, 0x65, 0xaa, 0x85, 0x7b, 0x14, 0x6a, 0xbb, + 0x6f, 0xd1, 0x82, 0xe9, 0x19, 0x39, 0xd0, 0xc7, 0x8c, 0x7c, 0x1e, 0x46, 0x6e, 0xc7, 0x81, 0xaf, + 0x22, 0xf1, 0x2a, 0x3d, 0x23, 0xf1, 0x0c, 0xac, 0xfc, 0x48, 0xbc, 0xc1, 0xa2, 0x22, 0xf1, 0x86, + 0x0e, 0x19, 0x89, 0xf7, 0xcd, 0x0a, 0xa8, 0xab, 0x41, 0xae, 0x93, 0xe4, 0x4e, 0x10, 0x6d, 0xba, + 0x7e, 0x8b, 0xe5, 0x83, 0x7f, 0xd5, 0x82, 0x11, 0xbe, 0x5e, 0x16, 0xcc, 0x4c, 0xaa, 0xf5, 0x82, + 0xee, 0x9c, 0x48, 0x31, 0x9b, 0x5c, 0x35, 0x18, 0x65, 0x2e, 0xf3, 0x34, 0x41, 0x38, 0xd5, 0x23, + 0xf4, 0x51, 0x00, 0xe9, 0x1f, 0x5d, 0x97, 0x22, 0x73, 0xbe, 0x98, 0xfe, 0x61, 0xb2, 0xae, 0x6d, + 0xe0, 0x55, 0xc5, 0x04, 0x1b, 0x0c, 0xd1, 0xa7, 0x75, 0x96, 0x19, 0x0f, 0xd9, 0xff, 0xf0, 0xb1, + 0x8c, 0x4d, 0x3f, 0x39, 0x66, 0x18, 0x86, 0x5c, 0xbf, 0x45, 0xe7, 0x89, 0x88, 0x58, 0x7a, 0x57, + 0x5e, 0x2d, 0x85, 0x85, 0xc0, 0x69, 0xd6, 0x1d, 0xcf, 0xf1, 0x1b, 0x24, 0x9a, 0xe7, 0xe8, 0xe6, + 0x15, 0xd6, 0xac, 0x01, 0x4b, 0x42, 0x5d, 0x97, 0xaa, 0x54, 0xfa, 0xb9, 0x54, 0xe5, 0xfc, 0xfb, + 0xe0, 0x54, 0xd7, 0xc7, 0x3c, 0x50, 0x4a, 0xd9, 0xe1, 0xb3, 0xd1, 0xec, 0x7f, 0x31, 0xa8, 0x95, + 0xd6, 0xf5, 0xa0, 0xc9, 0xaf, 0xf6, 0x88, 0xf4, 0x17, 0x15, 0x36, 0x6e, 0x81, 0x53, 0xc4, 0xb8, + 0x06, 0x5b, 0x35, 0x62, 0x93, 0x25, 0x9d, 0xa3, 0xa1, 0x13, 0x11, 0xff, 0xb8, 0xe7, 0xe8, 0xb2, + 0x62, 0x82, 0x0d, 0x86, 0x68, 0x23, 0x95, 0x53, 0x72, 0xf9, 0xe8, 0x39, 0x25, 0xac, 0xca, 0x54, + 0x5e, 0x35, 0xfe, 0x2f, 0x58, 0x30, 0xe6, 0xa7, 0x66, 0x6e, 0x31, 0x61, 0xa4, 0xf9, 0xab, 0x82, + 0xdf, 0x2c, 0x95, 0x6e, 0xc3, 0x19, 0xfe, 0x79, 0x2a, 0xad, 0x72, 0x40, 0x95, 0xa6, 0xef, 0x08, + 0x1a, 0xec, 0x75, 0x47, 0x10, 0xf2, 0xd5, 0x25, 0x69, 0x43, 0x85, 0x5f, 0x92, 0x06, 0x39, 0x17, + 0xa4, 0xdd, 0x82, 0x5a, 0x23, 0x22, 0x4e, 0x72, 0xc8, 0xfb, 0xb2, 0xd8, 0x01, 0xfd, 0x8c, 0x24, + 0x80, 0x35, 0x2d, 0xfb, 0xff, 0x0c, 0xc0, 0x49, 0x39, 0x22, 0x32, 0x04, 0x9d, 0xea, 0x47, 0xce, + 0x57, 0x1b, 0xb7, 0x4a, 0x3f, 0x5e, 0x91, 0x00, 0xac, 0x71, 0xa8, 0x3d, 0xd6, 0x89, 0xc9, 0x52, + 0x48, 0xfc, 0x05, 0x77, 0x2d, 0x16, 0xe7, 0x9c, 0x6a, 0xa1, 0xdc, 0xd0, 0x20, 0x6c, 0xe2, 0x51, + 0x63, 0x9c, 0xdb, 0xc5, 0x71, 0x36, 0x7d, 0x45, 0xd8, 0xdb, 0x58, 0xc2, 0xd1, 0x2f, 0xe4, 0x56, + 0x8e, 0x2d, 0x26, 0x71, 0xab, 0x2b, 0xf2, 0xfe, 0x80, 0x57, 0x2c, 0xfe, 0x6d, 0x0b, 0xce, 0xf2, + 0x56, 0x39, 0x92, 0x37, 0xc2, 0xa6, 0x93, 0x90, 0xb8, 0x98, 0x4a, 0xee, 0x39, 0xfd, 0xd3, 0x4e, + 0xde, 0x3c, 0xb6, 0x38, 0xbf, 0x37, 0xe8, 0x4d, 0x0b, 0x4e, 0x6c, 0xa6, 0x6a, 0x7e, 0x48, 0xd5, + 0x71, 0xd4, 0x74, 0xfc, 0x14, 0x51, 0xbd, 0xd4, 0xd2, 0xed, 0x31, 0xce, 0x72, 0xb7, 0xff, 0xcc, + 0x02, 0x53, 0x8c, 0xde, 0xff, 0x52, 0x21, 0x07, 0x37, 0x05, 0xa5, 0x75, 0x59, 0xe9, 0x69, 0x5d, + 0x3e, 0x0e, 0xe5, 0x8e, 0xdb, 0x14, 0xfb, 0x0b, 0x7d, 0xfa, 0x3a, 0x3f, 0x8b, 0x69, 0xbb, 0xfd, + 0x4f, 0x2b, 0xda, 0x6f, 0x21, 0xf2, 0xa2, 0xbe, 0x27, 0x5e, 0x7b, 0x5d, 0x15, 0x1b, 0xe3, 0x6f, + 0x7e, 0xbd, 0xab, 0xd8, 0xd8, 0x8f, 0x1e, 0x3c, 0xed, 0x8d, 0x0f, 0x50, 0xaf, 0x5a, 0x63, 0x43, + 0xfb, 0xe4, 0xbc, 0xdd, 0x86, 0x2a, 0xdd, 0x82, 0x31, 0x07, 0x64, 0x35, 0xd5, 0xa9, 0xea, 0x15, + 0xd1, 0x7e, 0x6f, 0x77, 0xe2, 0x47, 0x0e, 0xde, 0x2d, 0xf9, 0x34, 0x56, 0xf4, 0x51, 0x0c, 0x35, + 0xfa, 0x9b, 0xa5, 0xe7, 0x89, 0xcd, 0xdd, 0x0d, 0x25, 0x33, 0x25, 0xa0, 0x90, 0xdc, 0x3f, 0xcd, + 0x07, 0xf9, 0x50, 0x63, 0xb7, 0xd1, 0x32, 0xa6, 0x7c, 0x0f, 0xb8, 0xac, 0x92, 0xe4, 0x24, 0xe0, + 0xde, 0xee, 0xc4, 0x4b, 0x07, 0x67, 0xaa, 0x1e, 0xc7, 0x9a, 0x85, 0xfd, 0xc5, 0x01, 0x3d, 0x77, + 0x45, 0x8d, 0xb9, 0xef, 0x89, 0xb9, 0xfb, 0x62, 0x66, 0xee, 0x5e, 0xe8, 0x9a, 0xbb, 0x63, 0xfa, + 0xd6, 0xd4, 0xd4, 0x6c, 0xbc, 0xdf, 0x86, 0xc0, 0xfe, 0xfe, 0x06, 0x66, 0x01, 0xbd, 0xde, 0x71, + 0x23, 0x12, 0x2f, 0x47, 0x1d, 0xdf, 0xf5, 0x5b, 0x6c, 0x3a, 0x56, 0x4d, 0x0b, 0x28, 0x05, 0xc6, + 0x59, 0x7c, 0xba, 0xa9, 0xa7, 0xdf, 0xfc, 0x96, 0xb3, 0xc5, 0x67, 0x95, 0x51, 0x76, 0x6b, 0x45, + 0xb4, 0x63, 0x85, 0x61, 0x7f, 0x9d, 0x9d, 0x65, 0x1b, 0x79, 0xc1, 0x74, 0x4e, 0x78, 0xec, 0xfa, + 0x5f, 0x5e, 0xb3, 0x4b, 0xcd, 0x09, 0x7e, 0xe7, 0x2f, 0x87, 0xa1, 0x3b, 0x30, 0xb4, 0xc6, 0xef, + 0xbf, 0x2b, 0xa6, 0x3e, 0xb9, 0xb8, 0x4c, 0x8f, 0xdd, 0x72, 0x22, 0x6f, 0xd6, 0xbb, 0xa7, 0x7f, + 0x62, 0xc9, 0xcd, 0xfe, 0xfd, 0x0a, 0x9c, 0xc8, 0x5c, 0x10, 0x9b, 0xaa, 0x96, 0x5a, 0xda, 0xb7, + 0x5a, 0xea, 0x87, 0x00, 0x9a, 0x24, 0xf4, 0x82, 0x1d, 0x66, 0x8e, 0x0d, 0x1c, 0xd8, 0x1c, 0x53, + 0x16, 0xfc, 0xac, 0xa2, 0x82, 0x0d, 0x8a, 0xa2, 0x50, 0x19, 0x2f, 0xbe, 0x9a, 0x29, 0x54, 0x66, + 0xdc, 0x62, 0x30, 0x78, 0x7f, 0x6f, 0x31, 0x70, 0xe1, 0x04, 0xef, 0xa2, 0xca, 0xbe, 0x3d, 0x44, + 0x92, 0x2d, 0xcb, 0x5f, 0x98, 0x4d, 0x93, 0xc1, 0x59, 0xba, 0x0f, 0xf2, 0xfe, 0x67, 0xf4, 0x6e, + 0xa8, 0xc9, 0xef, 0x1c, 0x8f, 0xd7, 0x74, 0x05, 0x03, 0x39, 0x0d, 0xd8, 0xbd, 0xcc, 0xe2, 0x67, + 0x57, 0x21, 0x01, 0x78, 0x50, 0x85, 0x04, 0xec, 0xcf, 0x97, 0xa8, 0x1d, 0xcf, 0xfb, 0xa5, 0x6a, + 0xe2, 0x3c, 0x05, 0x83, 0x4e, 0x27, 0xd9, 0x08, 0xba, 0x6e, 0xf3, 0x9b, 0x66, 0xad, 0x58, 0x40, + 0xd1, 0x02, 0x0c, 0x34, 0x75, 0x9d, 0x93, 0x83, 0x7c, 0x4f, 0xed, 0x12, 0x75, 0x12, 0x82, 0x19, + 0x15, 0xf4, 0x18, 0x0c, 0x24, 0x4e, 0x4b, 0xa6, 0x5c, 0xb1, 0x34, 0xdb, 0x55, 0xa7, 0x15, 0x63, + 0xd6, 0x6a, 0xaa, 0xef, 0x81, 0x7d, 0xd4, 0xf7, 0x4b, 0x30, 0x1a, 0xbb, 0x2d, 0xdf, 0x49, 0x3a, + 0x11, 0x31, 0x8e, 0xf9, 0x74, 0xe4, 0x86, 0x09, 0xc4, 0x69, 0x5c, 0xfb, 0x37, 0x47, 0xe0, 0xcc, + 0xca, 0xcc, 0xa2, 0xac, 0xde, 0x7d, 0x6c, 0x59, 0x53, 0x79, 0x3c, 0xee, 0x5f, 0xd6, 0x54, 0x0f, + 0xee, 0x9e, 0x91, 0x35, 0xe5, 0x19, 0x59, 0x53, 0xe9, 0x14, 0x96, 0x72, 0x11, 0x29, 0x2c, 0x79, + 0x3d, 0xe8, 0x27, 0x85, 0xe5, 0xd8, 0xd2, 0xa8, 0xf6, 0xec, 0xd0, 0x81, 0xd2, 0xa8, 0x54, 0x8e, + 0x59, 0x21, 0xc9, 0x05, 0x3d, 0x3e, 0x55, 0x6e, 0x8e, 0x99, 0xca, 0xef, 0xe1, 0x89, 0x33, 0x42, + 0xd4, 0xbf, 0x5a, 0x7c, 0x07, 0xfa, 0xc8, 0xef, 0x11, 0xb9, 0x3b, 0x66, 0x4e, 0xd9, 0x50, 0x11, + 0x39, 0x65, 0x79, 0xdd, 0xd9, 0x37, 0xa7, 0xec, 0x25, 0x18, 0x6d, 0x78, 0x81, 0x4f, 0x96, 0xa3, + 0x20, 0x09, 0x1a, 0x81, 0x27, 0xcc, 0x7a, 0x25, 0x12, 0x66, 0x4c, 0x20, 0x4e, 0xe3, 0xf6, 0x4a, + 0x48, 0xab, 0x1d, 0x35, 0x21, 0x0d, 0x1e, 0x50, 0x42, 0xda, 0xcf, 0xea, 0xd4, 0xe9, 0x61, 0xf6, + 0x45, 0x3e, 0x54, 0xfc, 0x17, 0xe9, 0x27, 0x7f, 0x1a, 0xbd, 0xc5, 0xaf, 0xd3, 0xa3, 0x86, 0xf1, + 0x4c, 0xd0, 0xa6, 0x86, 0xdf, 0x08, 0x1b, 0x92, 0xd7, 0x8e, 0x61, 0xc2, 0xde, 0x5a, 0xd1, 0x6c, + 0xd4, 0x15, 0x7b, 0xba, 0x09, 0xa7, 0x3b, 0x72, 0x94, 0xd4, 0xee, 0x2f, 0x97, 0xe0, 0xfb, 0xf6, + 0xed, 0x02, 0xba, 0x03, 0x90, 0x38, 0x2d, 0x31, 0x51, 0xc5, 0x81, 0xc9, 0x11, 0xc3, 0x2b, 0x57, + 0x25, 0x3d, 0x5e, 0x93, 0x44, 0xfd, 0x65, 0x47, 0x11, 0xf2, 0x37, 0x8b, 0xaa, 0x0c, 0xbc, 0xae, + 0xd2, 0x8d, 0x38, 0xf0, 0x08, 0x66, 0x10, 0xaa, 0xfe, 0x23, 0xd2, 0xd2, 0xf7, 0x3f, 0xab, 0xcf, + 0x87, 0x59, 0x2b, 0x16, 0x50, 0xf4, 0x02, 0x0c, 0x3b, 0x9e, 0xc7, 0xf3, 0x63, 0x48, 0x2c, 0xee, + 0xd3, 0xd1, 0x35, 0xe4, 0x34, 0x08, 0x9b, 0x78, 0xf6, 0x9f, 0x96, 0x60, 0x62, 0x1f, 0x99, 0xd2, + 0x95, 0xf1, 0x57, 0xe9, 0x3b, 0xe3, 0x4f, 0xe4, 0x28, 0x0c, 0xf6, 0xc8, 0x51, 0x78, 0x01, 0x86, + 0x13, 0xe2, 0xb4, 0x45, 0x40, 0x96, 0xf0, 0x04, 0xe8, 0x13, 0x60, 0x0d, 0xc2, 0x26, 0x1e, 0x95, + 0x62, 0x63, 0x4e, 0xa3, 0x41, 0xe2, 0x58, 0x26, 0x21, 0x08, 0x6f, 0x6a, 0x61, 0x19, 0x0e, 0xcc, + 0x49, 0x3d, 0x9d, 0x62, 0x81, 0x33, 0x2c, 0xb3, 0x03, 0x5e, 0xeb, 0x73, 0xc0, 0xbf, 0x56, 0x82, + 0xc7, 0xf7, 0xd4, 0x6e, 0x7d, 0xe7, 0x87, 0x74, 0x62, 0x12, 0x65, 0x27, 0xce, 0x8d, 0x98, 0x44, + 0x98, 0x41, 0xf8, 0x28, 0x85, 0xa1, 0x71, 0xbf, 0x76, 0xd1, 0xc9, 0x4b, 0x7c, 0x94, 0x52, 0x2c, + 0x70, 0x86, 0xe5, 0x61, 0xa7, 0xe5, 0xdf, 0x2f, 0xc1, 0x93, 0x7d, 0xd8, 0x00, 0x05, 0x26, 0x79, + 0xa5, 0x53, 0xed, 0xca, 0x0f, 0x28, 0x23, 0xf2, 0x90, 0xc3, 0xf5, 0xf5, 0x12, 0x9c, 0xef, 0xad, + 0x8a, 0xd1, 0x8f, 0xc1, 0x89, 0x48, 0x45, 0x61, 0x99, 0x59, 0x7a, 0xa7, 0xb9, 0x27, 0x21, 0x05, + 0xc2, 0x59, 0x5c, 0x34, 0x09, 0x10, 0x3a, 0xc9, 0x46, 0x7c, 0x69, 0xdb, 0x8d, 0x13, 0x51, 0x85, + 0x66, 0x8c, 0x9f, 0x5d, 0xc9, 0x56, 0x6c, 0x60, 0x50, 0x76, 0xec, 0xdf, 0x6c, 0x70, 0x3d, 0x48, + 0xf8, 0x43, 0x7c, 0x1b, 0x71, 0x5a, 0xde, 0xd9, 0x61, 0x80, 0x70, 0x16, 0x97, 0xb2, 0x63, 0xa7, + 0xa3, 0xbc, 0xa3, 0x7c, 0x7f, 0xc1, 0xd8, 0x2d, 0xa8, 0x56, 0x6c, 0x60, 0x64, 0xf3, 0x0f, 0x2b, + 0xfb, 0xe7, 0x1f, 0xda, 0xff, 0xa4, 0x04, 0xe7, 0x7a, 0x9a, 0x72, 0xfd, 0x2d, 0xc0, 0x87, 0x2f, + 0x67, 0xf0, 0x70, 0x73, 0xe7, 0x80, 0xb9, 0x6d, 0x7f, 0xdc, 0x63, 0xa6, 0x89, 0xdc, 0xb6, 0xc3, + 0x27, 0x87, 0x3f, 0x7c, 0xe3, 0xd9, 0x95, 0xce, 0x36, 0x70, 0x80, 0x74, 0xb6, 0xcc, 0xc7, 0xa8, + 0xf4, 0xb9, 0x90, 0xff, 0xbc, 0xdc, 0x73, 0x78, 0xe9, 0xd6, 0xaf, 0x2f, 0x3f, 0xed, 0x2c, 0x9c, + 0x74, 0x7d, 0x76, 0x7f, 0xd3, 0x4a, 0x67, 0x4d, 0x14, 0x26, 0x29, 0xa5, 0x6f, 0x4f, 0x9f, 0xcf, + 0xc0, 0x71, 0xd7, 0x13, 0x0f, 0x61, 0x7a, 0xe1, 0xe1, 0x86, 0xf4, 0x60, 0x09, 0xae, 0x68, 0x09, + 0xce, 0xca, 0xa1, 0xd8, 0x70, 0x22, 0xd2, 0x14, 0x6a, 0x24, 0x16, 0x09, 0x15, 0xe7, 0x78, 0x52, + 0x46, 0x0e, 0x02, 0xce, 0x7f, 0x8e, 0x5d, 0x99, 0x13, 0x84, 0x6e, 0x43, 0x6c, 0x72, 0xf4, 0x95, + 0x39, 0xb4, 0x11, 0x73, 0x98, 0xfd, 0x21, 0xa8, 0xa9, 0xf7, 0xe7, 0x61, 0xdd, 0x6a, 0xd2, 0x75, + 0x85, 0x75, 0xab, 0x19, 0x67, 0x60, 0xd1, 0xaf, 0x45, 0x4d, 0xe2, 0xcc, 0xea, 0xb9, 0x46, 0x76, + 0x98, 0x7d, 0x6c, 0xff, 0x10, 0x8c, 0x28, 0x3f, 0x4b, 0xbf, 0x17, 0x09, 0xd9, 0x5f, 0x1c, 0x84, + 0xd1, 0x54, 0x71, 0xc0, 0x94, 0x83, 0xd5, 0xda, 0xd7, 0xc1, 0xca, 0xc2, 0xf4, 0x3b, 0xbe, 0xbc, + 0x65, 0xcc, 0x08, 0xd3, 0xef, 0xf8, 0x04, 0x73, 0x18, 0x35, 0x6f, 0x9b, 0xd1, 0x0e, 0xee, 0xf8, + 0x22, 0x9c, 0x56, 0x99, 0xb7, 0xb3, 0xac, 0x15, 0x0b, 0x28, 0xfa, 0xb8, 0x05, 0x23, 0x31, 0xf3, + 0xde, 0x73, 0xf7, 0xb4, 0x98, 0x74, 0x57, 0x8f, 0x5e, 0xfb, 0x50, 0x15, 0xc2, 0x64, 0x11, 0x32, + 0x66, 0x0b, 0x4e, 0x71, 0x44, 0x9f, 0xb2, 0xa0, 0xa6, 0x2e, 0x43, 0x11, 0x57, 0x01, 0xae, 0x14, + 0x5b, 0x7b, 0x91, 0xfb, 0x35, 0xd5, 0x41, 0x88, 0x2a, 0x82, 0x87, 0x35, 0x63, 0x14, 0x2b, 0xdf, + 0xf1, 0xd0, 0xf1, 0xf8, 0x8e, 0x21, 0xc7, 0x6f, 0xfc, 0x6e, 0xa8, 0xb5, 0x1d, 0xdf, 0x5d, 0x27, + 0x71, 0xc2, 0xdd, 0xb9, 0xb2, 0x24, 0xac, 0x6c, 0xc4, 0x1a, 0x4e, 0x15, 0x72, 0xcc, 0x5e, 0x2c, + 0x31, 0xfc, 0xaf, 0x4c, 0x21, 0xaf, 0xe8, 0x66, 0x6c, 0xe2, 0x98, 0xce, 0x62, 0x78, 0xa0, 0xce, + 0xe2, 0xe1, 0xbd, 0x9d, 0xc5, 0xf6, 0x3f, 0xb4, 0xe0, 0x6c, 0xee, 0x57, 0x7b, 0x78, 0x03, 0x1f, + 0xed, 0x2f, 0x55, 0xe0, 0x74, 0x4e, 0x95, 0x4f, 0xb4, 0x63, 0xce, 0x67, 0xab, 0x88, 0x18, 0x82, + 0xf4, 0x91, 0xb8, 0x1c, 0xc6, 0x9c, 0x49, 0x7c, 0xb0, 0xa3, 0x1a, 0x7d, 0x5c, 0x52, 0xbe, 0xbf, + 0xc7, 0x25, 0xc6, 0xb4, 0x1c, 0x78, 0xa0, 0xd3, 0xb2, 0xb2, 0xcf, 0x19, 0xc6, 0xaf, 0x59, 0x30, + 0xde, 0xee, 0x51, 0x5a, 0x5e, 0x38, 0x1e, 0x6f, 0x1e, 0x4f, 0xe1, 0xfa, 0xfa, 0x63, 0x77, 0x77, + 0x27, 0x7a, 0x56, 0xf4, 0xc7, 0x3d, 0x7b, 0x65, 0x7f, 0xbb, 0x0c, 0xac, 0xc4, 0x2c, 0xab, 0xe4, + 0xb6, 0x83, 0x3e, 0x66, 0x16, 0x0b, 0xb6, 0x8a, 0x2a, 0x6c, 0xcb, 0x89, 0xab, 0x62, 0xc3, 0x7c, + 0x04, 0xf3, 0x6a, 0x0f, 0x67, 0x85, 0x56, 0xa9, 0x0f, 0xa1, 0xe5, 0xc9, 0xaa, 0xcc, 0xe5, 0xe2, + 0xab, 0x32, 0xd7, 0xb2, 0x15, 0x99, 0xf7, 0xfe, 0xc4, 0x03, 0x0f, 0xe5, 0x27, 0xfe, 0x45, 0x8b, + 0x0b, 0x9e, 0xcc, 0x57, 0xd0, 0x96, 0x81, 0xb5, 0x87, 0x65, 0xf0, 0x0c, 0x54, 0x63, 0xe2, 0xad, + 0x5f, 0x21, 0x8e, 0x27, 0x2c, 0x08, 0x7d, 0x7e, 0x2d, 0xda, 0xb1, 0xc2, 0x60, 0xd7, 0xb6, 0x7a, + 0x5e, 0x70, 0xe7, 0x52, 0x3b, 0x4c, 0x76, 0x84, 0x2d, 0xa1, 0xaf, 0x6d, 0x55, 0x10, 0x6c, 0x60, + 0xd9, 0x7f, 0xab, 0xc4, 0x67, 0xa0, 0x08, 0x82, 0x78, 0x31, 0x73, 0xd1, 0x5e, 0xff, 0xf1, 0x03, + 0x1f, 0x01, 0x68, 0xa8, 0x2b, 0xea, 0xc5, 0x99, 0xd0, 0x95, 0x23, 0xdf, 0x9f, 0x2d, 0xe8, 0xe9, + 0xd7, 0xd0, 0x6d, 0xd8, 0xe0, 0x97, 0x92, 0xa5, 0xe5, 0x7d, 0x65, 0x69, 0x4a, 0xac, 0x0c, 0xec, + 0xa3, 0xed, 0xfe, 0xd4, 0x82, 0x94, 0x45, 0x84, 0x42, 0xa8, 0xd0, 0xee, 0xee, 0x14, 0x73, 0xfb, + 0xbe, 0x49, 0x9a, 0x8a, 0x46, 0x31, 0xed, 0xd9, 0x4f, 0xcc, 0x19, 0x21, 0x4f, 0xc4, 0x4a, 0xf0, + 0x51, 0xbd, 0x5e, 0x1c, 0xc3, 0x2b, 0x41, 0xb0, 0xc9, 0x0f, 0x36, 0x75, 0xdc, 0x85, 0xfd, 0x22, + 0x9c, 0xea, 0xea, 0x14, 0xbb, 0x53, 0x2b, 0xa0, 0xda, 0x27, 0x33, 0x5d, 0x59, 0x02, 0x27, 0xe6, + 0x30, 0xfb, 0xeb, 0x16, 0x9c, 0xcc, 0x92, 0x47, 0x6f, 0x59, 0x70, 0x2a, 0xce, 0xd2, 0x3b, 0xae, + 0xb1, 0x53, 0xf1, 0x8e, 0x5d, 0x20, 0xdc, 0xdd, 0x09, 0xfb, 0xff, 0x8a, 0xc9, 0x7f, 0xcb, 0xf5, + 0x9b, 0xc1, 0x1d, 0x65, 0x98, 0x58, 0x3d, 0x0d, 0x13, 0xba, 0x1e, 0x1b, 0x1b, 0xa4, 0xd9, 0xf1, + 0xba, 0x32, 0x47, 0x57, 0x44, 0x3b, 0x56, 0x18, 0x2c, 0x51, 0xae, 0x23, 0xca, 0xb6, 0x67, 0x26, + 0xe5, 0xac, 0x68, 0xc7, 0x0a, 0x03, 0x3d, 0x0f, 0x23, 0xc6, 0x4b, 0xca, 0x79, 0xc9, 0x0c, 0x72, + 0x43, 0x65, 0xc6, 0x38, 0x85, 0x85, 0x26, 0x01, 0x94, 0x91, 0x23, 0x55, 0x24, 0x73, 0x14, 0x29, + 0x49, 0x14, 0x63, 0x03, 0x83, 0xa5, 0xa5, 0x7a, 0x9d, 0x98, 0xf9, 0xf8, 0x07, 0x75, 0x29, 0xd1, + 0x19, 0xd1, 0x86, 0x15, 0x94, 0x4a, 0x93, 0xb6, 0xe3, 0x77, 0x1c, 0x8f, 0x8e, 0x90, 0xd8, 0xfa, + 0xa9, 0x65, 0xb8, 0xa8, 0x20, 0xd8, 0xc0, 0xa2, 0x6f, 0x9c, 0xb8, 0x6d, 0xf2, 0x4a, 0xe0, 0xcb, + 0x38, 0x35, 0x7d, 0xec, 0x23, 0xda, 0xb1, 0xc2, 0xb0, 0xff, 0xab, 0x05, 0x27, 0x74, 0x92, 0x3b, + 0xbf, 0x3d, 0xdb, 0xdc, 0xa9, 0x5a, 0xfb, 0xee, 0x54, 0xd3, 0xd9, 0xbf, 0xa5, 0xbe, 0xb2, 0x7f, + 0xcd, 0xc4, 0xdc, 0xf2, 0x9e, 0x89, 0xb9, 0xdf, 0xaf, 0x6f, 0x66, 0xe5, 0x19, 0xbc, 0xc3, 0x79, + 0xb7, 0xb2, 0x22, 0x1b, 0x06, 0x1b, 0x8e, 0xaa, 0xf0, 0x32, 0xc2, 0xf7, 0x0e, 0x33, 0xd3, 0x0c, + 0x49, 0x40, 0xec, 0x25, 0xa8, 0xa9, 0xd3, 0x0f, 0xb9, 0x51, 0xb5, 0xf2, 0x37, 0xaa, 0x7d, 0x25, + 0x08, 0xd6, 0xd7, 0xbe, 0xf1, 0x9d, 0x27, 0xde, 0xf1, 0x7b, 0xdf, 0x79, 0xe2, 0x1d, 0x7f, 0xf4, + 0x9d, 0x27, 0xde, 0xf1, 0xf1, 0xbb, 0x4f, 0x58, 0xdf, 0xb8, 0xfb, 0x84, 0xf5, 0x7b, 0x77, 0x9f, + 0xb0, 0xfe, 0xe8, 0xee, 0x13, 0xd6, 0xb7, 0xef, 0x3e, 0x61, 0x7d, 0xe1, 0x3f, 0x3d, 0xf1, 0x8e, + 0x57, 0x72, 0x03, 0x15, 0xe9, 0x8f, 0x67, 0x1b, 0xcd, 0xa9, 0xad, 0x8b, 0x2c, 0x56, 0x8e, 0x2e, + 0xaf, 0x29, 0x63, 0x4e, 0x4d, 0xc9, 0xe5, 0xf5, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x87, 0xd4, + 0x96, 0xc0, 0xad, 0xe1, 0x00, 0x00, } func (m *AWSAuthConfig) Marshal() (dAtA []byte, err error) { @@ -5158,6 +5161,11 @@ func (m *AWSAuthConfig) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + i -= len(m.Profile) + copy(dAtA[i:], m.Profile) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Profile))) + i-- + dAtA[i] = 0x1a i -= len(m.RoleARN) copy(dAtA[i:], m.RoleARN) i = encodeVarintGenerated(dAtA, i, uint64(len(m.RoleARN))) @@ -7255,6 +7263,14 @@ func (m *ApplicationSourceKustomize) MarshalToSizedBuffer(dAtA []byte) (int, err _ = i var l int _ = l + i-- + if m.LabelWithoutSelector { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x70 if len(m.Components) > 0 { for iNdEx := len(m.Components) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Components[iNdEx]) @@ -14357,6 +14373,8 @@ func (m *AWSAuthConfig) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) l = len(m.RoleARN) n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Profile) + n += 1 + l + sovGenerated(uint64(l)) return n } @@ -15184,6 +15202,7 @@ func (m *ApplicationSourceKustomize) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } } + n += 2 return n } @@ -17806,6 +17825,7 @@ func (this *AWSAuthConfig) String() string { s := strings.Join([]string{`&AWSAuthConfig{`, `ClusterName:` + fmt.Sprintf("%v", this.ClusterName) + `,`, `RoleARN:` + fmt.Sprintf("%v", this.RoleARN) + `,`, + `Profile:` + fmt.Sprintf("%v", this.Profile) + `,`, `}`, }, "") return s @@ -18397,6 +18417,7 @@ func (this *ApplicationSourceKustomize) String() string { `Replicas:` + repeatedStringForReplicas + `,`, `Patches:` + repeatedStringForPatches + `,`, `Components:` + fmt.Sprintf("%v", this.Components) + `,`, + `LabelWithoutSelector:` + fmt.Sprintf("%v", this.LabelWithoutSelector) + `,`, `}`, }, "") return s @@ -20456,6 +20477,38 @@ func (m *AWSAuthConfig) Unmarshal(dAtA []byte) error { } m.RoleARN = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Profile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Profile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -27291,6 +27344,26 @@ func (m *ApplicationSourceKustomize) Unmarshal(dAtA []byte) error { } m.Components = append(m.Components, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 14: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LabelWithoutSelector", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.LabelWithoutSelector = bool(v != 0) default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/application/v1alpha1/generated.proto b/pkg/apis/application/v1alpha1/generated.proto index a2b17373de259..5916e42a53922 100644 --- a/pkg/apis/application/v1alpha1/generated.proto +++ b/pkg/apis/application/v1alpha1/generated.proto @@ -22,6 +22,9 @@ message AWSAuthConfig { // RoleARN contains optional role ARN. If set then AWS IAM Authenticator assume a role to perform cluster operations instead of the default AWS credential provider chain. optional string roleARN = 2; + + // Profile contains optional role ARN. If set then AWS IAM Authenticator uses the profile to perform cluster operations instead of the default AWS credential provider chain. + optional string profile = 3; } // AppProject provides a logical grouping of applications, providing controls for: @@ -526,6 +529,9 @@ message ApplicationSourceKustomize { // Components specifies a list of kustomize components to add to the kustomization before building repeated string components = 13; + + // LabelWithoutSelector specifies whether to apply common labels to resource selectors or not + optional bool labelWithoutSelector = 14; } // ApplicationSourcePlugin holds options specific to config management plugins diff --git a/pkg/apis/application/v1alpha1/openapi_generated.go b/pkg/apis/application/v1alpha1/openapi_generated.go index 2274e252a4148..32eb8a725f353 100644 --- a/pkg/apis/application/v1alpha1/openapi_generated.go +++ b/pkg/apis/application/v1alpha1/openapi_generated.go @@ -191,6 +191,13 @@ func schema_pkg_apis_application_v1alpha1_AWSAuthConfig(ref common.ReferenceCall Format: "", }, }, + "profile": { + SchemaProps: spec.SchemaProps{ + Description: "Profile contains optional role ARN. If set then AWS IAM Authenticator uses the profile to perform cluster operations instead of the default AWS credential provider chain.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, @@ -1978,6 +1985,13 @@ func schema_pkg_apis_application_v1alpha1_ApplicationSourceKustomize(ref common. }, }, }, + "labelWithoutSelector": { + SchemaProps: spec.SchemaProps{ + Description: "LabelWithoutSelector specifies whether to apply common labels to resource selectors or not", + Type: []string{"boolean"}, + Format: "", + }, + }, }, }, }, diff --git a/pkg/apis/application/v1alpha1/types.go b/pkg/apis/application/v1alpha1/types.go index e271003ad6ad0..fc2908c4643dc 100644 --- a/pkg/apis/application/v1alpha1/types.go +++ b/pkg/apis/application/v1alpha1/types.go @@ -469,6 +469,8 @@ type ApplicationSourceKustomize struct { Patches KustomizePatches `json:"patches,omitempty" protobuf:"bytes,12,opt,name=patches"` // Components specifies a list of kustomize components to add to the kustomization before building Components []string `json:"components,omitempty" protobuf:"bytes,13,rep,name=components"` + //LabelWithoutSelector specifies whether to apply common labels to resource selectors or not + LabelWithoutSelector bool `json:"labelWithoutSelector,omitempty" protobuf:"bytes,14,opt,name=labelWithoutSelector"` } type KustomizeReplica struct { @@ -1856,6 +1858,9 @@ type AWSAuthConfig struct { // RoleARN contains optional role ARN. If set then AWS IAM Authenticator assume a role to perform cluster operations instead of the default AWS credential provider chain. RoleARN string `json:"roleARN,omitempty" protobuf:"bytes,2,opt,name=roleARN"` + + // Profile contains optional role ARN. If set then AWS IAM Authenticator uses the profile to perform cluster operations instead of the default AWS credential provider chain. + Profile string `json:"profile,omitempty" protobuf:"bytes,3,opt,name=profile"` } // ExecProviderConfig is config used to call an external command to perform cluster authentication @@ -2987,6 +2992,9 @@ func (c *Cluster) RawRestConfig() *rest.Config { if c.Config.AWSAuthConfig.RoleARN != "" { args = append(args, "--role-arn", c.Config.AWSAuthConfig.RoleARN) } + if c.Config.AWSAuthConfig.Profile != "" { + args = append(args, "--profile", c.Config.AWSAuthConfig.Profile) + } config = &rest.Config{ Host: c.Server, TLSClientConfig: tlsClientConfig, diff --git a/pkg/ratelimiter/ratelimiter.go b/pkg/ratelimiter/ratelimiter.go index 32507d883e8ae..1c491a584873e 100644 --- a/pkg/ratelimiter/ratelimiter.go +++ b/pkg/ratelimiter/ratelimiter.go @@ -11,7 +11,7 @@ import ( type AppControllerRateLimiterConfig struct { BucketSize int64 - BucketQPS int64 + BucketQPS float64 FailureCoolDown time.Duration BaseDelay time.Duration MaxDelay time.Duration @@ -22,7 +22,8 @@ func GetDefaultAppRateLimiterConfig() *AppControllerRateLimiterConfig { return &AppControllerRateLimiterConfig{ // global queue rate limit config 500, - 50, + // when WORKQUEUE_BUCKET_QPS is MaxFloat64 global bucket limiting is disabled(default) + math.MaxFloat64, // individual item rate limit config // when WORKQUEUE_FAILURE_COOLDOWN is 0 per item rate limiting is disabled(default) 0, diff --git a/reposerver/gpgwatcher.go b/reposerver/gpgwatcher.go index 9c2c9be790813..5b43d6a24ac76 100644 --- a/reposerver/gpgwatcher.go +++ b/reposerver/gpgwatcher.go @@ -19,7 +19,7 @@ func StartGPGWatcher(sourcePath string) error { forceSync := false watcher, err := fsnotify.NewWatcher() if err != nil { - return err + return fmt.Errorf("failed to create fsnotify Watcher: %w", err) } defer func(watcher *fsnotify.Watcher) { if err = watcher.Close(); err != nil { @@ -83,7 +83,7 @@ func StartGPGWatcher(sourcePath string) error { err = watcher.Add(sourcePath) if err != nil { - return err + return fmt.Errorf("failed to add a new source to the watcher: %w", err) } <-done return fmt.Errorf("Abnormal termination of GPG watcher, refusing to continue.") diff --git a/reposerver/metrics/githandlers_test.go b/reposerver/metrics/githandlers_test.go new file mode 100644 index 0000000000000..6eaeeca82cc36 --- /dev/null +++ b/reposerver/metrics/githandlers_test.go @@ -0,0 +1,122 @@ +package metrics + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/sync/semaphore" +) + +func TestMain(m *testing.M) { + os.Exit(m.Run()) +} + +func TestEdgeCasesAndErrorHandling(t *testing.T) { + tests := []struct { + name string + setup func() + teardown func() + testFunc func(t *testing.T) + }{ + { + name: "lsRemoteParallelismLimitSemaphore is nil", + testFunc: func(t *testing.T) { + lsRemoteParallelismLimitSemaphore = nil + assert.NotPanics(t, func() { + NewGitClientEventHandlers(&MetricsServer{}) + }) + }, + }, + { + name: "lsRemoteParallelismLimitSemaphore is not nil", + setup: func() { + lsRemoteParallelismLimitSemaphore = semaphore.NewWeighted(1) + }, + teardown: func() { + lsRemoteParallelismLimitSemaphore = nil + }, + testFunc: func(t *testing.T) { + assert.NotPanics(t, func() { + NewGitClientEventHandlers(&MetricsServer{}) + }) + }, + }, + { + name: "lsRemoteParallelismLimitSemaphore is not nil and Acquire returns error", + setup: func() { + lsRemoteParallelismLimitSemaphore = semaphore.NewWeighted(1) + }, + teardown: func() { + lsRemoteParallelismLimitSemaphore = nil + }, + testFunc: func(t *testing.T) { + assert.NotPanics(t, func() { + NewGitClientEventHandlers(&MetricsServer{}) + }) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup() + } + if tt.teardown != nil { + defer tt.teardown() + } + tt.testFunc(t) + }) + } +} + +func TestSemaphoreFunctionality(t *testing.T) { + os.Setenv("ARGOCD_GIT_LSREMOTE_PARALLELISM_LIMIT", "1") + + tests := []struct { + name string + setup func() + teardown func() + testFunc func(t *testing.T) + }{ + { + name: "lsRemoteParallelismLimitSemaphore is not nil", + setup: func() { + lsRemoteParallelismLimitSemaphore = semaphore.NewWeighted(1) + }, + teardown: func() { + lsRemoteParallelismLimitSemaphore = nil + }, + testFunc: func(t *testing.T) { + assert.NotPanics(t, func() { + NewGitClientEventHandlers(&MetricsServer{}) + }) + }, + }, + { + name: "lsRemoteParallelismLimitSemaphore is not nil and Acquire returns error", + setup: func() { + lsRemoteParallelismLimitSemaphore = semaphore.NewWeighted(1) + }, + teardown: func() { + lsRemoteParallelismLimitSemaphore = nil + }, + testFunc: func(t *testing.T) { + assert.NotPanics(t, func() { + NewGitClientEventHandlers(&MetricsServer{}) + }) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup() + } + if tt.teardown != nil { + defer tt.teardown() + } + tt.testFunc(t) + }) + } +} diff --git a/reposerver/metrics/metrics.go b/reposerver/metrics/metrics.go index e629b75e63d3c..44f3dbd01e1bb 100644 --- a/reposerver/metrics/metrics.go +++ b/reposerver/metrics/metrics.go @@ -12,6 +12,7 @@ import ( type MetricsServer struct { handler http.Handler + gitFetchFailCounter *prometheus.CounterVec gitRequestCounter *prometheus.CounterVec gitRequestHistogram *prometheus.HistogramVec repoPendingRequestsGauge *prometheus.GaugeVec @@ -32,6 +33,15 @@ func NewMetricsServer() *MetricsServer { registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) registry.MustRegister(collectors.NewGoCollector()) + gitFetchFailCounter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "argocd_git_fetch_fail_total", + Help: "Number of git fetch requests failures by repo server", + }, + []string{"repo", "revision"}, + ) + registry.MustRegister(gitFetchFailCounter) + gitRequestCounter := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "argocd_git_request_total", @@ -81,6 +91,7 @@ func NewMetricsServer() *MetricsServer { return &MetricsServer{ handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), + gitFetchFailCounter: gitFetchFailCounter, gitRequestCounter: gitRequestCounter, gitRequestHistogram: gitRequestHistogram, repoPendingRequestsGauge: repoPendingRequestsGauge, @@ -93,6 +104,10 @@ func (m *MetricsServer) GetHandler() http.Handler { return m.handler } +func (m *MetricsServer) IncGitFetchFail(repo string, revision string) { + m.gitFetchFailCounter.WithLabelValues(repo, revision).Inc() +} + // IncGitRequest increments the git requests counter func (m *MetricsServer) IncGitRequest(repo string, requestType GitRequestType) { m.gitRequestCounter.WithLabelValues(repo, string(requestType)).Inc() diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index 41f26b1f434b8..629fdbe60ded4 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -1389,7 +1389,7 @@ func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string, if q.KustomizeOptions != nil { kustomizeBinary = q.KustomizeOptions.BinaryPath } - k := kustomize.NewKustomizeApp(appPath, q.Repo.GetGitCreds(gitCredsStore), repoURL, kustomizeBinary) + k := kustomize.NewKustomizeApp(repoRoot, appPath, q.Repo.GetGitCreds(gitCredsStore), repoURL, kustomizeBinary) targetObjs, _, err = k.Build(q.ApplicationSource.Kustomize, q.KustomizeOptions, env) case v1alpha1.ApplicationSourceTypePlugin: pluginName := "" @@ -1976,7 +1976,7 @@ func (s *Service) GetAppDetails(ctx context.Context, q *apiclient.RepoServerAppD return err } case v1alpha1.ApplicationSourceTypeKustomize: - if err := populateKustomizeAppDetails(res, q, opContext.appPath, commitSHA, s.gitCredsStore); err != nil { + if err := populateKustomizeAppDetails(res, q, repoRoot, opContext.appPath, commitSHA, s.gitCredsStore); err != nil { return err } case v1alpha1.ApplicationSourceTypePlugin: @@ -2117,13 +2117,13 @@ func walkHelmValueFilesInPath(root string, valueFiles *[]string) filepath.WalkFu } } -func populateKustomizeAppDetails(res *apiclient.RepoAppDetailsResponse, q *apiclient.RepoServerAppDetailsQuery, appPath string, reversion string, credsStore git.CredsStore) error { +func populateKustomizeAppDetails(res *apiclient.RepoAppDetailsResponse, q *apiclient.RepoServerAppDetailsQuery, repoRoot string, appPath string, reversion string, credsStore git.CredsStore) error { res.Kustomize = &apiclient.KustomizeAppSpec{} kustomizeBinary := "" if q.KustomizeOptions != nil { kustomizeBinary = q.KustomizeOptions.BinaryPath } - k := kustomize.NewKustomizeApp(appPath, q.Repo.GetGitCreds(credsStore), q.Repo.Repo, kustomizeBinary) + k := kustomize.NewKustomizeApp(repoRoot, appPath, q.Repo.GetGitCreds(credsStore), q.Repo.Repo, kustomizeBinary) fakeManifestRequest := apiclient.ManifestRequest{ AppName: q.AppName, Namespace: "", // FIXME: omit it for now @@ -2397,7 +2397,11 @@ func directoryPermissionInitializer(rootPath string) goio.Closer { // nolint:unparam func (s *Service) checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bool) (goio.Closer, error) { closer := s.gitRepoInitializer(gitClient.Root()) - return closer, checkoutRevision(gitClient, revision, submoduleEnabled) + err := checkoutRevision(gitClient, revision, submoduleEnabled) + if err != nil { + s.metricsServer.IncGitFetchFail(gitClient.Root(), revision) + } + return closer, err } func checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bool) error { diff --git a/reposerver/server.go b/reposerver/server.go index 007b7136e41ed..e1d611801c3ec 100644 --- a/reposerver/server.go +++ b/reposerver/server.go @@ -102,7 +102,7 @@ func NewServer(metricsServer *metrics.MetricsServer, cache *reposervercache.Cach } repoService := repository.NewService(metricsServer, cache, initConstants, argo.NewResourceTracking(), gitCredsStore, filepath.Join(os.TempDir(), "_argocd-repo")) if err := repoService.Init(); err != nil { - return nil, err + return nil, fmt.Errorf("failed to initialize the repo service: %w", err) } return &ArgoCDRepoServer{ diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/health.lua b/resource_customizations/beat.k8s.elastic.co/Beat/health.lua new file mode 100644 index 0000000000000..c7639dbbd94f0 --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/health.lua @@ -0,0 +1,31 @@ +local hs = {} + +if obj.status ~= nil and (obj.status.health ~= nil or obj.status.expectedNodes ~= nil) then + if obj.status.health == "red" then + hs.status = "Degraded" + hs.message = "Elastic Beat status is Red" + return hs + elseif obj.status.health == "green" then + hs.status = "Healthy" + hs.message = "Elastic Beat status is Green" + return hs + elseif obj.status.health == "yellow" then + if obj.status.availableNodes ~= nil and obj.status.expectedNodes ~= nil then + hs.status = "Progressing" + hs.message = "Elastic Beat status is deploying, there is " .. obj.status.availableNodes .. " instance(s) on " .. obj.status.expectedNodes .. " expected" + return hs + else + hs.status = "Progressing" + hs.message = "Elastic Beat phase is progressing" + return hs + end + elseif obj.status.health == nil then + hs.status = "Progressing" + hs.message = "Elastic Beat phase is progressing" + return hs + end +end + +hs.status = "Unknown" +hs.message = "Elastic Beat status is unknown. Ensure your ArgoCD is current and then check for/file a bug report: https://github.com/argoproj/argo-cd/issues" +return hs diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/health_test.yaml b/resource_customizations/beat.k8s.elastic.co/Beat/health_test.yaml new file mode 100644 index 0000000000000..fb44e998ffaf1 --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/health_test.yaml @@ -0,0 +1,29 @@ +tests: +- healthStatus: + status: Healthy + message: "Elastic Beat status is Green" + inputPath: testdata/ready_green.yaml +- healthStatus: + status: Progressing + message: "Elastic Beat phase is progressing" + inputPath: testdata/ready_yellow_single_node.yaml +- healthStatus: + status: Progressing + message: "Elastic Beat status is deploying, there is 1 instance(s) on 2 expected" + inputPath: testdata/ready_yellow.yaml +- healthStatus: + status: Progressing + message: "Elastic Beat phase is progressing" + inputPath: testdata/progressing.yaml +- healthStatus: + status: Degraded + message: "Elastic Beat status is Red" + inputPath: testdata/ready_red.yaml +- healthStatus: + status: Unknown + message: "Elastic Beat status is unknown. Ensure your ArgoCD is current and then check for/file a bug report: https://github.com/argoproj/argo-cd/issues" + inputPath: testdata/unknown.yaml +- healthStatus: + status: Unknown + message: "Elastic Beat status is unknown. Ensure your ArgoCD is current and then check for/file a bug report: https://github.com/argoproj/argo-cd/issues" + inputPath: testdata/invalid.yaml diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/testdata/invalid.yaml b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/invalid.yaml new file mode 100644 index 0000000000000..3eca183165a5c --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/invalid.yaml @@ -0,0 +1,12 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: quickstart +spec: + version: 8.8.8 + type: metricbeat +status: + expectedNodes: 1 + health: invalid + observedGeneration: 1 + version: 8.8.1 diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/testdata/progressing.yaml b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/progressing.yaml new file mode 100644 index 0000000000000..b007ad72ae3fe --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/progressing.yaml @@ -0,0 +1,11 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: quickstart +spec: + version: 8.8.8 + type: metricbeat +status: + expectedNodes: 1 + observedGeneration: 1 + version: 8.8.1 diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_green.yaml b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_green.yaml new file mode 100644 index 0000000000000..3f3c1866793d8 --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_green.yaml @@ -0,0 +1,13 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: quickstart +spec: + version: 8.8.8 + type: metricbeat +status: + expectedNodes: 1 + availableNodes: 1 + health: green + observedGeneration: 1 + version: 8.8.1 diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_red.yaml b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_red.yaml new file mode 100644 index 0000000000000..fc2433c8076a8 --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_red.yaml @@ -0,0 +1,10 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: quickstart +spec: + version: 8.8.8 + type: metricbeat +status: + expectedNodes: 1 + health: red diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_yellow.yaml b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_yellow.yaml new file mode 100644 index 0000000000000..831ee281ef02d --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_yellow.yaml @@ -0,0 +1,11 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: quickstart +spec: + version: 8.8.8 + type: metricbeat +status: + availableNodes: 1 + expectedNodes: 2 + health: yellow diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_yellow_single_node.yaml b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_yellow_single_node.yaml new file mode 100644 index 0000000000000..d652b5a55d0ff --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/ready_yellow_single_node.yaml @@ -0,0 +1,10 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: quickstart +spec: + version: 8.8.8 + type: metricbeat +status: + expectedNodes: 1 + health: yellow diff --git a/resource_customizations/beat.k8s.elastic.co/Beat/testdata/unknown.yaml b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/unknown.yaml new file mode 100644 index 0000000000000..dbcca36c9e691 --- /dev/null +++ b/resource_customizations/beat.k8s.elastic.co/Beat/testdata/unknown.yaml @@ -0,0 +1,8 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: quickstart +spec: + version: 8.8.8 + type: metricbeat +status: {} diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/health.lua b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/health.lua new file mode 100644 index 0000000000000..3e07226b3cf89 --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/health.lua @@ -0,0 +1,42 @@ +local hs = {} +if obj.status ~= nil then + if obj.status.conditions ~= nil then + local ready = false + local synced = false + local suspended = false + + for i, condition in ipairs(obj.status.conditions) do + + if condition.type == "Ready" then + ready = condition.status == "True" + ready_message = condition.reason + elseif condition.type == "Synced" then + synced = condition.status == "True" + if condition.reason == "ReconcileError" then + synced_message = condition.message + elseif condition.reason == "ReconcilePaused" then + suspended = true + suspended_message = condition.reason + end + end + end + if ready and synced then + hs.status = "Healthy" + hs.message = ready_message + elseif synced == false and suspended == true then + hs.status = "Suspended" + hs.message = suspended_message + elseif ready == false and synced == true and suspended == false then + hs.status = "Progressing" + hs.message = "Waiting for distribution to be available" + else + hs.status = "Degraded" + hs.message = synced_message + end + return hs + end +end + +hs.status = "Progressing" +hs.message = "Waiting for distribution to be created" +return hs \ No newline at end of file diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/health_test.yaml b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/health_test.yaml new file mode 100644 index 0000000000000..981a6000ecb88 --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/health_test.yaml @@ -0,0 +1,37 @@ +tests: +- healthStatus: + status: Progressing + message: Waiting for distribution to be available + inputPath: testdata/progressing_creating.yaml +- healthStatus: + status: Progressing + message: Waiting for distribution to be available + inputPath: testdata/progressing_noavailable.yaml +- healthStatus: + status: Progressing + message: Waiting for distribution to be available + inputPath: testdata/progressing.yaml +- healthStatus: + status: Progressing + message: Waiting for distribution to be created + inputPath: testdata/progressing_noStatus.yaml +- healthStatus: + status: Degraded + message: > + update failed: cannot update Distribution in AWS: InvalidParameter: 2 + validation error(s) found. + + - missing required field, + UpdateDistributionInput.DistributionConfig.Origins.Items[0].DomainName. + + - missing required field, + UpdateDistributionInput.DistributionConfig.Origins.Items[0].Id. + inputPath: testdata/degraded_reconcileError.yaml +- healthStatus: + status: Suspended + message: ReconcilePaused + inputPath: testdata/suspended.yaml +- healthStatus: + status: Healthy + message: Available + inputPath: testdata/healthy.yaml diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/degraded_reconcileError.yaml b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/degraded_reconcileError.yaml new file mode 100644 index 0000000000000..80ea7930574ac --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/degraded_reconcileError.yaml @@ -0,0 +1,96 @@ +apiVersion: cloudfront.aws.crossplane.io/v1alpha1 +kind: Distribution +metadata: + creationTimestamp: '2024-01-17T07:26:02Z' + generation: 2 + name: crossplane.io + resourceVersion: '261942288' + uid: 4b50c88b-165c-4176-be8e-aa28fdec0a94 +spec: + deletionPolicy: Orphan + forProvider: + distributionConfig: + comment: 'crossplane' + customErrorResponses: + items: [] + defaultCacheBehavior: + allowedMethods: + cachedMethods: + items: + - HEAD + - GET + items: + - HEAD + - GET + compress: false + defaultTTL: 600 + fieldLevelEncryptionID: '' + forwardedValues: + cookies: + forward: none + headers: + items: [] + queryString: false + queryStringCacheKeys: {} + functionAssociations: {} + lambdaFunctionAssociations: {} + maxTTL: 600 + minTTL: 0 + smoothStreaming: false + targetOriginID: crossplane.io + trustedKeyGroups: + enabled: false + trustedSigners: + enabled: false + viewerProtocolPolicy: allow-all + defaultRootObject: index.html + enabled: true + httpVersion: http2 + isIPV6Enabled: true + logging: + bucket: '' + enabled: false + includeCookies: false + prefix: '' + originGroups: {} + origins: + items: + - connectionAttempts: 3 + connectionTimeout: 10 + customOriginConfig: + httpPort: 8080 + httpSPort: 443 + originKeepaliveTimeout: 5 + originProtocolPolicy: http-only + originReadTimeout: 10 + originSSLProtocols: + items: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + priceClass: PriceClass_200 + restrictions: + geoRestriction: + restrictionType: none + region: ap-northeast-2 + providerConfigRef: + name: crossplane +status: + conditions: + - lastTransitionTime: '2024-01-17T07:26:02Z' + message: > + update failed: cannot update Distribution in AWS: InvalidParameter: 2 + validation error(s) found. + + - missing required field, + UpdateDistributionInput.DistributionConfig.Origins.Items[0].DomainName. + + - missing required field, + UpdateDistributionInput.DistributionConfig.Origins.Items[0].Id. + reason: ReconcileError + status: 'False' + type: Synced + - lastTransitionTime: '2024-01-17T07:26:03Z' + reason: Available + status: 'True' + type: Ready diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/healthy.yaml b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/healthy.yaml new file mode 100644 index 0000000000000..23d0287445e83 --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/healthy.yaml @@ -0,0 +1,92 @@ +apiVersion: cloudfront.aws.crossplane.io/v1alpha1 +kind: Distribution +metadata: + creationTimestamp: "2023-09-07T01:01:16Z" + generation: 121 + name: crossplane.io + resourceVersion: "254225966" + uid: 531d989c-a3d2-4ab4-841d-ab380cce0bdb +spec: + deletionPolicy: Orphan + forProvider: + distributionConfig: + comment: 'crossplane' + customErrorResponses: + items: [] + defaultCacheBehavior: + allowedMethods: + cachedMethods: + items: + - HEAD + - GET + items: + - HEAD + - GET + compress: false + defaultTTL: 600 + fieldLevelEncryptionID: '' + forwardedValues: + cookies: + forward: none + headers: + items: [] + queryString: false + queryStringCacheKeys: {} + functionAssociations: {} + lambdaFunctionAssociations: {} + maxTTL: 600 + minTTL: 0 + smoothStreaming: false + targetOriginID: crossplane.io + trustedKeyGroups: + enabled: false + trustedSigners: + enabled: false + viewerProtocolPolicy: allow-all + defaultRootObject: index.html + enabled: true + httpVersion: http2 + isIPV6Enabled: true + logging: + bucket: '' + enabled: false + includeCookies: false + prefix: '' + originGroups: {} + origins: + items: + - connectionAttempts: 3 + connectionTimeout: 10 + customHeaders: {} + customOriginConfig: + httpPort: 8080 + httpSPort: 443 + originKeepaliveTimeout: 5 + originProtocolPolicy: http-only + originReadTimeout: 10 + originSSLProtocols: + items: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + domainName: crossplane.io + id: crossplane.io + originShield: + enabled: false + priceClass: PriceClass_200 + restrictions: + geoRestriction: + restrictionType: none + region: ap-northeast-2 + providerConfigRef: + name: crossplane +status: + conditions: + - lastTransitionTime: "2024-01-11T06:23:18Z" + reason: ReconcileSuccess + status: "True" + type: Synced + - lastTransitionTime: "2024-01-10T03:23:02Z" + reason: Available + status: "True" + type: Ready diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing.yaml b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing.yaml new file mode 100644 index 0000000000000..3dbde7e040867 --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing.yaml @@ -0,0 +1,92 @@ +apiVersion: cloudfront.aws.crossplane.io/v1alpha1 +kind: Distribution +metadata: + creationTimestamp: '2023-06-16T04:42:04Z' + generation: 37 + name: crossplane.io + resourceVersion: '254326453' + uid: fd357670-b762-4285-ae83-00859c40dd6b +spec: + deletionPolicy: Orphan + forProvider: + distributionConfig: + comment: 'crossplane' + customErrorResponses: + items: [] + defaultCacheBehavior: + allowedMethods: + cachedMethods: + items: + - HEAD + - GET + items: + - GET + - HEAD + compress: false + defaultTTL: 600 + fieldLevelEncryptionID: "" + forwardedValues: + cookies: + forward: none + headers: + items: [] + queryString: false + queryStringCacheKeys: {} + functionAssociations: {} + lambdaFunctionAssociations: {} + maxTTL: 600 + minTTL: 0 + smoothStreaming: false + targetOriginID: crossplane.io + trustedKeyGroups: + enabled: false + trustedSigners: + enabled: false + viewerProtocolPolicy: allow-all + defaultRootObject: index.html + enabled: true + httpVersion: http2 + isIPV6Enabled: true + logging: + bucket: "" + enabled: false + includeCookies: false + prefix: "" + originGroups: {} + origins: + items: + - connectionAttempts: 3 + connectionTimeout: 10 + customHeaders: {} + customOriginConfig: + httpPort: 8080 + httpSPort: 443 + originKeepaliveTimeout: 5 + originProtocolPolicy: http-only + originReadTimeout: 10 + originSSLProtocols: + items: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + domainName: crossplane.io + id: crossplane.io + originShield: + enabled: false + priceClass: PriceClass_200 + restrictions: + geoRestriction: + restrictionType: none + region: ap-northeast-2 + providerConfigRef: + name: crossplane +status: + conditions: + - lastTransitionTime: '2024-01-11T08:11:27Z' + reason: Unavailable + status: 'False' + type: Ready + - lastTransitionTime: '2024-01-11T08:11:02Z' + reason: ReconcileSuccess + status: 'True' + type: Synced diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_creating.yaml b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_creating.yaml new file mode 100644 index 0000000000000..122ab330d593b --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_creating.yaml @@ -0,0 +1,92 @@ +apiVersion: cloudfront.aws.crossplane.io/v1alpha1 +kind: Distribution +metadata: + creationTimestamp: "2023-09-07T01:01:16Z" + generation: 121 + name: crossplane.io + resourceVersion: "254225966" + uid: 531d989c-a3d2-4ab4-841d-ab380cce0bdb +spec: + deletionPolicy: Orphan + forProvider: + distributionConfig: + comment: 'crossplane' + customErrorResponses: + items: [] + defaultCacheBehavior: + allowedMethods: + cachedMethods: + items: + - HEAD + - GET + items: + - GET + - HEAD + compress: false + defaultTTL: 600 + fieldLevelEncryptionID: "" + forwardedValues: + cookies: + forward: none + headers: + items: [] + queryString: false + queryStringCacheKeys: {} + functionAssociations: {} + lambdaFunctionAssociations: {} + maxTTL: 600 + minTTL: 0 + smoothStreaming: false + targetOriginID: crossplane.io + trustedKeyGroups: + enabled: false + trustedSigners: + enabled: false + viewerProtocolPolicy: allow-all + defaultRootObject: index.html + enabled: true + httpVersion: http2 + isIPV6Enabled: true + logging: + bucket: "" + enabled: false + includeCookies: false + prefix: "" + originGroups: {} + origins: + items: + - connectionAttempts: 3 + connectionTimeout: 10 + customHeaders: {} + customOriginConfig: + httpPort: 8080 + httpSPort: 443 + originKeepaliveTimeout: 5 + originProtocolPolicy: http-only + originReadTimeout: 10 + originSSLProtocols: + items: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + domainName: crossplane.io + id: crossplane.io + originShield: + enabled: false + priceClass: PriceClass_200 + restrictions: + geoRestriction: + restrictionType: none + region: ap-northeast-2 + providerConfigRef: + name: crossplane +status: + conditions: + - lastTransitionTime: "2023-11-16T04:44:27Z" + reason: Creating + status: "False" + type: Ready + - lastTransitionTime: "2023-11-16T04:44:25Z" + reason: ReconcileSuccess + status: "True" + type: Synced diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_noStatus.yaml b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_noStatus.yaml new file mode 100644 index 0000000000000..2985ec2dea657 --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_noStatus.yaml @@ -0,0 +1,82 @@ +apiVersion: cloudfront.aws.crossplane.io/v1alpha1 +kind: Distribution +metadata: + creationTimestamp: "2023-09-07T01:01:16Z" + generation: 121 + name: crossplane.io + resourceVersion: "254225966" + uid: 531d989c-a3d2-4ab4-841d-ab380cce0bdb +spec: + deletionPolicy: Orphan + forProvider: + distributionConfig: + comment: 'crossplane' + customErrorResponses: + items: [] + defaultCacheBehavior: + allowedMethods: + cachedMethods: + items: + - HEAD + - GET + items: + - GET + - HEAD + compress: false + defaultTTL: 600 + fieldLevelEncryptionID: "" + forwardedValues: + cookies: + forward: none + headers: + items: [] + queryString: false + queryStringCacheKeys: {} + functionAssociations: {} + lambdaFunctionAssociations: {} + maxTTL: 600 + minTTL: 0 + smoothStreaming: false + targetOriginID: crossplane.io + trustedKeyGroups: + enabled: false + trustedSigners: + enabled: false + viewerProtocolPolicy: allow-all + defaultRootObject: index.html + enabled: true + httpVersion: http2 + isIPV6Enabled: true + logging: + bucket: "" + enabled: false + includeCookies: false + prefix: "" + originGroups: {} + origins: + items: + - connectionAttempts: 3 + connectionTimeout: 10 + customHeaders: {} + customOriginConfig: + httpPort: 8080 + httpSPort: 443 + originKeepaliveTimeout: 5 + originProtocolPolicy: http-only + originReadTimeout: 10 + originSSLProtocols: + items: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + domainName: crossplane.io + id: crossplane.io + originShield: + enabled: false + priceClass: PriceClass_200 + restrictions: + geoRestriction: + restrictionType: none + region: ap-northeast-2 + providerConfigRef: + name: crossplane diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_noavailable.yaml b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_noavailable.yaml new file mode 100644 index 0000000000000..7a47b0f48eea7 --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/progressing_noavailable.yaml @@ -0,0 +1,88 @@ +apiVersion: cloudfront.aws.crossplane.io/v1alpha1 +kind: Distribution +metadata: + generation: 1 + name: crossplane.io + resourceVersion: "261937039" + uid: a52c105f-b0e1-4027-aa19-7e93f269f2a6 +spec: + deletionPolicy: Orphan + forProvider: + distributionConfig: + comment: 'crossplane' + customErrorResponses: + items: [] + defaultCacheBehavior: + allowedMethods: + cachedMethods: + items: + - HEAD + - GET + items: + - GET + - HEAD + compress: false + defaultTTL: 600 + fieldLevelEncryptionID: "" + forwardedValues: + cookies: + forward: none + headers: + items: [] + queryString: false + queryStringCacheKeys: {} + functionAssociations: {} + lambdaFunctionAssociations: {} + maxTTL: 600 + minTTL: 0 + smoothStreaming: false + targetOriginID: crossplane.io + trustedKeyGroups: + enabled: false + trustedSigners: + enabled: false + viewerProtocolPolicy: allow-all + defaultRootObject: index.html + enabled: true + httpVersion: http2 + isIPV6Enabled: true + logging: + bucket: "" + enabled: false + includeCookies: false + prefix: "" + originGroups: {} + origins: + items: + - connectionAttempts: 3 + connectionTimeout: 10 + customHeaders: {} + customOriginConfig: + httpPort: 8080 + httpSPort: 443 + originKeepaliveTimeout: 5 + originProtocolPolicy: http-only + originReadTimeout: 10 + originSSLProtocols: + items: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + domainName: crossplane.io + id: crossplane.io + originShield: + enabled: false + priceClass: PriceClass_200 + restrictions: + geoRestriction: + restrictionType: none + region: ap-northeast-2 + providerConfigRef: + name: crossplane +status: + atProvider: {} + conditions: + - lastTransitionTime: "2024-01-17T07:20:35Z" + reason: ReconcileSuccess + status: "True" + type: Synced diff --git a/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/suspended.yaml b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/suspended.yaml new file mode 100644 index 0000000000000..d15713737ff72 --- /dev/null +++ b/resource_customizations/cloudfront.aws.crossplane.io/Distribution/testdata/suspended.yaml @@ -0,0 +1,94 @@ +apiVersion: cloudfront.aws.crossplane.io/v1alpha1 +kind: Distribution +metadata: + annotations: + crossplane.io/paused: "true" + creationTimestamp: "2023-06-16T04:42:04Z" + generation: 34 + name: crossplane.io + resourceVersion: "254259056" + uid: fd357670-b762-4285-ae83-00859c40dd6b +spec: + deletionPolicy: Orphan + forProvider: + distributionConfig: + comment: 'crossplane' + customErrorResponses: + items: [] + defaultCacheBehavior: + allowedMethods: + cachedMethods: + items: + - HEAD + - GET + items: + - GET + - HEAD + compress: false + defaultTTL: 600 + fieldLevelEncryptionID: "" + forwardedValues: + cookies: + forward: none + headers: + items: [] + queryString: false + queryStringCacheKeys: {} + functionAssociations: {} + lambdaFunctionAssociations: {} + maxTTL: 600 + minTTL: 0 + smoothStreaming: false + targetOriginID: crossplane.io + trustedKeyGroups: + enabled: false + trustedSigners: + enabled: false + viewerProtocolPolicy: allow-all + defaultRootObject: index.html + enabled: true + httpVersion: http2 + isIPV6Enabled: true + logging: + bucket: "" + enabled: false + includeCookies: false + prefix: "" + originGroups: {} + origins: + items: + - connectionAttempts: 3 + connectionTimeout: 10 + customHeaders: {} + customOriginConfig: + httpPort: 8080 + httpSPort: 443 + originKeepaliveTimeout: 5 + originProtocolPolicy: http-only + originReadTimeout: 10 + originSSLProtocols: + items: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + domainName: crossplane.io + id: crossplane.io + originShield: + enabled: false + priceClass: PriceClass_200 + restrictions: + geoRestriction: + restrictionType: none + region: ap-northeast-2 + providerConfigRef: + name: crossplane +status: + conditions: + - lastTransitionTime: "2023-10-16T07:40:47Z" + reason: Available + status: "True" + type: Ready + - lastTransitionTime: "2024-01-11T06:59:47Z" + reason: ReconcilePaused + status: "False" + type: Synced diff --git a/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/health_test.yaml b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/health_test.yaml new file mode 100644 index 0000000000000..aa83951d5a2db --- /dev/null +++ b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/health_test.yaml @@ -0,0 +1,25 @@ +tests: +- healthStatus: + status: Progressing + message: Waiting for resourcrecordset to be available + inputPath: testdata/progressing_creating.yaml +- healthStatus: + status: Progressing + message: Waiting for resourcrecordset to be created + inputPath: testdata/progressing_noStatus.yaml +- healthStatus: + status: Degraded + message: >- + create failed: failed to create the ResourceRecordSet resource: + InvalidChangeBatch: [RRSet of type CNAME with DNS name + www.crossplane.io. is not permitted as it conflicts with other + records with the same DNS name in zone crossplane.io.] + inputPath: testdata/degraded_reconcileError.yaml +- healthStatus: + status: Suspended + message: ReconcilePaused + inputPath: testdata/suspended_reconcilePaused.yaml +- healthStatus: + status: Healthy + message: Available + inputPath: testdata/healthy.yaml diff --git a/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/heatlh.lua b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/heatlh.lua new file mode 100644 index 0000000000000..0cf5253e910ff --- /dev/null +++ b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/heatlh.lua @@ -0,0 +1,41 @@ +local hs = {} +if obj.status ~= nil then + if obj.status.conditions ~= nil then + local ready = false + local synced = false + local suspended = false + for i, condition in ipairs(obj.status.conditions) do + + if condition.type == "Ready" then + ready = condition.status == "True" + ready_message = condition.reason + elseif condition.type == "Synced" then + synced = condition.status == "True" + if condition.reason == "ReconcileError" then + synced_message = condition.message + elseif condition.reason == "ReconcilePaused" then + suspended = true + suspended_message = condition.reason + end + end + end + if ready and synced then + hs.status = "Healthy" + hs.message = ready_message + elseif synced == false and suspended == true then + hs.status = "Suspended" + hs.message = suspended_message + elseif ready == false and synced == true and suspended == false then + hs.status = "Progressing" + hs.message = "Waiting for resourcrecordset to be available" + else + hs.status = "Degraded" + hs.message = synced_message + end + return hs + end +end + +hs.status = "Progressing" +hs.message = "Waiting for resourcrecordset to be created" +return hs diff --git a/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/degraded_reconcileError.yaml b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/degraded_reconcileError.yaml new file mode 100644 index 0000000000000..31bc5123c7bfd --- /dev/null +++ b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/degraded_reconcileError.yaml @@ -0,0 +1,35 @@ +apiVersion: route53.aws.crossplane.io/v1alpha1 +kind: ResourceRecordSet +metadata: + creationTimestamp: '2024-01-11T03:48:32Z' + generation: 1 + name: www-domain + resourceVersion: '187731157' + selfLink: /apis/route53.aws.crossplane.io/v1alpha1/resourcerecordsets/www-domain + uid: c9c85395-0830-4549-b255-e9e426663547 +spec: + providerConfigRef: + name: crossplane + forProvider: + resourceRecords: + - value: www.crossplane.io + setIdentifier: www + ttl: 60 + type: CNAME + weight: 0 + zoneId: ABCDEFGAB07CD +status: + conditions: + - lastTransitionTime: '2024-01-11T03:48:57Z' + message: >- + create failed: failed to create the ResourceRecordSet resource: + InvalidChangeBatch: [RRSet of type CNAME with DNS name + www.crossplane.io. is not permitted as it conflicts with other + records with the same DNS name in zone crossplane.io.] + reason: ReconcileError + status: 'False' + type: Synced + - lastTransitionTime: '2024-01-11T03:48:34Z' + reason: Creating + status: 'False' + type: Ready diff --git a/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/healthy.yaml b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/healthy.yaml new file mode 100644 index 0000000000000..f808e46cc8c92 --- /dev/null +++ b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/healthy.yaml @@ -0,0 +1,29 @@ +apiVersion: route53.aws.crossplane.io/v1alpha1 +kind: ResourceRecordSet +metadata: + creationTimestamp: "2023-11-16T04:44:19Z" + generation: 4 + name: www-domain + resourceVersion: "140397563" + selfLink: /apis/route53.aws.crossplane.io/v1alpha1/resourcerecordsets/www-domain + uid: 11f0d48d-134f-471b-9340-b6d45d953fcb +spec: + providerConfigRef: + name: crossplane + forProvider: + zoneId: A1B2C3D4 + type: A + aliasTarget: + dnsName: abcdefg.cloudfront.net. + evaluateTargetHealth: false + hostedZoneId: AZBZCZDEFG +status: + conditions: + - lastTransitionTime: "2023-11-16T04:44:27Z" + reason: Available + status: "True" + type: Ready + - lastTransitionTime: "2023-11-16T04:44:25Z" + reason: ReconcileSuccess + status: "True" + type: Synced diff --git a/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/progressing_creating.yaml b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/progressing_creating.yaml new file mode 100644 index 0000000000000..abf59775fb8e0 --- /dev/null +++ b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/progressing_creating.yaml @@ -0,0 +1,29 @@ +apiVersion: route53.aws.crossplane.io/v1alpha1 +kind: ResourceRecordSet +metadata: + creationTimestamp: "2023-11-16T04:44:19Z" + generation: 4 + name: www-domain + resourceVersion: "140397563" + selfLink: /apis/route53.aws.crossplane.io/v1alpha1/resourcerecordsets/www-domain + uid: 11f0d48d-134f-471b-9340-b6d45d953fcb +spec: + providerConfigRef: + name: crossplane + forProvider: + zoneId: A1B2C3D4 + type: A + aliasTarget: + dnsName: abcdefg.cloudfront.net. + evaluateTargetHealth: false + hostedZoneId: AZBZCZDEFG +status: + conditions: + - lastTransitionTime: "2023-11-16T04:44:27Z" + reason: Creating + status: "False" + type: Ready + - lastTransitionTime: "2023-11-16T04:44:25Z" + reason: ReconcileSuccess + status: "True" + type: Synced diff --git a/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/progressing_noStatus.yaml b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/progressing_noStatus.yaml new file mode 100644 index 0000000000000..28d778d055050 --- /dev/null +++ b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/progressing_noStatus.yaml @@ -0,0 +1,19 @@ +apiVersion: route53.aws.crossplane.io/v1alpha1 +kind: ResourceRecordSet +metadata: + creationTimestamp: "2023-11-16T04:44:19Z" + generation: 4 + name: www-domain + resourceVersion: "140397563" + selfLink: /apis/route53.aws.crossplane.io/v1alpha1/resourcerecordsets/www-domain + uid: 11f0d48d-134f-471b-9340-b6d45d953fcb +spec: + providerConfigRef: + name: crossplane + forProvider: + zoneId: A1B2C3D4 + type: A + aliasTarget: + dnsName: abcdefg.cloudfront.net. + evaluateTargetHealth: false + hostedZoneId: AZBZCZDEFG diff --git a/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/suspended_reconcilePaused.yaml b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/suspended_reconcilePaused.yaml new file mode 100644 index 0000000000000..522c0e878dcf8 --- /dev/null +++ b/resource_customizations/route53.aws.crossplane.io/ResourceRecordSet/testdata/suspended_reconcilePaused.yaml @@ -0,0 +1,27 @@ +apiVersion: route53.aws.crossplane.io/v1alpha1 +kind: ResourceRecordSet +metadata: + annotations: + crossplane.io/paused: "true" + creationTimestamp: "2024-01-11T04:16:15Z" + generation: 1 + name: www-domain + resourceVersion: "187746011" + uid: 5517b419-5052-43d9-941e-c32f60d8c7e5 +spec: + providerConfigRef: + name: crossplane + forProvider: + resourceRecords: + - value: www.crossplane.io + setIdentifier: www + ttl: 60 + type: CNAME + weight: 0 + zoneId: ABCDEFGAB07CD +status: + conditions: + - lastTransitionTime: "2024-01-11T04:16:16Z" + reason: ReconcilePaused + status: "False" + type: Synced diff --git a/server/badge/badge.go b/server/badge/badge.go index cf291d589501e..5787d530c15f7 100644 --- a/server/badge/badge.go +++ b/server/badge/badge.go @@ -42,10 +42,28 @@ var ( leftTextPattern = regexp.MustCompile(`id="leftText" [^>]*>([^<]*)`) rightTextPattern = regexp.MustCompile(`id="rightText" [^>]*>([^<]*)`) revisionTextPattern = regexp.MustCompile(`id="revisionText" [^>]*>([^<]*)`) + titleTextPattern = regexp.MustCompile(`id="titleText" [^>]*>([^<]*)`) + titleRectWidthPattern = regexp.MustCompile(`(id="titleRect" .* width=)("0")`) + rightRectWidthPattern = regexp.MustCompile(`(id="rightRect" .* width=)("\d*")`) + leftRectYCoodPattern = regexp.MustCompile(`(id="leftRect" .* y=)("\d*")`) + rightRectYCoodPattern = regexp.MustCompile(`(id="rightRect" .* y=)("\d*")`) + revisionRectYCoodPattern = regexp.MustCompile(`(id="revisionRect" .* y=)("\d*")`) + leftTextYCoodPattern = regexp.MustCompile(`(id="leftText" .* y=)("\d*")`) + rightTextYCoodPattern = regexp.MustCompile(`(id="rightText" .* y=)("\d*")`) + revisionTextYCoodPattern = regexp.MustCompile(`(id="revisionText" .* y=)("\d*")`) + svgHeightPattern = regexp.MustCompile(`^( 0 { + handler = enforceContentTypes(handler, a.ContentTypes) + } else { + log.WithField(common.SecurityField, common.SecurityHigh).Warnf("Content-Type enforcement is disabled, which may make your API vulnerable to CSRF attacks") + } mux.Handle("/api/", handler) terminal := application.NewHandler(a.appLister, a.Namespace, a.ApplicationNamespaces, a.db, a.enf, a.Cache, appResourceTreeFn, a.settings.ExecShells, *a.sessionMgr). @@ -1003,7 +1022,7 @@ func (a *ArgoCDServer) newHTTPServer(ctx context.Context, port int, grpcWebHandl // API server won't panic if extensions fail to register. In // this case an error log will be sent and no extension route // will be added in mux. - registerExtensions(mux, a) + registerExtensions(mux, a, metricsReg) } mustRegisterGWHandler(versionpkg.RegisterVersionServiceHandler, ctx, gwmux, conn) @@ -1056,16 +1075,32 @@ func (a *ArgoCDServer) newHTTPServer(ctx context.Context, port int, grpcWebHandl return &httpS } +func enforceContentTypes(handler http.Handler, types []string) http.Handler { + allowedTypes := map[string]bool{} + for _, t := range types { + allowedTypes[strings.ToLower(t)] = true + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet || allowedTypes[strings.ToLower(r.Header.Get("Content-Type"))] { + handler.ServeHTTP(w, r) + } else { + http.Error(w, "Invalid content type", http.StatusUnsupportedMediaType) + } + }) +} + // registerExtensions will try to register all configured extensions // in the given mux. If any error is returned while registering // extensions handlers, no route will be added in the given mux. -func registerExtensions(mux *http.ServeMux, a *ArgoCDServer) { +func registerExtensions(mux *http.ServeMux, a *ArgoCDServer, metricsReg HTTPMetricsRegistry) { a.log.Info("Registering extensions...") extHandler := http.HandlerFunc(a.extensionManager.CallExtension()) authMiddleware := a.sessionMgr.AuthMiddlewareFunc(a.DisableAuth) // auth middleware ensures that requests to all extensions are authenticated first mux.Handle(fmt.Sprintf("%s/", extension.URLPrefix), authMiddleware(extHandler)) + a.extensionManager.AddMetricsRegistry(metricsReg) + err := a.extensionManager.RegisterExtensions() if err != nil { a.log.Errorf("Error registering extensions: %s", err) diff --git a/server/server_test.go b/server/server_test.go index acfb32e57e5d4..c4f4153f24d89 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1526,3 +1526,46 @@ func TestReplaceBaseHRef(t *testing.T) { }) } } + +func Test_enforceContentTypes(t *testing.T) { + getBaseHandler := func(t *testing.T, allow bool) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + assert.True(t, allow, "http handler was hit when it should have been blocked by content type enforcement") + writer.WriteHeader(200) + }) + } + + t.Parallel() + + t.Run("GET - not providing a content type, should still succeed", func(t *testing.T) { + handler := enforceContentTypes(getBaseHandler(t, true), []string{"application/json"}).(http.HandlerFunc) + req := httptest.NewRequest("GET", "/", nil) + w := httptest.NewRecorder() + handler(w, req) + resp := w.Result() + assert.Equal(t, 200, resp.StatusCode) + }) + + t.Run("POST", func(t *testing.T) { + handler := enforceContentTypes(getBaseHandler(t, true), []string{"application/json"}).(http.HandlerFunc) + req := httptest.NewRequest("POST", "/", nil) + w := httptest.NewRecorder() + handler(w, req) + resp := w.Result() + assert.Equal(t, 415, resp.StatusCode, "didn't provide a content type, should have gotten an error") + + req = httptest.NewRequest("POST", "/", nil) + req.Header = map[string][]string{"Content-Type": {"application/json"}} + w = httptest.NewRecorder() + handler(w, req) + resp = w.Result() + assert.Equal(t, 200, resp.StatusCode, "should have passed, since an allowed content type was provided") + + req = httptest.NewRequest("POST", "/", nil) + req.Header = map[string][]string{"Content-Type": {"not-allowed"}} + w = httptest.NewRecorder() + handler(w, req) + resp = w.Result() + assert.Equal(t, 415, resp.StatusCode, "should not have passed, since a disallowed content type was provided") + }) +} diff --git a/test/container/Dockerfile b/test/container/Dockerfile index c86fbb1f387b1..8258be1af72aa 100644 --- a/test/container/Dockerfile +++ b/test/container/Dockerfile @@ -1,20 +1,20 @@ -FROM docker.io/library/redis:7.0.11@sha256:f50031a49f41e493087fb95f96fdb3523bb25dcf6a3f0b07c588ad3cdbe1d0aa as redis +FROM docker.io/library/redis:7.2.4@sha256:e647cfe134bf5e8e74e620f66346f93418acfc240b71dd85640325cb7cd01402 as redis # There are libraries we will want to copy from here in the final stage of the # build, but the COPY directive does not have a way to determine system # architecture, so we create a symlink here to facilitate copying. -RUN ln -s /usr/lib/$(uname -m)-linux-gnu /usr/lib/linux-gnu +RUN ln -s /usr/lib/$(uname -m)-linux-gnu /usr/lib/linux-gnu # Please make sure to also check the contained yarn version and update the references below when upgrading this image's version -FROM docker.io/library/node:20.7.0@sha256:f08c20b9f9c55dd47b1841793f0ee480c5395aa165cd02edfd68b068ed64bfb5 as node +FROM docker.io/library/node:21.6.2@sha256:65998e325b06014d4f1417a8a6afb1540d1ac66521cca76f2221a6953947f9ee as node FROM docker.io/library/golang:1.21.3@sha256:02d7116222536a5cf0fcf631f90b507758b669648e0f20186d2dc94a9b419a9b as golang -FROM docker.io/library/registry:2.8@sha256:41f413c22d6156587e2a51f3e80c09808b8c70e82be149b82b5e0196a88d49b4 as registry +FROM docker.io/library/registry:2.8@sha256:f4e1b878d4bc40a1f65532d68c94dcfbab56aa8cba1f00e355a206e7f6cc9111 as registry -FROM docker.io/bitnami/kubectl:1.27@sha256:670fe3f50d45c0511bb0f2af018e2fc082ac8cdfaea02dba4e32866296036926 as kubectl +FROM docker.io/bitnami/kubectl:1.27@sha256:14ab746e857d96c105df4989cc2bf841292f2d143f7c60f9d7f549ae660eab43 as kubectl -FROM docker.io/library/ubuntu:22.04@sha256:0bced47fffa3361afa981854fcabcd4577cd43cebbb808cea2b1f33a3dd7f508 +FROM docker.io/library/ubuntu:22.04@sha256:f9d633ff6640178c2d0525017174a688e2c1aef28f0a0130b26bd5554491f0da ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install --fix-missing -y \ @@ -72,10 +72,10 @@ COPY --from=redis /usr/local/bin/* /usr/local/bin/ # Copy redis dependencies/shared libraries # Ubuntu 22.04+ has moved to OpenSSL3 and no longer provides these libraries -COPY --from=redis /usr/lib/linux-gnu/libssl.so.1.1 /usr/lib/linux-gnu/ -COPY --from=redis /usr/lib/linux-gnu/libcrypto.so.1.1 /usr/lib/linux-gnu/ -RUN mv /usr/lib/linux-gnu/libssl.so.1.1 /usr/lib/$(uname -m)-linux-gnu/ && \ - mv /usr/lib/linux-gnu/libcrypto.so.1.1 /usr/lib/$(uname -m)-linux-gnu/ && \ +COPY --from=redis /usr/lib/linux-gnu/libssl.so.3 /usr/lib/linux-gnu/ +COPY --from=redis /usr/lib/linux-gnu/libcrypto.so.3 /usr/lib/linux-gnu/ +RUN mv /usr/lib/linux-gnu/libssl.so.3 /usr/lib/$(uname -m)-linux-gnu/ && \ + mv /usr/lib/linux-gnu/libcrypto.so.3 /usr/lib/$(uname -m)-linux-gnu/ && \ rm -rf /usr/lib/linux-gnu/ # Copy registry binaries to the image diff --git a/test/container/Procfile b/test/container/Procfile index ef5100e71bab3..3ec9add44d5a7 100644 --- a/test/container/Procfile +++ b/test/container/Procfile @@ -1,6 +1,6 @@ controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-application-controller $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''}" api-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_BINARY_NAME=argocd-server $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --disable-auth=${ARGOCD_E2E_DISABLE_AUTH:-'true'} --insecure --dex-server http://localhost:${ARGOCD_E2E_DEX_PORT:-5556} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --port ${ARGOCD_E2E_APISERVER_PORT:-8080} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''} " -dex: sh -c "test $ARGOCD_IN_CI = true && exit 0; ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/cmd gendexcfg -o `pwd`/dist/dex.yaml && docker run --rm -p ${ARGOCD_E2E_DEX_PORT:-5556}:${ARGOCD_E2E_DEX_PORT:-5556} -v `pwd`/dist/dex.yaml:/dex.yaml ghcr.io/dexidp/dex:v2.37.0 serve /dex.yaml" +dex: sh -c "test $ARGOCD_IN_CI = true && exit 0; ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/cmd gendexcfg -o `pwd`/dist/dex.yaml && docker run --rm -p ${ARGOCD_E2E_DEX_PORT:-5556}:${ARGOCD_E2E_DEX_PORT:-5556} -v `pwd`/dist/dex.yaml:/dex.yaml ghcr.io/dexidp/dex:v2.38.0 serve /dex.yaml" redis: sh -c "/usr/local/bin/redis-server --save "" --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}" repo-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_GNUPGHOME=${ARGOCD_GNUPGHOME:-/tmp/argocd-local/gpg/keys} ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} ARGOCD_GPG_DATA_PATH=${ARGOCD_GPG_DATA_PATH:-/tmp/argocd-local/gpg/source} ARGOCD_BINARY_NAME=argocd-repo-server $COMMAND --loglevel debug --port ${ARGOCD_E2E_REPOSERVER_PORT:-8081} --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379}" ui: sh -c "test $ARGOCD_IN_CI = true && exit 0; cd ui && ARGOCD_E2E_YARN_HOST=0.0.0.0 ${ARGOCD_E2E_YARN_CMD:-yarn} start" diff --git a/test/e2e/app_deletion_test.go b/test/e2e/app_deletion_test.go index 1194edcb37df3..9158dddffa06a 100644 --- a/test/e2e/app_deletion_test.go +++ b/test/e2e/app_deletion_test.go @@ -67,3 +67,18 @@ func TestDeletingAppByLabel(t *testing.T) { // delete is successful Expect(DoesNotExist()) } + +func TestDeletingAppByLabelWait(t *testing.T) { + Given(t). + Path(guestbookPath). + When(). + CreateApp("--label=foo=bar"). + Sync(). + Then(). + Expect(SyncStatusIs(SyncStatusCode(SyncStatusCodeSynced))). + When(). + DeleteBySelectorWithWait("foo=bar"). + Then(). + // delete is successful + Expect(DoesNotExistNow()) +} diff --git a/test/e2e/app_management_ns_test.go b/test/e2e/app_management_ns_test.go index 3e13131791ab9..32636e2b52c49 100644 --- a/test/e2e/app_management_ns_test.go +++ b/test/e2e/app_management_ns_test.go @@ -748,7 +748,7 @@ func TestNamespacedResourceDiffing(t *testing.T) { Then(). Expect(SyncStatusIs(SyncStatusCodeOutOfSync)). And(func(app *Application) { - diffOutput, err := RunCli("app", "diff", ctx.AppQualifiedName(), "--local", "testdata/guestbook") + diffOutput, err := RunCli("app", "diff", ctx.AppQualifiedName(), "--local-repo-root", ".", "--local", "testdata/guestbook") assert.Error(t, err) assert.Contains(t, diffOutput, fmt.Sprintf("===== apps/Deployment %s/guestbook-ui ======", DeploymentNamespace())) }). @@ -761,7 +761,7 @@ func TestNamespacedResourceDiffing(t *testing.T) { Then(). Expect(SyncStatusIs(SyncStatusCodeSynced)). And(func(app *Application) { - diffOutput, err := RunCli("app", "diff", ctx.AppQualifiedName(), "--local", "testdata/guestbook") + diffOutput, err := RunCli("app", "diff", ctx.AppQualifiedName(), "--local-repo-root", ".", "--local", "testdata/guestbook") assert.NoError(t, err) assert.Empty(t, diffOutput) }). @@ -897,7 +897,7 @@ func testNSEdgeCasesApplicationResources(t *testing.T, appPath string, statusCod expect. Expect(HealthIs(statusCode)). And(func(app *Application) { - diffOutput, err := RunCli("app", "diff", ctx.AppQualifiedName(), "--local", path.Join("testdata", appPath)) + diffOutput, err := RunCli("app", "diff", ctx.AppQualifiedName(), "--local-repo-root", ".", "--local", path.Join("testdata", appPath)) assert.Empty(t, diffOutput) assert.NoError(t, err) }) @@ -998,7 +998,7 @@ func TestNamespacedLocalManifestSync(t *testing.T) { Given(). LocalPath(guestbookPathLocal). When(). - Sync(). + Sync("--local-repo-root", "."). Then(). Expect(SyncStatusIs(SyncStatusCodeSynced)). And(func(app *Application) { @@ -1066,7 +1066,7 @@ func TestNamespacedLocalSyncDryRunWithASEnabled(t *testing.T) { assert.NoError(t, err) appBefore := app.DeepCopy() - _, err = RunCli("app", "sync", app.QualifiedName(), "--dry-run", "--local", guestbookPathLocal) + _, err = RunCli("app", "sync", app.QualifiedName(), "--dry-run", "--local-repo-root", ".", "--local", guestbookPathLocal) assert.NoError(t, err) appAfter := app.DeepCopy() diff --git a/test/e2e/app_management_test.go b/test/e2e/app_management_test.go index d2902e27c97d8..10b2cf926723c 100644 --- a/test/e2e/app_management_test.go +++ b/test/e2e/app_management_test.go @@ -1324,7 +1324,7 @@ func TestLocalManifestSync(t *testing.T) { Given(). LocalPath(guestbookPathLocal). When(). - Sync(). + Sync("--local-repo-root", "."). Then(). Expect(SyncStatusIs(SyncStatusCodeSynced)). And(func(app *Application) { @@ -1385,7 +1385,7 @@ func TestLocalSyncDryRunWithAutosyncEnabled(t *testing.T) { assert.NoError(t, err) appBefore := app.DeepCopy() - _, err = RunCli("app", "sync", app.Name, "--dry-run", "--local", guestbookPathLocal) + _, err = RunCli("app", "sync", app.Name, "--dry-run", "--local-repo-root", ".", "--local", guestbookPathLocal) assert.NoError(t, err) appAfter := app.DeepCopy() diff --git a/test/e2e/app_sync_options_test.go b/test/e2e/app_sync_options_test.go new file mode 100644 index 0000000000000..7d0a0ffeabb99 --- /dev/null +++ b/test/e2e/app_sync_options_test.go @@ -0,0 +1,61 @@ +package e2e + +import ( + "testing" + + "github.com/argoproj/gitops-engine/pkg/health" + . "github.com/argoproj/gitops-engine/pkg/sync/common" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + + . "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + . "github.com/argoproj/argo-cd/v2/test/e2e/fixture" + . "github.com/argoproj/argo-cd/v2/test/e2e/fixture/app" +) + +// Given application is set with --sync-option CreateNamespace=true and --sync-option ServerSideApply=true +// +// application --dest-namespace exists +// +// Then, --dest-namespace is created with server side apply +// application is synced and healthy with resource +// application resources created with server side apply in the newly created namespace. +func TestNamespaceCreationWithSSA(t *testing.T) { + SkipOnEnv(t, "OPENSHIFT") + namespace := "guestbook-ui-with-ssa" + defer func() { + if !t.Skipped() { + _, err := Run("", "kubectl", "delete", "namespace", namespace) + assert.NoError(t, err) + } + }() + + ctx := Given(t) + ctx. + SetAppNamespace(AppNamespace()). + Timeout(30). + Path("guestbook"). + When(). + CreateFromFile(func(app *Application) { + app.Spec.SyncPolicy = &SyncPolicy{ + SyncOptions: SyncOptions{"CreateNamespace=true", "ServerSideApply=true"}, + } + }). + Then(). + Expect(NoNamespace(namespace)). + When(). + AppSet("--dest-namespace", namespace). + Sync(). + Then(). + Expect(Success("")). + Expect(Namespace(namespace, func(app *Application, ns *v1.Namespace) { + assert.NotContains(t, ns.Annotations, "kubectl.kubernetes.io/last-applied-configuration") + })). + Expect(SyncStatusIs(SyncStatusCodeSynced)). + Expect(HealthIs(health.HealthStatusHealthy)). + Expect(OperationPhaseIs(OperationSucceeded)). + Expect(ResourceHealthWithNamespaceIs("Deployment", "guestbook-ui", namespace, health.HealthStatusHealthy)). + Expect(ResourceSyncStatusWithNamespaceIs("Deployment", "guestbook-ui", namespace, SyncStatusCodeSynced)). + Expect(ResourceHealthWithNamespaceIs("Service", "guestbook-ui", namespace, health.HealthStatusHealthy)). + Expect(ResourceSyncStatusWithNamespaceIs("Service", "guestbook-ui", namespace, SyncStatusCodeSynced)) +} diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index e57b2132b7472..2074a6aa1b7b1 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -38,7 +38,7 @@ https://kubernetes.default.svc in-cluster %v Successful `, GetVe When(). CreateApp() - tries := 2 + tries := 5 for i := 0; i <= tries; i += 1 { clusterFixture.GivenWithSameState(t). When(). diff --git a/test/e2e/fixture/app/actions.go b/test/e2e/fixture/app/actions.go index f4fd167db1024..a2b1d5e01371b 100644 --- a/test/e2e/fixture/app/actions.go +++ b/test/e2e/fixture/app/actions.go @@ -417,6 +417,12 @@ func (a *Actions) DeleteBySelector(selector string) *Actions { return a } +func (a *Actions) DeleteBySelectorWithWait(selector string) *Actions { + a.context.t.Helper() + a.runCli("app", "delete", fmt.Sprintf("--selector=%s", selector), "--yes", "--wait") + return a +} + func (a *Actions) Wait(args ...string) *Actions { a.context.t.Helper() args = append([]string{"app", "wait"}, args...) diff --git a/test/e2e/fixture/app/expectation.go b/test/e2e/fixture/app/expectation.go index c7cf20ab27729..4d4918e981751 100644 --- a/test/e2e/fixture/app/expectation.go +++ b/test/e2e/fixture/app/expectation.go @@ -216,6 +216,19 @@ func DoesNotExist() Expectation { } } +func DoesNotExistNow() Expectation { + return func(c *Consequences) (state, string) { + _, err := c.get() + if err != nil { + if apierr.IsNotFound(err) { + return succeeded, "app does not exist" + } + return failed, err.Error() + } + return failed, "app should not exist" + } +} + func Pod(predicate func(p v1.Pod) bool) Expectation { return func(c *Consequences) (state, string) { pods, err := pods() diff --git a/test/e2e/fixture/applicationsets/context.go b/test/e2e/fixture/applicationsets/context.go index c10b2c99bfe5f..a7e91f4d0c8ff 100644 --- a/test/e2e/fixture/applicationsets/context.go +++ b/test/e2e/fixture/applicationsets/context.go @@ -5,7 +5,6 @@ import ( "time" "github.com/argoproj/argo-cd/v2/test/e2e/fixture/applicationsets/utils" - . "github.com/argoproj/argo-cd/v2/test/e2e/fixture/applicationsets/utils" ) // Context implements the "given" part of given/when/then @@ -19,7 +18,7 @@ type Context struct { } func Given(t *testing.T) *Context { - EnsureCleanState(t) + utils.EnsureCleanState(t) return &Context{t: t} } diff --git a/test/e2e/fixture/http.go b/test/e2e/fixture/http.go index 1e818f5262024..00c123ab5d893 100644 --- a/test/e2e/fixture/http.go +++ b/test/e2e/fixture/http.go @@ -28,6 +28,7 @@ func DoHttpRequest(method string, path string, data ...byte) (*http.Response, er return nil, err } req.AddCookie(&http.Cookie{Name: common.AuthCookieName, Value: token}) + req.Header.Set("Content-Type", "application/json") httpClient := &http.Client{ Transport: &http.Transport{ diff --git a/test/e2e/multiarch-container/Dockerfile b/test/e2e/multiarch-container/Dockerfile index d4d49adc0d746..fb9b9224f24c4 100644 --- a/test/e2e/multiarch-container/Dockerfile +++ b/test/e2e/multiarch-container/Dockerfile @@ -1,2 +1,2 @@ -FROM docker.io/library/busybox@sha256:2376a0c12759aa1214ba83e771ff252c7b1663216b192fbe5e0fb364e952f85c +FROM docker.io/library/busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 CMD exec sh -c "trap : TERM INT; echo 'Hi' && tail -f /dev/null" diff --git a/test/e2e/sync_waves_test.go b/test/e2e/sync_waves_test.go index ac5db15eee57d..8d0ee14e487d1 100644 --- a/test/e2e/sync_waves_test.go +++ b/test/e2e/sync_waves_test.go @@ -9,6 +9,8 @@ import ( "github.com/argoproj/gitops-engine/pkg/health" . "github.com/argoproj/gitops-engine/pkg/sync/common" + + v1 "k8s.io/api/core/v1" ) func TestFixingDegradedApp(t *testing.T) { @@ -100,3 +102,46 @@ func TestDegradedDeploymentIsSucceededAndSynced(t *testing.T) { Expect(SyncStatusIs(SyncStatusCodeSynced)). Expect(ResourceResultNumbering(1)) } + +// resources should be pruned in reverse of creation order(syncwaves order) +func TestSyncPruneOrderWithSyncWaves(t *testing.T) { + ctx := Given(t).Timeout(60) + + // remove finalizer to ensure proper cleanup if test fails at early stage + defer func() { + _, _ = RunCli("app", "patch-resource", ctx.AppQualifiedName(), + "--kind", "Pod", + "--resource-name", "pod-with-finalizers", + "--patch", `[{"op": "remove", "path": "/metadata/finalizers"}]`, + "--patch-type", "application/json-patch+json", "--all", + ) + }() + + ctx.Path("syncwaves-prune-order"). + When(). + CreateApp(). + // creation order: sa & role -> rolebinding -> pod + Sync(). + Wait(). + Then(). + Expect(SyncStatusIs(SyncStatusCodeSynced)). + Expect(HealthIs(health.HealthStatusHealthy)). + When(). + // delete files to remove resources + DeleteFile("pod.yaml"). + DeleteFile("rbac.yaml"). + Refresh(RefreshTypeHard). + IgnoreErrors(). + Then(). + Expect(SyncStatusIs(SyncStatusCodeOutOfSync)). + When(). + // prune order: pod -> rolebinding -> sa & role + Sync("--prune"). + Wait(). + Then(). + Expect(OperationPhaseIs(OperationSucceeded)). + Expect(SyncStatusIs(SyncStatusCodeSynced)). + Expect(HealthIs(health.HealthStatusHealthy)). + Expect(NotPod(func(p v1.Pod) bool { return p.Name == "pod-with-finalizers" })). + Expect(ResourceResultNumbering(4)) +} diff --git a/test/e2e/testdata/syncwaves-prune-order/README.md b/test/e2e/testdata/syncwaves-prune-order/README.md new file mode 100644 index 0000000000000..92a62fdfe109d --- /dev/null +++ b/test/e2e/testdata/syncwaves-prune-order/README.md @@ -0,0 +1,15 @@ +## Test Scenario + +This test example is for testing the reverse pruning of resources with syncwaves during sync operation. + +Resource creation happens in below order +- wave 0: sa & role +- wave 1: rolebinding +- wave 2: pod + +They are setup in such a way that the resources will be cleaned up properly only if they are deleted in the reverse order of creation i.e +- wave 0: pod +- wave 1: rolebinding +- wave 2: sa & role + +If above delete order is not followed the pod gets stuck in terminating state due to a finalizer which is supposed to be removed by k8s container lifecycle hook on delete if delete order is correct. \ No newline at end of file diff --git a/test/e2e/testdata/syncwaves-prune-order/pod.yaml b/test/e2e/testdata/syncwaves-prune-order/pod.yaml new file mode 100644 index 0000000000000..f801a3992aa37 --- /dev/null +++ b/test/e2e/testdata/syncwaves-prune-order/pod.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-finalizers + annotations: + argocd.argoproj.io/sync-wave: "2" + # remove this finalizers using container preStop lifecycle hook on delete + finalizers: + - example.com/block-delete +spec: + serviceAccountName: modify-pods-sa # sa with permissions to modify pods + terminationGracePeriodSeconds: 15 + containers: + - name: container + image: nginx:alpine + command: ["/bin/sh", "-c"] + args: ["sleep 10h"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + lifecycle: + # remove finalizers for successful delete of pod + preStop: + exec: + command: + - /bin/sh + - -c + - | + set -e + + SERVICE_ACCOUNT_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + POD_URL="https://kubernetes.default.svc/api/v1/namespaces/$NAMESPACE/pods/$POD_NAME" + PATCH_PAYLOAD='[{"op": "remove", "path": "/metadata/finalizers"}]' + + curl -k -v -H "Authorization: Bearer $SERVICE_ACCOUNT_TOKEN" -H "Content-Type: application/json-patch+json" -X PATCH --data "$PATCH_PAYLOAD" $POD_URL diff --git a/test/e2e/testdata/syncwaves-prune-order/rbac.yaml b/test/e2e/testdata/syncwaves-prune-order/rbac.yaml new file mode 100644 index 0000000000000..9512644b731db --- /dev/null +++ b/test/e2e/testdata/syncwaves-prune-order/rbac.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: modify-pods-sa + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: modify-pods-role + annotations: + argocd.argoproj.io/sync-wave: "0" +rules: + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - delete + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: modify-pods-rolebinding + annotations: + argocd.argoproj.io/sync-wave: "1" +subjects: + - kind: ServiceAccount + name: modify-pods-sa +roleRef: + kind: Role + name: modify-pods-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/test/remote/Dockerfile b/test/remote/Dockerfile index 8d03d1321d25b..cf43ee355567d 100644 --- a/test/remote/Dockerfile +++ b/test/remote/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_IMAGE=docker.io/library/ubuntu:22.04 -FROM docker.io/library/golang:1.21.3@sha256:02d7116222536a5cf0fcf631f90b507758b669648e0f20186d2dc94a9b419a9b AS go +FROM docker.io/library/golang:1.22.0@sha256:7b297d9abee021bab9046e492506b3c2da8a3722cbf301653186545ecc1e00bb AS go RUN go install github.com/mattn/goreman@latest && \ go install github.com/kisielk/godepgraph@latest diff --git a/test/testutil.go b/test/testutil.go index 34264772fa54f..e97de4a762bcf 100644 --- a/test/testutil.go +++ b/test/testutil.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/tools/cache" @@ -84,6 +85,15 @@ func YamlToUnstructured(yamlStr string) *unstructured.Unstructured { return &unstructured.Unstructured{Object: obj} } +func YamlToApplication(yamlStr string) *v1alpha1.Application { + app := v1alpha1.Application{} + err := yaml.Unmarshal([]byte(yamlStr), &app) + if err != nil { + panic(err) + } + return &app +} + // ToMap converts any object to a map[string]interface{} func ToMap(obj interface{}) map[string]interface{} { data, err := json.Marshal(obj) diff --git a/ui-test/Dockerfile b/ui-test/Dockerfile index a5a77710eca52..4868c11d66056 100644 --- a/ui-test/Dockerfile +++ b/ui-test/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/library/node:20.7.0@sha256:f08c20b9f9c55dd47b1841793f0ee480c5395aa165cd02edfd68b068ed64bfb5 as node +FROM docker.io/library/node:21.6.2@sha256:65998e325b06014d4f1417a8a6afb1540d1ac66521cca76f2221a6953947f9ee as node RUN apt-get update && apt-get install --no-install-recommends -y \ software-properties-common diff --git a/ui-test/package.json b/ui-test/package.json index 1875e31b6fd62..fd34ca2edab4a 100644 --- a/ui-test/package.json +++ b/ui-test/package.json @@ -27,6 +27,6 @@ "tslint-config-prettier": "^1.18.0", "tslint-plugin-prettier": "^2.0.1", "typescript": "^4.0.3", - "yarn": "^1.22.10" + "yarn": "^1.22.13" } } diff --git a/ui-test/yarn.lock b/ui-test/yarn.lock index b80910028fb7f..6765cbf79d61b 100644 --- a/ui-test/yarn.lock +++ b/ui-test/yarn.lock @@ -540,9 +540,9 @@ flat@^5.0.2: integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== follow-redirects@^1.14.0: - version "1.14.9" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7" - integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w== + version "1.15.5" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.5.tgz#54d4d6d062c0fa7d9d17feb008461550e3ba8020" + integrity sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw== foreach@^2.0.5: version "2.0.5" @@ -1510,10 +1510,10 @@ yargs@13.3.2: y18n "^4.0.0" yargs-parser "^13.1.2" -yarn@^1.22.10: - version "1.22.10" - resolved "https://registry.npmjs.org/yarn/-/yarn-1.22.10.tgz" - integrity sha512-IanQGI9RRPAN87VGTF7zs2uxkSyQSrSPsju0COgbsKQOOXr5LtcVPeyXWgwVa0ywG3d8dg6kSYKGBuYK021qeA== +yarn@^1.22.13: + version "1.22.13" + resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.22.13.tgz#8789ef23b630fe99b819b044f4b7b93ab1bc1b8f" + integrity sha512-G8qG4t7Ef5cLVpzbM3HWWsow4hpfeSCfKtMnjfERmp9V5qSCOKz0uGAIQCM/x3gWfCzH8Bvb4hl3ZfhG/XD1Jg== yauzl@^2.10.0: version "2.10.0" diff --git a/ui/.nvmrc b/ui/.nvmrc index 376d26203e61e..a8d3ff91fa10d 100644 --- a/ui/.nvmrc +++ b/ui/.nvmrc @@ -1 +1 @@ -v20.7.0 +v21.6.1 diff --git a/ui/package.json b/ui/package.json index d290c93be08cb..828d0c6e7f97d 100644 --- a/ui/package.json +++ b/ui/package.json @@ -40,7 +40,7 @@ "react-dom": "^16.9.3", "react-form": "2.16.3", "react-ga": "^2.7.0", - "react-helmet": "^5.2.0", + "react-helmet": "^6.1.0", "react-hot-loader": "^3.1.3", "react-moment": "^0.9.7", "react-paginate": "^8.1.4", @@ -84,7 +84,7 @@ "@types/react-autocomplete": "^1.8.4", "@types/react-dom": "^16.9.14", "@types/react-form": "^2.16.0", - "@types/react-helmet": "^5.0.17", + "@types/react-helmet": "^6.1.6", "@types/react-paginate": "^6.2.0", "@types/react-router": "^4.0.27", "@types/react-router-dom": "^4.2.3", @@ -102,7 +102,7 @@ "jest-junit": "^6.4.0", "jest-transform-css": "^2.0.0", "monaco-editor-webpack-plugin": "^7.0.0", - "postcss": "^8.2.13", + "postcss": "^8.4.35", "prettier": "1.19", "raw-loader": "^0.5.1", "react-test-renderer": "16.8.3", @@ -120,6 +120,6 @@ "webpack": "^5.84.1", "webpack-cli": "^4.9.2", "webpack-dev-server": "^4.7.4", - "yarn": "^1.22.10" + "yarn": "^1.22.21" } } diff --git a/ui/src/app/applications/components/application-details/application-resource-list.scss b/ui/src/app/applications/components/application-details/application-resource-list.scss new file mode 100644 index 0000000000000..9bc4b17bfe7ed --- /dev/null +++ b/ui/src/app/applications/components/application-details/application-resource-list.scss @@ -0,0 +1,13 @@ +.application-details__item { + display: flex; + + .application-details__item_text { + overflow: hidden; + text-overflow: ellipsis; + } + + .application-details__external_link { + flex: 0; + min-width: 13px; + } +} diff --git a/ui/src/app/applications/components/application-details/application-resource-list.tsx b/ui/src/app/applications/components/application-details/application-resource-list.tsx index c5519fc4b6ff9..d1e01adb52c04 100644 --- a/ui/src/app/applications/components/application-details/application-resource-list.tsx +++ b/ui/src/app/applications/components/application-details/application-resource-list.tsx @@ -10,6 +10,7 @@ import * as _ from 'lodash'; import Moment from 'react-moment'; import {format} from 'date-fns'; import {ResourceNode, ResourceRef} from '../../../shared/models'; +import './application-resource-list.scss'; export const ApplicationResourceList = ({ resources, @@ -89,8 +90,8 @@ export const ApplicationResourceList = ({
{ResourceLabel({kind: res.kind})}
-
- {res.name} +
+ {res.name} {res.kind === 'Application' && ( {ctx => ( diff --git a/ui/src/app/applications/components/application-node-info/application-node-info.scss b/ui/src/app/applications/components/application-node-info/application-node-info.scss index f50e67279cc52..27ab11d776c17 100644 --- a/ui/src/app/applications/components/application-node-info/application-node-info.scss +++ b/ui/src/app/applications/components/application-node-info/application-node-info.scss @@ -1,4 +1,5 @@ @import 'node_modules/argo-ui/src/styles/config'; +@import 'node_modules/argo-ui/src/styles/theme'; .application-node-info { &__manifest { @@ -6,6 +7,9 @@ .tabs__content { background-color: white; + @include themify($themes){ + background-color: themed('background-2'); + } } &--raw { @@ -37,6 +41,9 @@ label { padding-right: 2em; color: $argo-color-gray-8; + @include themify($themes){ + color: themed('text-2'); + } } } &__err_msg { diff --git a/ui/src/app/applications/components/application-node-info/application-node-info.tsx b/ui/src/app/applications/components/application-node-info/application-node-info.tsx index 18ff44e381c55..edd787e0240c1 100644 --- a/ui/src/app/applications/components/application-node-info/application-node-info.tsx +++ b/ui/src/app/applications/components/application-node-info/application-node-info.tsx @@ -21,7 +21,20 @@ const RenderContainerState = (props: {container: any}) => { return (
-
{props.container.name}
+
+ {props.container.state?.running && ( + + + + )} + {(props.container.state.terminated && props.container.state.terminated?.exitCode !== 0) || + (lastState && lastState?.exitCode !== 0 && ( + + + + ))} + {props.container.name} +
{state && ( <> diff --git a/ui/src/app/applications/components/application-parameters/application-parameters.tsx b/ui/src/app/applications/components/application-parameters/application-parameters.tsx index 27f292ff7d2e8..38a6d151a90c2 100644 --- a/ui/src/app/applications/components/application-parameters/application-parameters.tsx +++ b/ui/src/app/applications/components/application-parameters/application-parameters.tsx @@ -286,7 +286,7 @@ export const ApplicationParameters = (props: { } else if (props.details.type === 'Plugin') { attributes.push({ title: 'NAME', - view:
{ValueEditor(app.spec.source.plugin && app.spec.source.plugin.name, null)}
, + view:
{ValueEditor(app.spec.source?.plugin?.name, null)}
, edit: (formApi: FormApi) => ( services.authService.plugins()}> {(plugins: Plugin[]) => ( @@ -299,12 +299,11 @@ export const ApplicationParameters = (props: { title: 'ENV', view: (
- {app.spec.source.plugin && - (app.spec.source.plugin.env || []).map(val => ( - - {NameValueEditor(val, null)} - - ))} + {(app.spec.source?.plugin?.env || []).map(val => ( + + {NameValueEditor(val, null)} + + ))}
), edit: (formApi: FormApi) => @@ -315,7 +314,7 @@ export const ApplicationParameters = (props: { parametersSet.add(announcement.name); } } - if (app.spec.source.plugin?.parameters) { + if (app.spec.source?.plugin?.parameters) { for (const appParameter of app.spec.source.plugin.parameters) { parametersSet.add(appParameter.name); } @@ -326,7 +325,7 @@ export const ApplicationParameters = (props: { } parametersSet.forEach(name => { const announcement = props.details.plugin.parametersAnnouncement?.find(param => param.name === name); - const liveParam = app.spec.source.plugin?.parameters?.find(param => param.name === name); + const liveParam = app.spec.source?.plugin?.parameters?.find(param => param.name === name); const pluginIcon = announcement && liveParam ? 'This parameter has been provided by plugin, but is overridden in application manifest.' : 'This parameter is provided by the plugin.'; const isPluginPar = !!announcement; diff --git a/ui/src/app/applications/components/application-resources-diff/application-resources-diff.scss b/ui/src/app/applications/components/application-resources-diff/application-resources-diff.scss index fbf23c95796bf..fb139f273a24c 100644 --- a/ui/src/app/applications/components/application-resources-diff/application-resources-diff.scss +++ b/ui/src/app/applications/components/application-resources-diff/application-resources-diff.scss @@ -7,6 +7,9 @@ label { padding-right: 2em; color: $argo-color-gray-8; + @include themify($themes){ + color: themed('text-2'); + } } } &__diff { @@ -28,4 +31,8 @@ .custom-diff-hunk { color: $argo-color-gray-6; + border-bottom: 1px dashed; + @include themify($themes){ + border-bottom: 1px dashed themed('text-2'); + } } \ No newline at end of file diff --git a/ui/src/app/applications/components/application-summary/application-summary.tsx b/ui/src/app/applications/components/application-summary/application-summary.tsx index 5e8fa2db22ba1..63bab3be0364c 100644 --- a/ui/src/app/applications/components/application-summary/application-summary.tsx +++ b/ui/src/app/applications/components/application-summary/application-summary.tsx @@ -15,7 +15,7 @@ import { RevisionHelpIcon } from '../../../shared/components'; import {BadgePanel, Spinner} from '../../../shared/components'; -import {Consumer, ContextApis} from '../../../shared/context'; +import {AuthSettingsCtx, Consumer, ContextApis} from '../../../shared/context'; import * as models from '../../../shared/models'; import {services} from '../../../shared/services'; @@ -37,6 +37,16 @@ function swap(array: any[], a: number, b: number) { return array; } +function processPath(path: string) { + if (path !== null && path !== undefined) { + if (path === '.') { + return '(root)'; + } + return path; + } + return ''; +} + export interface ApplicationSummaryProps { app: models.Application; updateApp: (app: models.Application, query: {validate?: boolean}) => Promise; @@ -47,6 +57,7 @@ export const ApplicationSummary = (props: ApplicationSummaryProps) => { const source = getAppDefaultSource(app); const isHelm = source.hasOwnProperty('chart'); const initialState = app.spec.destination.server === undefined ? 'NAME' : 'URL'; + const useAuthSettingsCtx = React.useContext(AuthSettingsCtx); const [destFormat, setDestFormat] = React.useState(initialState); const [changeSync, setChangeSync] = React.useState(false); @@ -238,7 +249,7 @@ export const ApplicationSummary = (props: ApplicationSummaryProps) => { title: 'PATH', view: ( - {source.path ?? ''} + {processPath(source.path)} ), edit: (formApi: FormApi) => @@ -255,7 +266,12 @@ export const ApplicationSummary = (props: ApplicationSummaryProps) => { view: app.spec.revisionHistoryLimit, edit: (formApi: FormApi) => (
- +
+
{tilesProps.applications.map((app, i) => { const source = getAppDefaultSource(app); return ( diff --git a/ui/src/app/applications/components/pod-logs-viewer/pod-logs-viewer.scss b/ui/src/app/applications/components/pod-logs-viewer/pod-logs-viewer.scss index 44e350f325137..9530d82dd1146 100644 --- a/ui/src/app/applications/components/pod-logs-viewer/pod-logs-viewer.scss +++ b/ui/src/app/applications/components/pod-logs-viewer/pod-logs-viewer.scss @@ -1,12 +1,19 @@ @import 'node_modules/argo-ui/src/styles/config'; +@import 'node_modules/argo-ui/src/styles/theme'; .pod-logs-viewer { height: 90%; font-size: 14px; font-family: monospace; background-color: white; + @include themify($themes){ + background-color: themed('background-2'); + } padding: 0; color: black; + @include themify($themes){ + color: themed('text-2'); + } &--inverted { background-color: black; diff --git a/ui/src/app/login/components/utils.ts b/ui/src/app/login/components/utils.ts index 90453ced77d4a..6c715077cc9cc 100644 --- a/ui/src/app/login/components/utils.ts +++ b/ui/src/app/login/components/utils.ts @@ -74,10 +74,6 @@ export const pkceLogin = async (oidcConfig: AuthSettings['oidcConfig'], redirect throw new PKCELoginError('No Authorization Server endpoint found'); } - if (!authorizationServer?.code_challenge_methods_supported?.includes('S256')) { - throw new PKCELoginError('Authorization Server does not support S256 code challenge method'); - } - const codeVerifier = generateRandomCodeVerifier(); const codeChallange = await calculatePKCECodeChallenge(codeVerifier); diff --git a/ui/src/app/settings/components/repo-details/repo-details.tsx b/ui/src/app/settings/components/repo-details/repo-details.tsx index 6075dfbde3b22..25fa983172700 100644 --- a/ui/src/app/settings/components/repo-details/repo-details.tsx +++ b/ui/src/app/settings/components/repo-details/repo-details.tsx @@ -65,7 +65,8 @@ export const RepoDetails = (props: {repo: models.Repository; save?: (params: New enableLfs: repo.enableLfs || false, proxy: repo.proxy || '', project: repo.project || '', - enableOCI: repo.enableOCI || false + enableOCI: repo.enableOCI || false, + forceHttpBasicAuth: repo.forceHttpBasicAuth || false }; return ( diff --git a/ui/src/app/shared/components/badge-panel/badge-panel.tsx b/ui/src/app/shared/components/badge-panel/badge-panel.tsx index ad6a4f6c187e2..d6ab8fa95ffd2 100644 --- a/ui/src/app/shared/components/badge-panel/badge-panel.tsx +++ b/ui/src/app/shared/components/badge-panel/badge-panel.tsx @@ -6,7 +6,7 @@ import {Context} from '../../context'; require('./badge-panel.scss'); -export const BadgePanel = ({app, project}: {app?: string; project?: string}) => { +export const BadgePanel = ({app, project, appNamespace, nsEnabled}: {app?: string; project?: string; appNamespace?: string; nsEnabled?: boolean}) => { const [badgeType, setBadgeType] = React.useState('URL'); const context = React.useContext(Context); if (!app && !project) { @@ -19,7 +19,10 @@ export const BadgePanel = ({app, project}: {app?: string; project?: string}) => let entityURL = ''; let alt = ''; if (app) { - badgeURL = `${root}api/badge?name=${app}&revision=true`; + badgeURL = `${root}api/badge?name=${app}&revision=true&showAppName=true`; + if (nsEnabled) { + badgeURL += `&namespace=${appNamespace}`; + } entityURL = `${root}applications/${app}`; alt = 'App Status'; } else if (project) { diff --git a/ui/src/app/shared/components/monaco-editor.tsx b/ui/src/app/shared/components/monaco-editor.tsx index 817e4c8ae6bcb..a30381638f0b5 100644 --- a/ui/src/app/shared/components/monaco-editor.tsx +++ b/ui/src/app/shared/components/monaco-editor.tsx @@ -1,6 +1,7 @@ import * as React from 'react'; import * as monacoEditor from 'monaco-editor'; +import {services} from '../services'; export interface EditorInput { text: string; @@ -28,6 +29,16 @@ const MonacoEditorLazy = React.lazy(() => const Component = (props: MonacoProps) => { const [height, setHeight] = React.useState(0); + React.useEffect(() => { + const subscription = services.viewPreferences.getPreferences().subscribe(preferences => { + monaco.editor.setTheme(preferences.theme === 'dark' ? 'vs-dark' : 'vs'); + }); + + return () => { + subscription.unsubscribe(); + }; + }, []); + return (
{ return requests .put(`/repositories/${encodeURIComponent(url)}`) - .send({type, name, repo: url, username, password, tlsClientCertData, tlsClientCertKey, insecure, enableLfs, proxy, project}) + .send({type, name, repo: url, username, password, tlsClientCertData, tlsClientCertKey, insecure, enableLfs, proxy, project, forceHttpBasicAuth, enableOCI}) .then(res => res.body as models.Repository); } diff --git a/ui/src/app/shared/services/requests.ts b/ui/src/app/shared/services/requests.ts index 207917a318529..4df6d1e4ddf19 100644 --- a/ui/src/app/shared/services/requests.ts +++ b/ui/src/app/shared/services/requests.ts @@ -51,19 +51,19 @@ export default { }, post(url: string) { - return initHandlers(agent.post(`${apiRoot()}${url}`)); + return initHandlers(agent.post(`${apiRoot()}${url}`)).set('Content-Type', 'application/json'); }, put(url: string) { - return initHandlers(agent.put(`${apiRoot()}${url}`)); + return initHandlers(agent.put(`${apiRoot()}${url}`)).set('Content-Type', 'application/json'); }, patch(url: string) { - return initHandlers(agent.patch(`${apiRoot()}${url}`)); + return initHandlers(agent.patch(`${apiRoot()}${url}`)).set('Content-Type', 'application/json'); }, delete(url: string) { - return initHandlers(agent.del(`${apiRoot()}${url}`)); + return initHandlers(agent.del(`${apiRoot()}${url}`)).set('Content-Type', 'application/json'); }, loadEventSource(url: string): Observable { diff --git a/ui/src/app/ui-banner/ui-banner.tsx b/ui/src/app/ui-banner/ui-banner.tsx index 29dbc45eadfbc..ac097bca55163 100644 --- a/ui/src/app/ui-banner/ui-banner.tsx +++ b/ui/src/app/ui-banner/ui-banner.tsx @@ -116,7 +116,7 @@ export const Banner = (props: React.Props) => { chatUrl = 'invalid-url'; } } - const shouldRenderTop = position === 'top' || position === 'both'; + const shouldRenderTop = position === 'top' || position === 'both' || (!position && content); const shouldRenderBottom = position === 'bottom' || position === 'both'; return ( diff --git a/ui/yarn.lock b/ui/yarn.lock index 604bdfb107b04..4e6dee439d56f 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -1961,10 +1961,10 @@ dependencies: "@types/react" "*" -"@types/react-helmet@^5.0.17": - version "5.0.19" - resolved "https://registry.yarnpkg.com/@types/react-helmet/-/react-helmet-5.0.19.tgz#e709f192eac0b873693a6f831bb2dbc7085a5c19" - integrity sha512-Ub4sOSkg/64UYRJG33kYLzMV2OsgD923DXS09K3Pp2Qsq0AId5Ih59NvX2eXDIUQPfxO3rj0rFeVxurN8IUksQ== +"@types/react-helmet@^6.1.6": + version "6.1.6" + resolved "https://registry.yarnpkg.com/@types/react-helmet/-/react-helmet-6.1.6.tgz#7d1afd8cbf099616894e8240e9ef70e3c6d7506d" + integrity sha512-ZKcoOdW/Tg+kiUbkFCBtvDw0k3nD4HJ/h/B9yWxN4uDO8OkRksWTO+EL+z/Qu3aHTeTll3Ro0Cc/8UhwBCMG5A== dependencies: "@types/react" "*" @@ -3201,11 +3201,6 @@ colorette@^1.2.0: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.4.0.tgz#5190fbb87276259a86ad700bff2c6d6faa3fca40" integrity sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g== -colorette@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" - integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== - colorette@^2.0.10, colorette@^2.0.14: version "2.0.16" resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.16.tgz#713b9af84fdb000139f04546bd4a93f62a5085da" @@ -4520,9 +4515,9 @@ find-up@^4.0.0, find-up@^4.1.0: path-exists "^4.0.0" follow-redirects@^1.0.0: - version "1.14.9" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7" - integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w== + version "1.15.5" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.5.tgz#54d4d6d062c0fa7d9d17feb008461550e3ba8020" + integrity sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw== for-in@^1.0.2: version "1.0.2" @@ -6497,10 +6492,10 @@ multicast-dns@^6.0.1: dns-packet "^1.3.1" thunky "^1.0.2" -nanoid@^3.1.22: - version "3.2.0" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.2.0.tgz#62667522da6673971cca916a6d3eff3f415ff80c" - integrity sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA== +nanoid@^3.3.7: + version "3.3.7" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" + integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== nanomatch@^1.2.9: version "1.2.13" @@ -7133,14 +7128,14 @@ postcss@^7.0.1: source-map "^0.6.1" supports-color "^6.1.0" -postcss@^8.2.13: - version "8.2.13" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.2.13.tgz#dbe043e26e3c068e45113b1ed6375d2d37e2129f" - integrity sha512-FCE5xLH+hjbzRdpbRb1IMCvPv9yZx2QnDarBEYSN0N0HYk+TcXsEhwdFcFb+SRWOKzKGErhIEbBK2ogyLdTtfQ== +postcss@^8.4.35: + version "8.4.35" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.35.tgz#60997775689ce09011edf083a549cea44aabe2f7" + integrity sha512-u5U8qYpBCpN13BsiEB0CbR1Hhh4Gc0zLFuedrHJKMctHCHAGrMdG0PRM/KErzAL3CU6/eckEtmHNB3x6e3c0vA== dependencies: - colorette "^1.2.2" - nanoid "^3.1.22" - source-map "^0.6.1" + nanoid "^3.3.7" + picocolors "^1.0.0" + source-map-js "^1.0.2" prelude-ls@~1.1.2: version "1.1.2" @@ -7793,11 +7788,6 @@ react-dom@^16.9.3: prop-types "^15.6.2" scheduler "^0.19.1" -react-fast-compare@^2.0.2: - version "2.0.4" - resolved "https://registry.yarnpkg.com/react-fast-compare/-/react-fast-compare-2.0.4.tgz#e84b4d455b0fec113e0402c329352715196f81f9" - integrity sha512-suNP+J1VU1MWFKcyt7RtjiSWUjvidmQSlqu+eHslq+342xCbGTYmC0mEhPCOHxlW0CywylOC1u2DFAT+bv4dBw== - react-fast-compare@^3.1.1: version "3.2.0" resolved "https://registry.yarnpkg.com/react-fast-compare/-/react-fast-compare-3.2.0.tgz#641a9da81b6a6320f270e89724fb45a0b39e43bb" @@ -7822,16 +7812,6 @@ react-ga@^2.7.0: resolved "https://registry.yarnpkg.com/react-ga/-/react-ga-2.7.0.tgz#24328f157f31e8cffbf4de74a3396536679d8d7c" integrity sha512-AjC7UOZMvygrWTc2hKxTDvlMXEtbmA0IgJjmkhgmQQ3RkXrWR11xEagLGFGaNyaPnmg24oaIiaNPnEoftUhfXA== -react-helmet@^5.2.0: - version "5.2.1" - resolved "https://registry.yarnpkg.com/react-helmet/-/react-helmet-5.2.1.tgz#16a7192fdd09951f8e0fe22ffccbf9bb3e591ffa" - integrity sha512-CnwD822LU8NDBnjCpZ4ySh8L6HYyngViTZLfBBb3NjtrpN8m49clH8hidHouq20I51Y6TpCTISCBbqiY5GamwA== - dependencies: - object-assign "^4.1.1" - prop-types "^15.5.4" - react-fast-compare "^2.0.2" - react-side-effect "^1.1.0" - react-helmet@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/react-helmet/-/react-helmet-6.1.0.tgz#a750d5165cb13cf213e44747502652e794468726" @@ -7930,13 +7910,6 @@ react-router@^4.3.1: prop-types "^15.6.1" warning "^4.0.1" -react-side-effect@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/react-side-effect/-/react-side-effect-1.2.0.tgz#0e940c78faba0c73b9b0eba9cd3dda8dfb7e7dae" - integrity sha512-v1ht1aHg5k/thv56DRcjw+WtojuuDHFUgGfc+bFHOWsF4ZK6C2V57DO0Or0GPsg6+LSTE0M6Ry/gfzhzSwbc5w== - dependencies: - shallowequal "^1.0.1" - react-side-effect@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/react-side-effect/-/react-side-effect-2.1.1.tgz#66c5701c3e7560ab4822a4ee2742dee215d72eb3" @@ -8551,7 +8524,7 @@ shallow-equal@^1.2.1: resolved "https://registry.yarnpkg.com/shallow-equal/-/shallow-equal-1.2.1.tgz#4c16abfa56043aa20d050324efa68940b0da79da" integrity sha512-S4vJDjHHMBaiZuT9NPb616CSmLf618jawtv3sufLl6ivK8WocjAo58cXwbRV1cgqxH0Qbv+iUt6m05eqEa2IRA== -shallowequal@^1.0.1, shallowequal@^1.1.0: +shallowequal@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/shallowequal/-/shallowequal-1.1.0.tgz#188d521de95b9087404fd4dcb68b13df0ae4e7f8" integrity sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ== @@ -8709,7 +8682,7 @@ source-list-map@^2.0.0, source-list-map@^2.0.1: resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34" integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw== -"source-map-js@>=0.6.2 <2.0.0": +"source-map-js@>=0.6.2 <2.0.0", source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== @@ -9941,10 +9914,10 @@ yargs@^17.0.1: y18n "^5.0.5" yargs-parser "^20.2.2" -yarn@^1.22.10: - version "1.22.10" - resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.22.10.tgz#c99daa06257c80f8fa2c3f1490724e394c26b18c" - integrity sha512-IanQGI9RRPAN87VGTF7zs2uxkSyQSrSPsju0COgbsKQOOXr5LtcVPeyXWgwVa0ywG3d8dg6kSYKGBuYK021qeA== +yarn@^1.22.21: + version "1.22.21" + resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.22.21.tgz#1959a18351b811cdeedbd484a8f86c3cc3bbaf72" + integrity sha512-ynXaJsADJ9JiZ84zU25XkPGOvVMmZ5b7tmTSpKURYwgELdjucAOydqIOrOfTxVYcNXe91xvLZwcRh68SR3liCg== yn@3.1.1: version "3.1.1" diff --git a/util/cache/cache.go b/util/cache/cache.go index d34fba5d38f7b..b632824e9c96b 100644 --- a/util/cache/cache.go +++ b/util/cache/cache.go @@ -27,6 +27,10 @@ const ( envRedisRetryCount = "REDIS_RETRY_COUNT" // defaultRedisRetryCount holds default number of retries defaultRedisRetryCount = 3 + // envRedisSentinelPassword is an env variable name which stores redis sentinel password + envRedisSentinelPassword = "REDIS_SENTINEL_PASSWORD" + // envRedisSentinelUsername is an env variable name which stores redis sentinel username + envRedisSentinelUsername = "REDIS_SENTINEL_USERNAME" ) const ( @@ -57,21 +61,23 @@ func buildRedisClient(redisAddress, password, username string, redisDB, maxRetri return client } -func buildFailoverRedisClient(sentinelMaster, password, username string, redisDB, maxRetries int, tlsConfig *tls.Config, sentinelAddresses []string) *redis.Client { +func buildFailoverRedisClient(sentinelMaster, sentinelUsername, sentinelPassword, password, username string, redisDB, maxRetries int, tlsConfig *tls.Config, sentinelAddresses []string) *redis.Client { opts := &redis.FailoverOptions{ - MasterName: sentinelMaster, - SentinelAddrs: sentinelAddresses, - DB: redisDB, - Password: password, - MaxRetries: maxRetries, - TLSConfig: tlsConfig, - Username: username, + MasterName: sentinelMaster, + SentinelAddrs: sentinelAddresses, + DB: redisDB, + Password: password, + MaxRetries: maxRetries, + TLSConfig: tlsConfig, + Username: username, + SentinelUsername: sentinelUsername, + SentinelPassword: sentinelPassword, } client := redis.NewFailoverClient(opts) client.AddHook(redis.Hook(NewArgoRedisHook(func() { - *client = *buildFailoverRedisClient(sentinelMaster, password, username, redisDB, maxRetries, tlsConfig, sentinelAddresses) + *client = *buildFailoverRedisClient(sentinelMaster, sentinelUsername, sentinelPassword, password, username, redisDB, maxRetries, tlsConfig, sentinelAddresses) }))) return client @@ -199,6 +205,8 @@ func AddCacheFlagsToCmd(cmd *cobra.Command, opts ...Options) func() (*Cache, err } password := os.Getenv(envRedisPassword) username := os.Getenv(envRedisUsername) + sentinelUsername := os.Getenv(envRedisSentinelUsername) + sentinelPassword := os.Getenv(envRedisSentinelPassword) if opt.FlagPrefix != "" { if val := os.Getenv(opt.getEnvPrefix() + envRedisUsername); val != "" { username = val @@ -206,14 +214,21 @@ func AddCacheFlagsToCmd(cmd *cobra.Command, opts ...Options) func() (*Cache, err if val := os.Getenv(opt.getEnvPrefix() + envRedisPassword); val != "" { password = val } + if val := os.Getenv(opt.getEnvPrefix() + envRedisSentinelUsername); val != "" { + sentinelUsername = val + } + if val := os.Getenv(opt.getEnvPrefix() + envRedisSentinelPassword); val != "" { + sentinelPassword = val + } } + maxRetries := env.ParseNumFromEnv(envRedisRetryCount, defaultRedisRetryCount, 0, math.MaxInt32) compression, err := CompressionTypeFromString(compressionStr) if err != nil { return nil, err } if len(sentinelAddresses) > 0 { - client := buildFailoverRedisClient(sentinelMaster, password, username, redisDB, maxRetries, tlsConfig, sentinelAddresses) + client := buildFailoverRedisClient(sentinelMaster, sentinelUsername, sentinelPassword, password, username, redisDB, maxRetries, tlsConfig, sentinelAddresses) opt.callOnClientCreated(client) return NewCache(NewRedisCache(client, defaultCacheExpiration, compression)), nil } diff --git a/util/cache/redis.go b/util/cache/redis.go index 7d5303bb3a9fa..4648a553f08cc 100644 --- a/util/cache/redis.go +++ b/util/cache/redis.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "io" + "net" "time" ioutil "github.com/argoproj/argo-cd/v2/util/io" @@ -159,41 +160,27 @@ type MetricsRegistry interface { ObserveRedisRequestDuration(duration time.Duration) } -var metricStartTimeKey = struct{}{} - type redisHook struct { registry MetricsRegistry } -func (rh *redisHook) BeforeProcess(ctx context.Context, cmd redis.Cmder) (context.Context, error) { - return context.WithValue(ctx, metricStartTimeKey, time.Now()), nil -} - -func (rh *redisHook) AfterProcess(ctx context.Context, cmd redis.Cmder) error { - cmdErr := cmd.Err() - rh.registry.IncRedisRequest(cmdErr != nil && cmdErr != redis.Nil) - - startTime := ctx.Value(metricStartTimeKey).(time.Time) - duration := time.Since(startTime) - rh.registry.ObserveRedisRequestDuration(duration) - - return nil -} - -func (redisHook) BeforeProcessPipeline(ctx context.Context, _ []redis.Cmder) (context.Context, error) { - return ctx, nil +func (rh *redisHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := next(ctx, network, addr) + return conn, err + } } -func (redisHook) AfterProcessPipeline(_ context.Context, _ []redis.Cmder) error { - return nil -} +func (rh *redisHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + startTime := time.Now() -func (redisHook) DialHook(next redis.DialHook) redis.DialHook { - return nil -} + err := next(ctx, cmd) + rh.registry.IncRedisRequest(err != nil && err != redis.Nil) + rh.registry.ObserveRedisRequestDuration(time.Since(startTime)) -func (redisHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { - return nil + return err + } } func (redisHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { diff --git a/util/cache/redis_hook.go b/util/cache/redis_hook.go index 455ad03eb5bbf..e7cc3f4bcc68e 100644 --- a/util/cache/redis_hook.go +++ b/util/cache/redis_hook.go @@ -2,14 +2,13 @@ package cache import ( "context" - "strings" + "errors" + "net" "github.com/redis/go-redis/v9" log "github.com/sirupsen/logrus" ) -const NoSuchHostErr = "no such host" - type argoRedisHooks struct { reconnectCallback func() } @@ -18,32 +17,23 @@ func NewArgoRedisHook(reconnectCallback func()) *argoRedisHooks { return &argoRedisHooks{reconnectCallback: reconnectCallback} } -func (hook *argoRedisHooks) BeforeProcess(ctx context.Context, cmd redis.Cmder) (context.Context, error) { - return ctx, nil -} - -func (hook *argoRedisHooks) AfterProcess(ctx context.Context, cmd redis.Cmder) error { - if cmd.Err() != nil && strings.Contains(cmd.Err().Error(), NoSuchHostErr) { - log.Warnf("Reconnect to redis because error: \"%v\"", cmd.Err()) - hook.reconnectCallback() - } - return nil -} - -func (hook *argoRedisHooks) BeforeProcessPipeline(ctx context.Context, cmds []redis.Cmder) (context.Context, error) { - return ctx, nil -} - -func (hook *argoRedisHooks) AfterProcessPipeline(ctx context.Context, cmds []redis.Cmder) error { - return nil -} - func (hook *argoRedisHooks) DialHook(next redis.DialHook) redis.DialHook { - return nil + return func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := next(ctx, network, addr) + return conn, err + } } func (hook *argoRedisHooks) ProcessHook(next redis.ProcessHook) redis.ProcessHook { - return nil + return func(ctx context.Context, cmd redis.Cmder) error { + var dnsError *net.DNSError + err := next(ctx, cmd) + if err != nil && errors.As(err, &dnsError) { + log.Warnf("Reconnect to redis because error: \"%v\"", err) + hook.reconnectCallback() + } + return err + } } func (hook *argoRedisHooks) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { diff --git a/util/cache/redis_hook_test.go b/util/cache/redis_hook_test.go index ef9e6a1c85537..4d7d9b7aaf41d 100644 --- a/util/cache/redis_hook_test.go +++ b/util/cache/redis_hook_test.go @@ -1,38 +1,53 @@ package cache import ( - "context" - "errors" "testing" + "time" + "github.com/alicebob/miniredis/v2" "github.com/stretchr/testify/assert" "github.com/redis/go-redis/v9" ) func Test_ReconnectCallbackHookCalled(t *testing.T) { + mr, err := miniredis.Run() + if err != nil { + panic(err) + } + defer mr.Close() + called := false hook := NewArgoRedisHook(func() { called = true }) - cmd := &redis.StringCmd{} - cmd.SetErr(errors.New("Failed to resync revoked tokens. retrying again in 1 minute: dial tcp: lookup argocd-redis on 10.179.0.10:53: no such host")) - - _ = hook.AfterProcess(context.Background(), cmd) + faultyDNSRedisClient := redis.NewClient(&redis.Options{Addr: "invalidredishost.invalid:12345"}) + faultyDNSRedisClient.AddHook(hook) + faultyDNSClient := NewRedisCache(faultyDNSRedisClient, 60*time.Second, RedisCompressionNone) + err = faultyDNSClient.Set(&Item{Key: "baz", Object: "foo"}) assert.Equal(t, called, true) + assert.Error(t, err) } func Test_ReconnectCallbackHookNotCalled(t *testing.T) { + mr, err := miniredis.Run() + if err != nil { + panic(err) + } + defer mr.Close() + called := false hook := NewArgoRedisHook(func() { called = true }) - cmd := &redis.StringCmd{} - cmd.SetErr(errors.New("Something wrong")) - _ = hook.AfterProcess(context.Background(), cmd) + redisClient := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + redisClient.AddHook(hook) + client := NewRedisCache(redisClient, 60*time.Second, RedisCompressionNone) + err = client.Set(&Item{Key: "foo", Object: "bar"}) assert.Equal(t, called, false) + assert.NoError(t, err) } diff --git a/util/cache/redis_test.go b/util/cache/redis_test.go index 3800753cee3ec..e05c7541f5ff1 100644 --- a/util/cache/redis_test.go +++ b/util/cache/redis_test.go @@ -2,14 +2,59 @@ package cache import ( "context" + "strconv" "testing" "time" + promcm "github.com/prometheus/client_model/go" + "github.com/alicebob/miniredis/v2" + "github.com/prometheus/client_golang/prometheus" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" ) +var ( + redisRequestCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "argocd_redis_request_total", + }, + []string{"initiator", "failed"}, + ) + redisRequestHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "argocd_redis_request_duration", + Buckets: []float64{0.1, 0.25, .5, 1, 2}, + }, + []string{"initiator"}, + ) +) + +type MockMetricsServer struct { + registry *prometheus.Registry + redisRequestCounter *prometheus.CounterVec + redisRequestHistogram *prometheus.HistogramVec +} + +func NewMockMetricsServer() *MockMetricsServer { + registry := prometheus.NewRegistry() + registry.MustRegister(redisRequestCounter) + registry.MustRegister(redisRequestHistogram) + return &MockMetricsServer{ + registry: registry, + redisRequestCounter: redisRequestCounter, + redisRequestHistogram: redisRequestHistogram, + } +} + +func (m *MockMetricsServer) IncRedisRequest(failed bool) { + m.redisRequestCounter.WithLabelValues("mock", strconv.FormatBool(failed)).Inc() +} + +func (m *MockMetricsServer) ObserveRedisRequestDuration(duration time.Duration) { + m.redisRequestHistogram.WithLabelValues("mock").Observe(duration.Seconds()) +} + func TestRedisSetCache(t *testing.T) { mr, err := miniredis.Run() if err != nil { @@ -70,3 +115,50 @@ func TestRedisSetCacheCompressed(t *testing.T) { assert.Equal(t, testValue, result) } + +func TestRedisMetrics(t *testing.T) { + mr, err := miniredis.Run() + if err != nil { + panic(err) + } + defer mr.Close() + + metric := &promcm.Metric{} + ms := NewMockMetricsServer() + redisClient := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + faultyRedisClient := redis.NewClient(&redis.Options{Addr: "invalidredishost.invalid:12345"}) + CollectMetrics(redisClient, ms) + CollectMetrics(faultyRedisClient, ms) + + client := NewRedisCache(redisClient, 60*time.Second, RedisCompressionNone) + faultyClient := NewRedisCache(faultyRedisClient, 60*time.Second, RedisCompressionNone) + var res string + + //client successful request + err = client.Set(&Item{Key: "foo", Object: "bar"}) + assert.NoError(t, err) + err = client.Get("foo", &res) + assert.NoError(t, err) + + c, err := ms.redisRequestCounter.GetMetricWithLabelValues("mock", "false") + assert.NoError(t, err) + err = c.Write(metric) + assert.NoError(t, err) + assert.Equal(t, metric.Counter.GetValue(), float64(2)) + + //faulty client failed request + err = faultyClient.Get("foo", &res) + assert.Error(t, err) + c, err = ms.redisRequestCounter.GetMetricWithLabelValues("mock", "true") + assert.NoError(t, err) + err = c.Write(metric) + assert.NoError(t, err) + assert.Equal(t, metric.Counter.GetValue(), float64(1)) + + //both clients histogram count + o, err := ms.redisRequestHistogram.GetMetricWithLabelValues("mock") + assert.NoError(t, err) + err = o.(prometheus.Metric).Write(metric) + assert.NoError(t, err) + assert.Equal(t, int(metric.Histogram.GetSampleCount()), 3) +} diff --git a/util/db/cluster.go b/util/db/cluster.go index dad8a62010adc..7b5f5508ef5fa 100644 --- a/util/db/cluster.go +++ b/util/db/cluster.go @@ -316,7 +316,7 @@ func (db *db) DeleteCluster(ctx context.Context, server string) error { return db.settingsMgr.ResyncInformers() } -// clusterToData converts a cluster object to string data for serialization to a secret +// clusterToSecret converts a cluster object to string data for serialization to a secret func clusterToSecret(c *appv1.Cluster, secret *apiv1.Secret) error { data := make(map[string][]byte) data["server"] = []byte(strings.TrimRight(c.Server, "/")) diff --git a/util/db/repository_secrets.go b/util/db/repository_secrets.go index 31152300b0b8b..2d96c1c3a99eb 100644 --- a/util/db/repository_secrets.go +++ b/util/db/repository_secrets.go @@ -489,6 +489,9 @@ func (s *secretsRepositoryBackend) getRepositoryCredentialIndex(repoCredentials for i, cred := range repoCredentials { credUrl := git.NormalizeGitURL(string(cred.Data["url"])) if strings.HasPrefix(repoURL, credUrl) { + if len(credUrl) == max { + log.Warnf("Found multiple credentials for repoURL: %s", repoURL) + } if len(credUrl) > max { max = len(credUrl) idx = i diff --git a/util/env/env.go b/util/env/env.go index 985484c1ae80b..0b8bf9e0c3708 100644 --- a/util/env/env.go +++ b/util/env/env.go @@ -151,8 +151,17 @@ func ParseDurationFromEnv(env string, defaultValue, min, max time.Duration) time return dur } -func StringFromEnv(env string, defaultValue string) string { - if str := os.Getenv(env); str != "" { +type StringFromEnvOpts struct { + // AllowEmpty allows the value to be empty as long as the environment variable is set. + AllowEmpty bool +} + +func StringFromEnv(env string, defaultValue string, opts ...StringFromEnvOpts) string { + opt := StringFromEnvOpts{} + for _, o := range opts { + opt.AllowEmpty = opt.AllowEmpty || o.AllowEmpty + } + if str, ok := os.LookupEnv(env); opt.AllowEmpty && ok || str != "" { return str } return defaultValue diff --git a/util/env/env_test.go b/util/env/env_test.go index 9178592ed3552..95c78ccde6d9d 100644 --- a/util/env/env_test.go +++ b/util/env/env_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "k8s.io/utils/pointer" ) func TestParseNumFromEnv(t *testing.T) { @@ -167,19 +168,25 @@ func TestStringFromEnv(t *testing.T) { testCases := []struct { name string - env string + env *string expected string def string + opts []StringFromEnvOpts }{ - {"Some string", "true", "true", def}, - {"Empty string with default", "", def, def}, - {"Empty string without default", "", "", ""}, + {"Some string", pointer.String("true"), "true", def, nil}, + {"Empty string with default", pointer.String(""), def, def, nil}, + {"Empty string without default", pointer.String(""), "", "", nil}, + {"No env variable with default allow empty", nil, "default", "default", []StringFromEnvOpts{{AllowEmpty: true}}}, + {"Some variable with default allow empty", pointer.String("true"), "true", "default", []StringFromEnvOpts{{AllowEmpty: true}}}, + {"Empty variable with default allow empty", pointer.String(""), "", "default", []StringFromEnvOpts{{AllowEmpty: true}}}, } for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { - t.Setenv(envKey, tt.env) - b := StringFromEnv(envKey, tt.def) + if tt.env != nil { + t.Setenv(envKey, *tt.env) + } + b := StringFromEnv(envKey, tt.def, tt.opts...) assert.Equal(t, tt.expected, b) }) } diff --git a/util/helm/cmd.go b/util/helm/cmd.go index 419c7daff5d3c..cc2a1388d65a2 100644 --- a/util/helm/cmd.go +++ b/util/helm/cmd.go @@ -91,6 +91,28 @@ func (c *Cmd) RegistryLogin(repo string, creds Creds) (string, error) { args = append(args, "--password", creds.Password) } + if creds.CAPath != "" { + args = append(args, "--ca-file", creds.CAPath) + } + + if len(creds.CertData) > 0 { + filePath, closer, err := writeToTmp(creds.CertData) + if err != nil { + return "", err + } + defer argoio.Close(closer) + args = append(args, "--cert-file", filePath) + } + + if len(creds.KeyData) > 0 { + filePath, closer, err := writeToTmp(creds.KeyData) + if err != nil { + return "", err + } + defer argoio.Close(closer) + args = append(args, "--key-file", filePath) + } + if creds.InsecureSkipVerify { args = append(args, "--insecure") } @@ -238,6 +260,25 @@ func (c *Cmd) PullOCI(repo string, chart string, version string, destination str if creds.CAPath != "" { args = append(args, "--ca-file", creds.CAPath) } + + if len(creds.CertData) > 0 { + filePath, closer, err := writeToTmp(creds.CertData) + if err != nil { + return "", err + } + defer argoio.Close(closer) + args = append(args, "--cert-file", filePath) + } + + if len(creds.KeyData) > 0 { + filePath, closer, err := writeToTmp(creds.KeyData) + if err != nil { + return "", err + } + defer argoio.Close(closer) + args = append(args, "--key-file", filePath) + } + if creds.InsecureSkipVerify && c.insecureSkipVerifySupported { args = append(args, "--insecure-skip-tls-verify") } diff --git a/util/kustomize/kustomize.go b/util/kustomize/kustomize.go index 2487b14b4903b..5c7ac51ac6122 100644 --- a/util/kustomize/kustomize.go +++ b/util/kustomize/kustomize.go @@ -35,8 +35,9 @@ type Kustomize interface { } // NewKustomizeApp create a new wrapper to run commands on the `kustomize` command-line tool. -func NewKustomizeApp(path string, creds git.Creds, fromRepo string, binaryPath string) Kustomize { +func NewKustomizeApp(repoRoot string, path string, creds git.Creds, fromRepo string, binaryPath string) Kustomize { return &kustomize{ + repoRoot: repoRoot, path: path, creds: creds, repo: fromRepo, @@ -45,6 +46,8 @@ func NewKustomizeApp(path string, creds git.Creds, fromRepo string, binaryPath s } type kustomize struct { + // path to the Git repository root + repoRoot string // path inside the checked out tree path string // creds structure @@ -181,6 +184,9 @@ func (k *kustomize) Build(opts *v1alpha1.ApplicationSourceKustomize, kustomizeOp if opts.ForceCommonLabels { args = append(args, "--force") } + if opts.LabelWithoutSelector { + args = append(args, "--without-selector") + } commonLabels := map[string]string{} for name, value := range opts.CommonLabels { commonLabels[name] = envVars.Envsubst(value) @@ -301,6 +307,7 @@ func (k *kustomize) Build(opts *v1alpha1.ApplicationSourceKustomize, kustomizeOp cmd = exec.Command(k.getBinaryPath(), "build", k.path) } cmd.Env = env + cmd.Dir = k.repoRoot out, err := executil.Run(cmd) if err != nil { return nil, nil, err @@ -315,7 +322,7 @@ func (k *kustomize) Build(opts *v1alpha1.ApplicationSourceKustomize, kustomizeOp } func parseKustomizeBuildOptions(path, buildOptions string) []string { - return append([]string{"build", path}, strings.Split(buildOptions, " ")...) + return append([]string{"build", path}, strings.Fields(buildOptions)...) } var KustomizationNames = []string{"kustomization.yaml", "kustomization.yml", "Kustomization"} diff --git a/util/kustomize/kustomize_test.go b/util/kustomize/kustomize_test.go index a6275cf01ae1b..cf57daaf4128c 100644 --- a/util/kustomize/kustomize_test.go +++ b/util/kustomize/kustomize_test.go @@ -24,6 +24,7 @@ const kustomization3 = "force_common" const kustomization4 = "custom_version" const kustomization5 = "kustomization_yaml_patches" const kustomization6 = "kustomization_yaml_components" +const kustomization7 = "label_without_selector" func testDataDir(tb testing.TB, testData string) (string, error) { res := tb.TempDir() @@ -40,7 +41,7 @@ func TestKustomizeBuild(t *testing.T) { namePrefix := "namePrefix-" nameSuffix := "-nameSuffix" namespace := "custom-namespace" - kustomize := NewKustomizeApp(appPath, git.NopCreds{}, "", "") + kustomize := NewKustomizeApp(appPath, appPath, git.NopCreds{}, "", "") env := &v1alpha1.Env{ &v1alpha1.EnvEntry{Name: "ARGOCD_APP_NAME", Value: "argo-cd-tests"}, } @@ -123,7 +124,7 @@ func TestKustomizeBuild(t *testing.T) { func TestFailKustomizeBuild(t *testing.T) { appPath, err := testDataDir(t, kustomization1) assert.Nil(t, err) - kustomize := NewKustomizeApp(appPath, git.NopCreds{}, "", "") + kustomize := NewKustomizeApp(appPath, appPath, git.NopCreds{}, "", "") kustomizeSource := v1alpha1.ApplicationSourceKustomize{ Replicas: []v1alpha1.KustomizeReplica{ { @@ -222,7 +223,7 @@ func TestKustomizeBuildForceCommonLabels(t *testing.T) { for _, tc := range testCases { appPath, err := testDataDir(t, tc.TestData) assert.Nil(t, err) - kustomize := NewKustomizeApp(appPath, git.NopCreds{}, "", "") + kustomize := NewKustomizeApp(appPath, appPath, git.NopCreds{}, "", "") objs, _, err := kustomize.Build(&tc.KustomizeSource, nil, tc.Env) switch tc.ExpectErr { case true: @@ -314,7 +315,7 @@ func TestKustomizeBuildForceCommonAnnotations(t *testing.T) { for _, tc := range testCases { appPath, err := testDataDir(t, tc.TestData) assert.Nil(t, err) - kustomize := NewKustomizeApp(appPath, git.NopCreds{}, "", "") + kustomize := NewKustomizeApp(appPath, appPath, git.NopCreds{}, "", "") objs, _, err := kustomize.Build(&tc.KustomizeSource, nil, tc.Env) switch tc.ExpectErr { case true: @@ -328,13 +329,91 @@ func TestKustomizeBuildForceCommonAnnotations(t *testing.T) { } } +func TestKustomizeLabelWithoutSelector(t *testing.T) { + type testCase struct { + TestData string + KustomizeSource v1alpha1.ApplicationSourceKustomize + ExpectedMetadataLabels map[string]string + ExpectedSelectorLabels map[string]string + ExpectedTemplateLabels map[string]string + ExpectErr bool + Env *v1alpha1.Env + } + testCases := []testCase{ + { + TestData: kustomization7, + KustomizeSource: v1alpha1.ApplicationSourceKustomize{ + CommonLabels: map[string]string{ + "foo": "bar", + }, + LabelWithoutSelector: true, + }, + ExpectedMetadataLabels: map[string]string{"app": "nginx", "managed-by": "helm", "foo": "bar"}, + ExpectedSelectorLabels: map[string]string{"app": "nginx"}, + ExpectedTemplateLabels: map[string]string{"app": "nginx"}, + Env: &v1alpha1.Env{ + &v1alpha1.EnvEntry{ + Name: "ARGOCD_APP_NAME", + Value: "argo-cd-tests", + }, + }, + }, + { + TestData: kustomization7, + KustomizeSource: v1alpha1.ApplicationSourceKustomize{ + CommonLabels: map[string]string{ + "managed-by": "argocd", + }, + LabelWithoutSelector: true, + ForceCommonLabels: true, + }, + ExpectedMetadataLabels: map[string]string{"app": "nginx", "managed-by": "argocd"}, + ExpectedSelectorLabels: map[string]string{"app": "nginx"}, + ExpectedTemplateLabels: map[string]string{"app": "nginx"}, + Env: &v1alpha1.Env{ + &v1alpha1.EnvEntry{ + Name: "ARGOCD_APP_NAME", + Value: "argo-cd-tests", + }, + }, + }, + } + + for _, tc := range testCases { + appPath, err := testDataDir(t, tc.TestData) + assert.Nil(t, err) + kustomize := NewKustomizeApp(appPath, appPath, git.NopCreds{}, "", "") + objs, _, err := kustomize.Build(&tc.KustomizeSource, nil, tc.Env) + + switch tc.ExpectErr { + case true: + assert.Error(t, err) + default: + assert.Nil(t, err) + if assert.Equal(t, len(objs), 1) { + obj := objs[0] + sl, found, err := unstructured.NestedStringMap(obj.Object, "spec", "selector", "matchLabels") + assert.Nil(t, err) + assert.Equal(t, found, true) + tl, found, err := unstructured.NestedStringMap(obj.Object, "spec", "template", "metadata", "labels") + assert.Nil(t, err) + assert.Equal(t, found, true) + assert.Equal(t, tc.ExpectedMetadataLabels, obj.GetLabels()) + assert.Equal(t, tc.ExpectedSelectorLabels, sl) + assert.Equal(t, tc.ExpectedTemplateLabels, tl) + } + } + } + +} + func TestKustomizeCustomVersion(t *testing.T) { appPath, err := testDataDir(t, kustomization1) assert.Nil(t, err) kustomizePath, err := testDataDir(t, kustomization4) assert.Nil(t, err) envOutputFile := kustomizePath + "/env_output" - kustomize := NewKustomizeApp(appPath, git.NopCreds{}, "", kustomizePath+"/kustomize.special") + kustomize := NewKustomizeApp(appPath, appPath, git.NopCreds{}, "", kustomizePath+"/kustomize.special") kustomizeSource := v1alpha1.ApplicationSourceKustomize{ Version: "special", } @@ -356,7 +435,7 @@ func TestKustomizeCustomVersion(t *testing.T) { func TestKustomizeBuildComponents(t *testing.T) { appPath, err := testDataDir(t, kustomization6) assert.Nil(t, err) - kustomize := NewKustomizeApp(appPath, git.NopCreds{}, "", "") + kustomize := NewKustomizeApp(appPath, appPath, git.NopCreds{}, "", "") kustomizeSource := v1alpha1.ApplicationSourceKustomize{ Components: []string{"./components"}, @@ -377,7 +456,7 @@ func TestKustomizeBuildComponents(t *testing.T) { func TestKustomizeBuildPatches(t *testing.T) { appPath, err := testDataDir(t, kustomization5) assert.Nil(t, err) - kustomize := NewKustomizeApp(appPath, git.NopCreds{}, "", "") + kustomize := NewKustomizeApp(appPath, appPath, git.NopCreds{}, "", "") kustomizeSource := v1alpha1.ApplicationSourceKustomize{ Patches: []v1alpha1.KustomizePatch{ diff --git a/util/kustomize/testdata/label_without_selector/deployment.yaml b/util/kustomize/testdata/label_without_selector/deployment.yaml new file mode 100644 index 0000000000000..fa161556fb2b0 --- /dev/null +++ b/util/kustomize/testdata/label_without_selector/deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx + managed-by: helm +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.15.4 + ports: + - containerPort: 80 diff --git a/util/kustomize/testdata/label_without_selector/kustomization.yaml b/util/kustomize/testdata/label_without_selector/kustomization.yaml new file mode 100644 index 0000000000000..9e90331c9bca0 --- /dev/null +++ b/util/kustomize/testdata/label_without_selector/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - ./deployment.yaml \ No newline at end of file diff --git a/util/notification/argocd/service.go b/util/notification/argocd/service.go index 61e0bebeca3c6..426217318ce31 100644 --- a/util/notification/argocd/service.go +++ b/util/notification/argocd/service.go @@ -18,7 +18,7 @@ import ( type Service interface { GetCommitMetadata(ctx context.Context, repoURL string, commitSHA string) (*shared.CommitMetadata, error) - GetAppDetails(ctx context.Context, appSource *v1alpha1.ApplicationSource) (*shared.AppDetail, error) + GetAppDetails(ctx context.Context, appSource *v1alpha1.ApplicationSource, appName string) (*shared.AppDetail, error) } func NewArgoCDService(clientset kubernetes.Interface, namespace string, repoClientset apiclient.Clientset) (*argoCDService, error) { @@ -76,7 +76,7 @@ func (svc *argoCDService) getKustomizeOptions(source *v1alpha1.ApplicationSource return kustomizeSettings.GetOptions(*source) } -func (svc *argoCDService) GetAppDetails(ctx context.Context, appSource *v1alpha1.ApplicationSource) (*shared.AppDetail, error) { +func (svc *argoCDService) GetAppDetails(ctx context.Context, appSource *v1alpha1.ApplicationSource, appName string) (*shared.AppDetail, error) { argocdDB := db.NewDB(svc.namespace, svc.settingsMgr, svc.clientset) repo, err := argocdDB.GetRepository(ctx, appSource.RepoURL) if err != nil { @@ -95,6 +95,7 @@ func (svc *argoCDService) GetAppDetails(ctx context.Context, appSource *v1alpha1 return nil, err } appDetail, err := svc.repoServerClient.GetAppDetails(ctx, &apiclient.RepoServerAppDetailsQuery{ + AppName: appName, Repo: repo, Source: appSource, Repos: helmRepos, diff --git a/util/notification/expression/repo/repo.go b/util/notification/expression/repo/repo.go index 060060cbccd68..a782c0b7c1725 100644 --- a/util/notification/expression/repo/repo.go +++ b/util/notification/expression/repo/repo.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "net/url" "regexp" "strings" @@ -22,25 +23,25 @@ var ( gitSuffix = regexp.MustCompile(`\.git$`) ) -func getApplicationSource(obj *unstructured.Unstructured) (*v1alpha1.ApplicationSource, error) { +func getApplicationSourceAndName(obj *unstructured.Unstructured) (*v1alpha1.ApplicationSource, string, error) { data, err := json.Marshal(obj) if err != nil { - return nil, err + return nil, "", err } application := &v1alpha1.Application{} err = json.Unmarshal(data, application) if err != nil { - return nil, err + return nil, "", err } - return application.Spec.GetSourcePtr(), nil + return application.Spec.GetSourcePtr(), application.GetName(), nil } func getAppDetails(app *unstructured.Unstructured, argocdService service.Service) (*shared.AppDetail, error) { - appSource, err := getApplicationSource(app) + appSource, appName, err := getApplicationSourceAndName(app) if err != nil { return nil, err } - appDetail, err := argocdService.GetAppDetails(context.Background(), appSource) + appDetail, err := argocdService.GetAppDetails(context.Background(), appSource, appName) if err != nil { return nil, err } @@ -90,6 +91,7 @@ func NewExprs(argocdService service.Service, app *unstructured.Unstructured) map return map[string]interface{}{ "RepoURLToHTTPS": repoURLToHTTPS, "FullNameByRepoURL": FullNameByRepoURL, + "QueryEscape": url.QueryEscape, "GetCommitMetadata": func(commitSHA string) interface{} { meta, err := getCommitMetadata(commitSHA, app, argocdService) if err != nil { diff --git a/util/oidc/provider.go b/util/oidc/provider.go index fcb1a95b60f4f..d75bcf97efecd 100644 --- a/util/oidc/provider.go +++ b/util/oidc/provider.go @@ -73,6 +73,18 @@ func (p *providerImpl) newGoOIDCProvider() (*gooidc.Provider, error) { return prov, nil } +type tokenVerificationError struct { + errorsByAudience map[string]error +} + +func (t tokenVerificationError) Error() string { + var errorStrings []string + for aud, err := range t.errorsByAudience { + errorStrings = append(errorStrings, fmt.Sprintf("error for aud %q: %v", aud, err)) + } + return fmt.Sprintf("token verification failed for all audiences: %s", strings.Join(errorStrings, ", ")) +} + func (p *providerImpl) Verify(tokenString string, argoSettings *settings.ArgoCDSettings) (*gooidc.IDToken, error) { // According to the JWT spec, the aud claim is optional. The spec also says (emphasis mine): // @@ -104,6 +116,7 @@ func (p *providerImpl) Verify(tokenString string, argoSettings *settings.ArgoCDS if len(allowedAudiences) == 0 { return nil, errors.New("token has an audience claim, but no allowed audiences are configured") } + tokenVerificationErrors := make(map[string]error) // Token must be verified for at least one allowed audience for _, aud := range allowedAudiences { idToken, err = p.verify(aud, tokenString, false) @@ -117,6 +130,13 @@ func (p *providerImpl) Verify(tokenString string, argoSettings *settings.ArgoCDS if err == nil { break } + // We store the error for each audience so that we can return a more detailed error message to the user. + // If this gets merged, we'll be able to detect failures unrelated to audiences and short-circuit this loop + // to avoid logging irrelevant warnings: https://github.com/coreos/go-oidc/pull/406 + tokenVerificationErrors[aud] = err + } + if len(tokenVerificationErrors) > 0 { + err = tokenVerificationError{errorsByAudience: tokenVerificationErrors} } } diff --git a/util/settings/settings.go b/util/settings/settings.go index baff450aa817e..82b4d72dc23c8 100644 --- a/util/settings/settings.go +++ b/util/settings/settings.go @@ -1495,27 +1495,6 @@ func (mgr *SettingsManager) updateSettingsFromSecret(settings *ArgoCDSettings, a } else { errs = append(errs, &incompleteSettingsError{message: "server.secretkey is missing"}) } - if githubWebhookSecret := argoCDSecret.Data[settingsWebhookGitHubSecretKey]; len(githubWebhookSecret) > 0 { - settings.WebhookGitHubSecret = string(githubWebhookSecret) - } - if gitlabWebhookSecret := argoCDSecret.Data[settingsWebhookGitLabSecretKey]; len(gitlabWebhookSecret) > 0 { - settings.WebhookGitLabSecret = string(gitlabWebhookSecret) - } - if bitbucketWebhookUUID := argoCDSecret.Data[settingsWebhookBitbucketUUIDKey]; len(bitbucketWebhookUUID) > 0 { - settings.WebhookBitbucketUUID = string(bitbucketWebhookUUID) - } - if bitbucketserverWebhookSecret := argoCDSecret.Data[settingsWebhookBitbucketServerSecretKey]; len(bitbucketserverWebhookSecret) > 0 { - settings.WebhookBitbucketServerSecret = string(bitbucketserverWebhookSecret) - } - if gogsWebhookSecret := argoCDSecret.Data[settingsWebhookGogsSecretKey]; len(gogsWebhookSecret) > 0 { - settings.WebhookGogsSecret = string(gogsWebhookSecret) - } - if azureDevOpsUsername := argoCDSecret.Data[settingsWebhookAzureDevOpsUsernameKey]; len(azureDevOpsUsername) > 0 { - settings.WebhookAzureDevOpsUsername = string(azureDevOpsUsername) - } - if azureDevOpsPassword := argoCDSecret.Data[settingsWebhookAzureDevOpsPasswordKey]; len(azureDevOpsPassword) > 0 { - settings.WebhookAzureDevOpsPassword = string(azureDevOpsPassword) - } // The TLS certificate may be externally managed. We try to load it from an // external secret first. If the external secret doesn't exist, we either @@ -1555,6 +1534,15 @@ func (mgr *SettingsManager) updateSettingsFromSecret(settings *ArgoCDSettings, a if len(errs) > 0 { return errs[0] } + + settings.WebhookGitHubSecret = ReplaceStringSecret(string(argoCDSecret.Data[settingsWebhookGitHubSecretKey]), settings.Secrets) + settings.WebhookGitLabSecret = ReplaceStringSecret(string(argoCDSecret.Data[settingsWebhookGitLabSecretKey]), settings.Secrets) + settings.WebhookBitbucketUUID = ReplaceStringSecret(string(argoCDSecret.Data[settingsWebhookBitbucketUUIDKey]), settings.Secrets) + settings.WebhookBitbucketServerSecret = ReplaceStringSecret(string(argoCDSecret.Data[settingsWebhookBitbucketServerSecretKey]), settings.Secrets) + settings.WebhookGogsSecret = ReplaceStringSecret(string(argoCDSecret.Data[settingsWebhookGogsSecretKey]), settings.Secrets) + settings.WebhookAzureDevOpsUsername = ReplaceStringSecret(string(argoCDSecret.Data[settingsWebhookAzureDevOpsUsernameKey]), settings.Secrets) + settings.WebhookAzureDevOpsPassword = ReplaceStringSecret(string(argoCDSecret.Data[settingsWebhookAzureDevOpsPasswordKey]), settings.Secrets) + return nil } diff --git a/util/settings/settings_test.go b/util/settings/settings_test.go index 07a2c268a6bd7..e11fa401a9fc9 100644 --- a/util/settings/settings_test.go +++ b/util/settings/settings_test.go @@ -1241,9 +1241,9 @@ func TestDownloadArgoCDBinaryUrls(t *testing.T) { func TestSecretKeyRef(t *testing.T) { data := map[string]string{ "oidc.config": `name: Okta -issuer: $acme:issuerSecret +issuer: $ext:issuerSecret clientID: aaaabbbbccccddddeee -clientSecret: $acme:clientSecret +clientSecret: $ext:clientSecret # Optional set of OIDC scopes to request. If omitted, defaults to: ["openid", "profile", "email", "groups"] requestedScopes: ["openid", "profile", "email"] # Optional set of OIDC claims to request on the ID token. @@ -1265,21 +1265,23 @@ requestedIDTokenClaims: {"groups": {"essential": true}}`, Namespace: "default", }, Data: map[string][]byte{ - "admin.password": nil, - "server.secretkey": nil, + "admin.password": nil, + "server.secretkey": nil, + "webhook.github.secret": []byte("$ext:webhook.github.secret"), }, } secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "acme", + Name: "ext", Namespace: "default", Labels: map[string]string{ "app.kubernetes.io/part-of": "argocd", }, }, Data: map[string][]byte{ - "issuerSecret": []byte("https://dev-123456.oktapreview.com"), - "clientSecret": []byte("deadbeef"), + "issuerSecret": []byte("https://dev-123456.oktapreview.com"), + "clientSecret": []byte("deadbeef"), + "webhook.github.secret": []byte("mywebhooksecret"), }, } kubeClient := fake.NewSimpleClientset(cm, secret, argocdSecret) @@ -1287,6 +1289,7 @@ requestedIDTokenClaims: {"groups": {"essential": true}}`, settings, err := settingsManager.GetSettings() assert.NoError(t, err) + assert.Equal(t, settings.WebhookGitHubSecret, "mywebhooksecret") oidcConfig := settings.OIDCConfig() assert.Equal(t, oidcConfig.Issuer, "https://dev-123456.oktapreview.com") diff --git a/util/test/testutil.go b/util/test/testutil.go index 1cb23bc08bb3e..bb6c43313358c 100644 --- a/util/test/testutil.go +++ b/util/test/testutil.go @@ -8,9 +8,9 @@ import ( "net/http/httptest" "testing" + "github.com/go-jose/go-jose/v3" "github.com/golang-jwt/jwt/v4" "github.com/stretchr/testify/require" - "gopkg.in/square/go-jose.v2" ) // Cert is a certificate for tests. It was generated like this: diff --git a/util/webhook/webhook.go b/util/webhook/webhook.go index 25bd92e11802c..04746a1df0e37 100644 --- a/util/webhook/webhook.go +++ b/util/webhook/webhook.go @@ -321,7 +321,7 @@ func getWebUrlRegex(webURL string) (*regexp.Regexp, error) { } regexEscapedHostname := regexp.QuoteMeta(urlObj.Hostname()) - regexEscapedPath := regexp.QuoteMeta(urlObj.Path[1:]) + regexEscapedPath := regexp.QuoteMeta(urlObj.EscapedPath()[1:]) regexpStr := fmt.Sprintf(`(?i)^(http://|https://|%s@|ssh://(%s@)?)%s(:[0-9]+|)[:/]%s(\.git)?$`, usernameRegex, usernameRegex, regexEscapedHostname, regexEscapedPath) repoRegexp, err := regexp.Compile(regexpStr) diff --git a/util/webhook/webhook_test.go b/util/webhook/webhook_test.go index b241d7c671841..3097dc58f574e 100644 --- a/util/webhook/webhook_test.go +++ b/util/webhook/webhook_test.go @@ -670,6 +670,7 @@ func Test_getWebUrlRegex(t *testing.T) { {true, "https://example.com/org/repo", "ssh://user-name@example.com/org/repo", "valid usernames with hyphens in repo should match"}, {false, "https://example.com/org/repo", "ssh://-user-name@example.com/org/repo", "invalid usernames with hyphens in repo should not match"}, {true, "https://example.com:443/org/repo", "GIT@EXAMPLE.COM:22:ORG/REPO", "matches aren't case-sensitive"}, + {true, "https://example.com/org/repo%20", "https://example.com/org/repo%20", "escape codes in path are preserved"}, } for _, testCase := range tests { testCopy := testCase