From 0ad65ded6ea2e42ae0c48b34c92c5c34d1800cc0 Mon Sep 17 00:00:00 2001 From: Vasilii Ilichev <110685051+mrmoonl1ght94@users.noreply.github.com> Date: Mon, 12 Feb 2024 15:47:04 +0800 Subject: [PATCH 01/50] Infra: Replace docker hub with GitHub packages (#74) --- .github/workflows/master.yaml | 11 ++++++++--- .github/workflows/release.yaml | 10 ++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml index cd2ee75ab..478bd1e77 100644 --- a/.github/workflows/master.yaml +++ b/.github/workflows/master.yaml @@ -8,7 +8,7 @@ jobs: build: runs-on: ubuntu-latest permissions: # TODO remove when public - contents: read + packages: write steps: - uses: actions/checkout@v4 with: @@ -48,8 +48,14 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push - if: false # TODO fix for github images id: docker_build_and_push uses: docker/build-push-action@v5 with: @@ -71,7 +77,6 @@ jobs: # # ################################# - name: update-master-deployment - if: false # TODO remove after infra fix run: | git clone https://infra-tech:${{ secrets.INFRA_USER_ACCESS_TOKEN }}@github.com/kafbat/ui-infra.git --branch main cd kafka-ui-infra/aws-infrastructure4eks/argocd/scripts diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 505d0e7e7..ca0e0ee18 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,8 @@ jobs: release: runs-on: ubuntu-latest permissions: # TODO remove when public - contents: read + contents: write + packages: write outputs: version: ${{steps.build.outputs.version}} steps: @@ -67,8 +68,13 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - if: false # TODO fix for github images id: docker_build_and_push uses: docker/build-push-action@v5 with: From 3be4707d20feb1ed01c2708b208fbc88e62dcaf2 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Mon, 12 Feb 2024 15:03:49 +0700 Subject: [PATCH 02/50] Infra: Workflows minor refactor (#118) --- .github/workflows/backend.yml | 29 +++++++++---- .github/workflows/codeql-analysis.yml | 43 ++++---------------- .github/workflows/e2e-tests.yaml | 9 ++-- .github/workflows/frontend.yaml | 11 ++--- .github/workflows/{master.yaml => main.yaml} | 31 ++++---------- .github/workflows/release_drafter.yml | 3 +- .github/workflows/workflow_linter.yaml | 6 +-- 7 files changed, 49 insertions(+), 83 deletions(-) rename .github/workflows/{master.yaml => main.yaml} (66%) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 82c9e532d..6b10cb43a 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -1,38 +1,48 @@ name: "Backend: PR/main build & test" + on: push: - branches: - - main - pull_request_target: - types: ["opened", "edited", "reopened", "synchronize"] - paths: - - "api/**" + branches: [ "main" ] + paths: &backend_paths - "pom.xml" -permissions: # TODO remove when public + - "contract/**" + - "api/**" + - "serde-api/**" + + pull_request_target: + types: [ "opened", "reopened", "synchronize" ] + paths: *backend_paths + +permissions: checks: write pull-requests: write - contents: read + jobs: build-and-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + + - name: Checkout + uses: actions/checkout@v4 with: token: ${{ github.token }} # TODO remove when public fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} + - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' + - name: Cache SonarCloud packages uses: actions/cache@v3 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar + - name: Build and analyze pull request target if: ${{ github.event_name == 'pull_request' }} env: @@ -49,6 +59,7 @@ jobs: -Dsonar.pullrequest.key=${{ github.event.pull_request.number }} \ -Dsonar.pullrequest.branch=$HEAD_REF \ -Dsonar.pullrequest.base=$BASE_REF + - name: Build and analyze push main if: ${{ github.event_name == 'push' }} env: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 935ccf47a..cf792b50f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,27 +1,19 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL" on: push: branches: [ main ] + pull_request: # The branches below must be a subset of the branches above branches: [ main ] paths: - - 'contract/**' - - 'frontend/**' - - 'api/**' - - 'serde-api/**' + - 'frontend/**' + - "pom.xml" + - "contract/**" + - 'api/**' + - 'serde-api/**' + schedule: - cron: '39 15 * * 6' @@ -35,9 +27,6 @@ jobs: fail-fast: false matrix: language: [ 'javascript', 'java' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - name: Checkout repository @@ -45,15 +34,10 @@ jobs: with: token: ${{ github.token }} # TODO remove when public - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - name: Set up JDK uses: actions/setup-java@v3 @@ -62,21 +46,8 @@ jobs: distribution: 'zulu' cache: 'maven' - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 320195f39..0e28e6b51 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -1,16 +1,19 @@ name: "E2E: PR healthcheck" on: pull_request_target: - types: [ "opened", "edited", "reopened", "synchronize" ] + types: [ "opened", "reopened", "synchronize" ] paths: - - "api/**" + - "pom.xml" - "contract/**" + - "api/**" + - "serde-api/**" - "frontend/**" - "e2e-tests/**" - - "pom.xml" + permissions: # TODO remove when public statuses: write contents: read + jobs: build-and-test: runs-on: ubuntu-latest diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index b5bbe6421..6caf2f042 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -1,14 +1,15 @@ name: "Frontend: PR/main build & test" on: push: - branches: - - main - pull_request_target: - types: ["opened", "edited", "reopened", "synchronize"] - paths: + branches: [ "main" ] + paths: &frontend_paths - "contract/**" - "frontend/**" + pull_request_target: + types: [ "opened", "reopened", "synchronize" ] + paths: *frontend_paths + jobs: build-and-test: env: diff --git a/.github/workflows/master.yaml b/.github/workflows/main.yaml similarity index 66% rename from .github/workflows/master.yaml rename to .github/workflows/main.yaml index 478bd1e77..f304cefa6 100644 --- a/.github/workflows/master.yaml +++ b/.github/workflows/main.yaml @@ -1,6 +1,7 @@ name: "Main: Build & deploy" on: workflow_dispatch: + push: branches: [ "main" ] @@ -10,7 +11,8 @@ jobs: permissions: # TODO remove when public packages: write steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} token: ${{ github.token }} # TODO remove when public @@ -22,18 +24,16 @@ jobs: distribution: 'zulu' cache: 'maven' - - name: Build + - name: Build a jar id: build run: | ./mvnw -B -ntp versions:set -DnewVersion=$GITHUB_SHA ./mvnw -V -B -ntp clean package -Pprod -DskipTests export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) echo "version=${VERSION}" >> $GITHUB_OUTPUT -################# -# # -# Docker images # -# # -################# + + # docker images + - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -55,7 +55,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push + - name: Build & push docker image id: docker_build_and_push uses: docker/build-push-action@v5 with: @@ -71,18 +71,3 @@ jobs: JAR_FILE=api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache -################################# -# # -# Main image digest update # -# # -################################# - - name: update-master-deployment - run: | - git clone https://infra-tech:${{ secrets.INFRA_USER_ACCESS_TOKEN }}@github.com/kafbat/ui-infra.git --branch main - cd kafka-ui-infra/aws-infrastructure4eks/argocd/scripts - echo "Image digest is:${{ steps.docker_build_and_push.outputs.digest }}" - ./kafka-ui-update-master-digest.sh ${{ steps.docker_build_and_push.outputs.digest }} - git config --global user.email "infra@kafbat.io" - git config --global user.name "infra-tech" - git add ../kafka-ui/* - git commit -m "updated master image digest: ${{ steps.docker_build_and_push.outputs.digest }}" && git push diff --git a/.github/workflows/release_drafter.yml b/.github/workflows/release_drafter.yml index 8c73bdd06..6614c88aa 100644 --- a/.github/workflows/release_drafter.yml +++ b/.github/workflows/release_drafter.yml @@ -2,8 +2,7 @@ name: "Infra: Release Drafter run" on: push: - branches: - - main + branches: [ "main" ] workflow_dispatch: inputs: version: diff --git a/.github/workflows/workflow_linter.yaml b/.github/workflows/workflow_linter.yaml index c0a05c668..bb6889bd0 100644 --- a/.github/workflows/workflow_linter.yaml +++ b/.github/workflows/workflow_linter.yaml @@ -1,11 +1,7 @@ name: "Infra: Workflow linter" on: pull_request: - types: - - "opened" - - "reopened" - - "synchronize" - - "edited" + types: ["opened", "reopened", "synchronize"] paths: - ".github/workflows/**" From 43030f1439c0beddf6a29adb0ff24e9511dd7b45 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Mon, 12 Feb 2024 15:07:05 +0700 Subject: [PATCH 03/50] Infra: Fix tmp checkout --- .github/workflows/main.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f304cefa6..14726665d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -8,8 +8,10 @@ on: jobs: build: runs-on: ubuntu-latest - permissions: # TODO remove when public + permissions: + contents: read # TODO remove when public packages: write + steps: - name: Checkout uses: actions/checkout@v4 From 9a64dcacc17642022a92d8ae698abe95bd4c1b39 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Mon, 12 Feb 2024 16:46:56 +0700 Subject: [PATCH 04/50] Infra: Update bots configs --- .github/dependabot.yml | 92 ++++--------------- .github/kapybro/config.yml | 175 +++++++++++++++++++++---------------- 2 files changed, 115 insertions(+), 152 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f004d6a2a..01f325c5e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,102 +1,42 @@ version: 2 updates: + - package-ecosystem: maven directory: "/" schedule: - interval: daily + interval: weekly time: "10:00" - timezone: Europe/Moscow + timezone: Europe/London reviewers: - - "Haarolean" + - "kafbat/backend" + open-pull-requests-limit: 10 labels: - - "scope/backend" - "type/dependencies" + - "scope/backend" + - package-ecosystem: npm directory: "/frontend" schedule: interval: weekly time: "10:00" - timezone: Europe/Moscow + timezone: Europe/London + reviewers: + - "kafbat/frontend" open-pull-requests-limit: 10 versioning-strategy: increase-if-necessary labels: - - "scope/frontend" - "type/dependencies" - ignore: - - dependency-name: react-hook-form - versions: - - 6.15.5 - - 7.0.0 - - 7.0.6 - - dependency-name: "@hookform/error-message" - versions: - - 1.1.0 - - dependency-name: use-debounce - versions: - - 6.0.0 - - 6.0.1 - - dependency-name: "@rooks/use-outside-click-ref" - versions: - - 4.10.1 - - dependency-name: react-multi-select-component - versions: - - 3.1.6 - - 4.0.0 - - dependency-name: husky - versions: - - 5.1.3 - - 5.2.0 - - 6.0.0 - - dependency-name: "@types/node-fetch" - versions: - - 2.5.9 - - dependency-name: "@testing-library/jest-dom" - versions: - - 5.11.10 - - dependency-name: "@typescript-eslint/eslint-plugin" - versions: - - 4.20.0 - - dependency-name: "@openapitools/openapi-generator-cli" - versions: - - 2.2.5 - - dependency-name: "@typescript-eslint/parser" - versions: - - 4.20.0 - - dependency-name: react-datepicker - versions: - - 3.7.0 - - dependency-name: eslint - versions: - - 7.23.0 - - dependency-name: "@testing-library/user-event" - versions: - - 13.0.6 - - dependency-name: immer - versions: - - 9.0.1 - - dependency-name: react-scripts - versions: - - 4.0.3 - - dependency-name: eslint-config-prettier - versions: - - 8.1.0 - - dependency-name: "@testing-library/react" - versions: - - 11.2.5 - - dependency-name: lodash - versions: - - 4.17.21 - - dependency-name: react-json-tree - versions: - - 0.15.0 + - "scope/frontend" + - package-ecosystem: "github-actions" directory: "/" schedule: interval: weekly time: "10:00" - timezone: Europe/Moscow + timezone: Europe/London reviewers: - - "Haarolean" + - "kafbat/devops" + open-pull-requests-limit: 10 labels: - - "scope/infrastructure" - "type/dependencies" + - "scope/infrastructure" diff --git a/.github/kapybro/config.yml b/.github/kapybro/config.yml index 591d4711e..5e2d7a1d1 100644 --- a/.github/kapybro/config.yml +++ b/.github/kapybro/config.yml @@ -1,95 +1,118 @@ -version: "0.1" +version: "0.2" settings: -maintainers: - teams: [ "maintainers" ] - users: [] -triage: - enabled: true +maintainers: &maintainers + teams: [ "maintainers" ] + users: [ ] - label: "status/triage" - remove-assignees: false +anchors: - skip: + skip_main: &skip_off enabled: false - do-manual-triage: false - labels: - add: [ "status/triage/skipped" ] - remove: [ ] + + skip_maintainers: &skip_maintainers + enabled: true conditions: - labels: [ "needs-attention", "needs-discussion" ] - authors: - teams: [ "maintainers" ] - comment: - enabled: false - message: "Automated triaging skipped." - - manual: + authors: *maintainers + + keyword-labeling: &keyword-labeling enabled: true + case-sensitive: false + scan: + title: true + body: false + rules: + # : [ ] + acl: [ "area/acl" ] + audit: [ "area/audit" ] + auth: [ "area/auth" ] + consumers: [ "area/consumers" ] + messages: [ "area/messages" ] + rbac: [ "area/rbac" ] + serde: [ "area/serde" ] + SR: [ "area/sr" ] + topics: [ "area/topics" ] + ux: [ "area/ux" ] + wizard: [ "area/wizard" ] + ### + backend: [ "scope/backend" ] + frontend: [ "scope/frontend" ] + infra: [ "scope/infra" ] + ### + bug: [ "type/bug" ] + feature: [ "type/feature" ] + enh: [ "type/enhancement" ] + + automatic: &automatic + enabled: true + skip: *skip_off + label: "status/triage" + remove-assignees: false + keyword-labeling: *keyword-labeling + completed: + labels: + add: [ "status/triage/completed" ] + remove: [] + + manual: &manual + enabled: true + skip: *skip_maintainers label: "status/triage/manual" labels: - add: [] - remove: [] - assign-maintainers: true - comment: - enabled: false - message: "Automated triaging completed. Pending human maintainer 🤖" - - repeating: - enabled: true + add: [ ] + remove: [ ] + assign-maintainers: false + + repeating: &repeating + enabled: false + skip: *skip_maintainers triggers: reopen: false - triage-labels: true + triage-labels: false labels: [ "status/retriage" ] - only-manual: false - completed: - labels: - add: [ "status/triage/completed" ] - remove: [] - -keyword-labeling: - enabled: true - case-sensitive: false - match-title: true - match-body: true - rules: - # :