Merge branch 'v3' into pmm3-mongo-mlaunch

Percona-Lab · Jan 7, 2025 · d8fedca · d8fedca
2 parents 1776cc3 + 136d172
commit d8fedca
Show file tree

Hide file tree

Showing 38 changed files with 369 additions and 177 deletions.
diff --git a/.github/workflows/PMM_PDPGSQL.yaml b/.github/workflows/PMM_PDPGSQL.yaml
@@ -3,9 +3,9 @@ on:
   workflow_dispatch:
     inputs:
       pdpgsql_version:
-        description: "PDPGSQL Version available Image versions on github, example: 14, 15, 16"
+        description: "PDPGSQL Version available Image versions on github, example: 15, 16, 17"
         required: true
-        default: '16'
+        default: '17'
         type: string
       pmm_qa_branch:
         description: "Branch for qa-integration to checkout"
@@ -39,7 +39,7 @@ jobs:
     timeout-minutes: 20
     env:
       ADMIN_PASSWORD: 'admin'
-      PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || '16' }}
+      PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || '17' }}
       PMM_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }}
       PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }}
       PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }}

diff --git a/.github/workflows/PMM_PXC_PROXYSQL.yaml → .github/workflows/PMM_PROXYSQL.yaml b/.github/workflows/PMM_PXC_PROXYSQL.yaml → .github/workflows/PMM_PROXYSQL.yaml
@@ -1,4 +1,4 @@
-name: PMM_PXC_PROXYSQL
+name: PMM_PROXYSQL
 on:
   workflow_dispatch:
     inputs:

diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml
@@ -7,8 +7,8 @@ on:
         default: 'https://github.com/Percona-QA/package-testing/blob/master/VERSIONS'
         required: true
       ps_glibc:
-        description: "ps glibc version, example: 2.28"
-        default: '2.28'
+        description: "ps glibc version, example: 2.35"
+        default: '2.35'
         required: false
       pmm_qa_branch:
         description: "Branch for qa-integration to checkout"
@@ -29,11 +29,11 @@ on:
         type: string
   push:
     branches:
-      - main
+      - v3
 
   pull_request:
     branches:
-      - main
+      - v3
 
 jobs:
   PMM_PS_TEST:
@@ -42,7 +42,7 @@ jobs:
     env:
       ADMIN_PASSWORD: 'admin'
       PS_VERSION: ${{ github.event.inputs.ps_version || '8.0' }}
-      PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.28' }}
+      PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.35' }}
       PMM_SERVER_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }}
       PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }}
       PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }}

diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml
@@ -20,19 +20,9 @@ on:
         default: "perconalab/pmm-server:3-dev-latest"
         required: false
 
-  push:
-    branches:
-      - main
-      - v3
-
-  pull_request:
-    branches:
-      - main
-      - v3
-
 jobs:
   test_replica_set:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 20
     env:
       PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }}
@@ -47,7 +37,7 @@ jobs:
       working-directory: ./pmm_psmdb-pbm_setup
 
   test_sharded_cluster:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 20
     env:
       PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }}
@@ -62,7 +52,7 @@ jobs:
       working-directory: ./pmm_psmdb-pbm_setup
 
   test_diff_auth:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 20
     env:
       PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }}

diff --git a/.github/workflows/PMM_PSMDB_PBM_FULL.yml b/.github/workflows/PMM_PSMDB_PBM_FULL.yml
@@ -0,0 +1,75 @@
+name: PMM_PSMDB_PBM_FULL
+
+on:
+  workflow_dispatch:
+    inputs:
+      pmm_version:
+        description: "pmm2-client version"
+        default: "3-dev-latest"
+        required: false
+      pmm_image:
+        description: "pmm-server docker image"
+        default: "perconalab/pmm-server:3-dev-latest"
+        required: false
+
+  push:
+    branches:
+      - main
+      - v3
+
+  pull_request:
+    branches:
+      - main
+      - v3
+
+jobs:
+  test_replica_set:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        psmdb: ["6.0", "7.0", "8.0"]
+    env:
+      PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }}
+      PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Test RS with PSMDB ${{ matrix.psmdb }}
+      run: |
+        PSMDB_VERSION=${{ matrix.psmdb }} ./start-rs.sh
+      working-directory: ./pmm_psmdb-pbm_setup
+
+  test_sharded_cluster:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        psmdb: ["6.0", "7.0", "8.0"]
+    env:
+      PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }}
+      PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Test sharded with PSMDB ${{ matrix.psmdb }}
+      run: |
+        PSMDB_VERSION=${{ matrix.psmdb }} ./start-sharded.sh
+      working-directory: ./pmm_psmdb-pbm_setup
+
+  test_diff_auth:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        psmdb: ["6.0", "7.0", "8.0"]
+    env:
+      PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }}
+      PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test auth with PSMDB ${{ matrix.psmdb }}
+      run: |
+        PSMDB_VERSION=${{ matrix.psmdb }} ./test-auth.sh
+      working-directory: ./pmm_psmdb_diffauth_setup
diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile
@@ -22,7 +22,7 @@ RUN set -ex; \
     export GNUPGHOME="$(mktemp -d)"; \
     export PERCONA_TELEMETRY_URL="https://check-dev.percona.com/v1/telemetry/GenericReport"; \
     yum install -y gnupg2; \
-    if [ $PSMDB_VERSION != "latest" ]; then export PSMDB_REPO="$(echo "${PSMDB_VERSION}" | awk -F '.' '{print "psmdb-"$1$2}')" ; else export PSMDB_REPO=psmdb-60 ; fi ; \
+    if [ $PSMDB_VERSION != "latest" ]; then export PSMDB_REPO="$(echo "${PSMDB_VERSION}" | awk -F '.' '{print "psmdb-"$1$2}')" ; else export PSMDB_REPO=psmdb-80 ; fi ; \
     gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 99DB70FAE1D7CE227FB6488205B555B38483C65D 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1; \
     gpg --batch --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/PERCONA-PACKAGING-KEY; \
     gpg --batch --export --armor 99DB70FAE1D7CE227FB6488205B555B38483C65D > ${GNUPGHOME}/RPM-GPG-KEY-centosofficial; \
@@ -38,7 +38,7 @@ RUN set -ex; \
     percona-release enable pmm3-client ${PMM_REPO} && \
     yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
     yum update --refresh -y && \
-    if [ $PSMDB_VERSION == "latest" ]; then \
+    if [[ $PSMDB_VERSION == "latest" || "$PSMDB_VERSION" =~ ^[0-9]\.[0-9]$ ]] ; then \
         yum -y install percona-server-mongodb \
                        percona-server-mongodb-tools \
                        percona-server-mongodb-server \

diff --git a/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf
@@ -9,10 +9,6 @@ storage:
 systemLog:
   destination: syslog
 
-processManagement:
-#  fork: true
-  pidFilePath: /var/run/mongod.pid
-
 net:
   port: 27017
   bindIp: 0.0.0.0

diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf
@@ -9,10 +9,6 @@ storage:
 systemLog:
   destination: syslog
 
-processManagement:
-#  fork: true
-  pidFilePath: /var/run/mongod.pid
-
 net:
   port: 27017
   bindIp: 0.0.0.0

diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf
@@ -9,10 +9,6 @@ storage:
 systemLog:
   destination: syslog
 
-processManagement:
-#  fork: true
-  pidFilePath: /var/run/mongod.pid
-
 net:
   port: 27017
   bindIp: 0.0.0.0

diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf
@@ -9,10 +9,6 @@ storage:
 systemLog:
   destination: syslog
 
-processManagement:
-#  fork: true
-  pidFilePath: /var/run/mongod.pid
-
 net:
   port: 27017
   bindIp: 0.0.0.0

diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh
@@ -13,14 +13,14 @@ nodes="rs101 rs102 rs103"
 for node in $nodes
 do
     echo "congiguring pbm agent on $node"
-    docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent"
+    docker compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent"
     echo "restarting pbm agent on $node"
-    docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent
+    docker compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent
 done
 
 if [[ $mongo_setup_type == "psa" ]]; then
   echo "stop pbm agent for arbiter node"
-  docker-compose -f docker-compose-rs.yaml exec -T rs103 systemctl stop pbm-agent
+  docker compose -f docker-compose-rs.yaml exec -T rs103 systemctl stop pbm-agent
 fi
 echo
 echo "configuring pmm agents"
@@ -29,17 +29,17 @@ nodes="rs101 rs102 rs103"
 for node in $nodes
 do
     echo "congiguring pmm agent on $node"
-    docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup
+    docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup
     if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then
-      docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017
+      docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017
     else
-      docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017
+      docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017
     fi
 done
 echo
 echo "adding some data"
-docker-compose -f docker-compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs
-docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs
+docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF
 use students;
 db.students.insertMany([
   {

diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh
@@ -13,14 +13,14 @@ nodes="rs201 rs202 rs203"
 for node in $nodes
 do
     echo "configuring pbm agent on $node"
-    docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent"
+    docker compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent"
     echo "restarting pbm agent on $node"
-    docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent
+    docker compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent
 done
 
 if [[ $mongo_setup_type == "psa" ]]; then
   echo "stop pbm agent for arbiter node rs203"
-  docker-compose -f docker-compose-rs.yaml exec -T rs203 systemctl stop pbm-agent
+  docker compose -f docker-compose-rs.yaml exec -T rs203 systemctl stop pbm-agent
 fi
 echo
 echo "configuring pmm agents"
@@ -29,10 +29,10 @@ nodes="rs201 rs202 rs203"
 for node in $nodes
 do
     echo "configuring pmm agent on $node"
-    docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup
+    docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup
     if [[ $mongo_setup_type == "psa" && $node == "rs203"  ]]; then
-      docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017
+      docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017
     else
-      docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017
+      docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017
     fi
 done
diff --git a/pmm_psmdb-pbm_setup/configure-extra-psa.sh b/pmm_psmdb-pbm_setup/configure-extra-psa.sh
@@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass}
 
 echo
 echo "configuring extra PSA replicaset with members priorities"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF
     config = {
         "_id" : "rs",
         "members" : [
@@ -35,12 +35,12 @@ echo
 sleep 60
 echo
 echo "configuring root user on primary"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF
 db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] });
 EOF
 echo
 echo "configuring pbm and pmm roles"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
 db.getSiblingDB("admin").createRole({
     "role": "pbmAnyAction",
     "privileges": [{
@@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({
 EOF
 echo
 echo "creating pbm user"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
 db.getSiblingDB("admin").createUser({
     user: "${pbm_user}",
     pwd: "${pbm_pass}",
@@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({
 EOF
 echo
 echo "creating pmm user"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
 db.getSiblingDB("admin").createUser({
     user: "${pmm_mongo_user}",
     pwd: "${pmm_mongo_user_pass}",

diff --git a/pmm_psmdb-pbm_setup/configure-extra-replset.sh b/pmm_psmdb-pbm_setup/configure-extra-replset.sh
@@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass}
 
 echo
 echo "configuring replicaset with members priorities"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF
     config = {
         "_id" : "rs",
         "members" : [
@@ -35,12 +35,12 @@ echo
 sleep 60
 echo
 echo "configuring root user on primary"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF
 db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] });
 EOF
 echo
 echo "configuring pbm and pmm roles"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
 db.getSiblingDB("admin").createRole({
     "role": "pbmAnyAction",
     "privileges": [{
@@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({
 EOF
 echo
 echo "creating pbm user"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
 db.getSiblingDB("admin").createUser({
     user: "${pbm_user}",
     pwd: "${pbm_pass}",
@@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({
 EOF
 echo
 echo "creating pmm user"
-docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
+docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF
 db.getSiblingDB("admin").createUser({
     user: "${pmm_mongo_user}",
     pwd: "${pmm_mongo_user_pass}",