From 6bde793bc114153084be1606e437c7bd41021c0d Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:29:18 +0530 Subject: [PATCH 01/17] Updates v3 changes for PMM-13331 from pmm2 (#79) --- .github/workflows/PMM_PSMDB_PBM.yml | 6 +-- pmm_psmdb-pbm_setup/configure-agents.sh | 16 +++--- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 12 ++--- pmm_psmdb-pbm_setup/configure-extra-psa.sh | 10 ++-- .../configure-extra-replset.sh | 10 ++-- pmm_psmdb-pbm_setup/configure-psa.sh | 10 ++-- pmm_psmdb-pbm_setup/configure-replset.sh | 10 ++-- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 6 +++ .../docker-compose-sharded.yaml | 10 ++++ pmm_psmdb-pbm_setup/start-rs-only.sh | 6 +-- pmm_psmdb-pbm_setup/start-rs.sh | 16 +++--- pmm_psmdb-pbm_setup/start-sharded.sh | 52 +++++++++---------- .../docker-compose-pmm-psmdb.yml | 1 + pmm_psmdb_diffauth_setup/test-auth.sh | 22 ++++---- 14 files changed, 102 insertions(+), 85 deletions(-) diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml index 94c0305..9dd6d25 100644 --- a/.github/workflows/PMM_PSMDB_PBM.yml +++ b/.github/workflows/PMM_PSMDB_PBM.yml @@ -32,7 +32,7 @@ on: jobs: test_replica_set: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 20 env: PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} @@ -47,7 +47,7 @@ jobs: working-directory: ./pmm_psmdb-pbm_setup test_sharded_cluster: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 20 env: PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} @@ -62,7 +62,7 @@ jobs: working-directory: ./pmm_psmdb-pbm_setup test_diff_auth: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 20 env: PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 97e2a56..2498b03 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -13,14 +13,14 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pbm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker compose-rs.yaml exec -T $node systemctl restart pbm-agent done if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node" - docker-compose -f docker-compose-rs.yaml exec -T rs103 systemctl stop pbm-agent + docker compose -f docker compose-rs.yaml exec -T rs103 systemctl stop pbm-agent fi echo echo "configuring pmm agents" @@ -29,17 +29,17 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup + docker compose -f docker compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo echo "adding some data" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF use students; db.students.insertMany([ { diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 1839ceb..10649ea 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -13,14 +13,14 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pbm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker compose-rs.yaml exec -T $node systemctl restart pbm-agent done if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node rs203" - docker-compose -f docker-compose-rs.yaml exec -T rs203 systemctl stop pbm-agent + docker compose -f docker compose-rs.yaml exec -T rs203 systemctl stop pbm-agent fi echo echo "configuring pmm agents" @@ -29,10 +29,10 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup + docker compose -f docker compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done diff --git a/pmm_psmdb-pbm_setup/configure-extra-psa.sh b/pmm_psmdb-pbm_setup/configure-extra-psa.sh index d667f6b..721029f 100755 --- a/pmm_psmdb-pbm_setup/configure-extra-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-psa.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring extra PSA replicaset with members priorities" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-extra-replset.sh b/pmm_psmdb-pbm_setup/configure-extra-replset.sh index cff511d..1f7e9c3 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-replset.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-replset.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring replicaset with members priorities" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-psa.sh b/pmm_psmdb-pbm_setup/configure-psa.sh index 879adb1..ce006e2 100755 --- a/pmm_psmdb-pbm_setup/configure-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-psa.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring PSA replicaset with members priorities" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-replset.sh b/pmm_psmdb-pbm_setup/configure-replset.sh index f50bc39..819ceeb 100644 --- a/pmm_psmdb-pbm_setup/configure-replset.sh +++ b/pmm_psmdb-pbm_setup/configure-replset.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring replicaset with members priorities" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 4b4451d..9da2b1a 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -35,6 +35,7 @@ services: - /tmp/backup_data:/tmp/backup_data:rw - /tmp/mongodb:/tmp privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -61,6 +62,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -87,6 +89,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -115,6 +118,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -141,6 +145,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -167,6 +172,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 2949bad..b223ad9 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -23,6 +23,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/mongodb:/tmp privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -45,6 +46,7 @@ services: - ./conf/mongod-rs1:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -65,6 +67,7 @@ services: - ./conf/mongod-rs1:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -85,6 +88,7 @@ services: - ./conf/mongod-rs2:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -105,6 +109,7 @@ services: - ./conf/mongod-rs2:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -125,6 +130,7 @@ services: - ./conf/mongod-rs2:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -145,6 +151,7 @@ services: - ./conf/mongod-cfg:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -165,6 +172,7 @@ services: - ./conf/mongod-cfg:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -185,6 +193,7 @@ services: - ./conf/mongod-cfg:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -206,6 +215,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - ./conf/datagen:/etc/datagen:ro privileged: true + cgroup: host command: "mongos --keyFile=/etc/keyfile --configdb rscfg/rscfg01:27017,rscfg02:27017,rscfg03:27017 --port 27017 --bind_ip 0.0.0.0" networks: - test-network diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index df21e30..23f2442 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -13,9 +13,9 @@ docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} export MONGO_SETUP_TYPE=${mongo_setup_type} -docker-compose -f docker-compose-rs.yaml down -v --remove-orphans -docker-compose -f docker-compose-rs.yaml build --no-cache -docker-compose -f docker-compose-rs.yaml up -d +docker compose -f docker compose-rs.yaml down -v --remove-orphans +docker compose -f docker compose-rs.yaml build --no-cache +docker compose -f docker compose-rs.yaml up -d echo echo "waiting 60 seconds for replica set members to start" sleep 60 diff --git a/pmm_psmdb-pbm_setup/start-rs.sh b/pmm_psmdb-pbm_setup/start-rs.sh index cddf62f..c6d3fb4 100755 --- a/pmm_psmdb-pbm_setup/start-rs.sh +++ b/pmm_psmdb-pbm_setup/start-rs.sh @@ -14,16 +14,16 @@ docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} export MONGO_SETUP_TYPE=${mongo_setup_type} -docker-compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans -docker-compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml build -docker-compose -f docker-compose-pmm.yaml -f docker-compose-rs.yaml up -d +docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml down -v --remove-orphans +docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml build +docker compose -f docker compose-pmm.yaml -f docker compose-rs.yaml up -d echo echo "waiting 30 seconds for pmm-server to start" sleep 30 echo "configuring pmm-server" -docker-compose -f docker-compose-pmm.yaml exec -T pmm-server change-admin-password $pmm_server_admin_pass +docker compose -f docker compose-pmm.yaml exec -T pmm-server change-admin-password $pmm_server_admin_pass echo "restarting pmm-server" -docker-compose -f docker-compose-pmm.yaml restart pmm-server +docker compose -f docker compose-pmm.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 if [ $mongo_setup_type == "pss" ]; then @@ -36,8 +36,8 @@ tests=${TESTS:-yes} if [ $tests != "no" ]; then echo echo "running tests" - docker-compose -f docker-compose-pmm.yaml run test pytest -s -x --verbose test.py - docker-compose -f docker-compose-pmm.yaml run test chmod -R 777 . + docker compose -f docker compose-pmm.yaml run test pytest -s -x --verbose test.py + docker compose -f docker compose-pmm.yaml run test chmod -R 777 . else echo echo "skipping tests" @@ -46,7 +46,7 @@ cleanup=${CLEANUP:-yes} if [ $cleanup != "no" ]; then echo echo "cleanup" - docker-compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans + docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml down -v --remove-orphans else echo echo "skipping cleanup" diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 91673d3..dd4cbcb 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -5,16 +5,16 @@ pmm_pass=${PMM_PASS:-pmmpass} pbm_user=${PBM_USER:-pbm} pbm_pass=${PBM_PASS:-pbmpass} -docker-compose -f docker-compose-sharded.yaml down -v --remove-orphans -docker-compose -f docker-compose-sharded.yaml build -docker-compose -f docker-compose-sharded.yaml up -d +docker compose -f docker compose-sharded.yaml down -v --remove-orphans +docker compose -f docker compose-sharded.yaml build +docker compose -f docker compose-sharded.yaml up -d echo "waiting 30 seconds for pmm-server to start" sleep 30 echo "configuring pmm-server" -docker-compose -f docker-compose-sharded.yaml exec -T pmm-server change-admin-password password +docker compose -f docker compose-sharded.yaml exec -T pmm-server change-admin-password password echo "restarting pmm-server" -docker-compose -f docker-compose-sharded.yaml restart pmm-server +docker compose -f docker compose-sharded.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 @@ -23,7 +23,7 @@ for node in $nodes do rs=$(echo $node | awk -F "0" '{print $1}') echo "configuring replicaset ${rs} with members priorities" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo --quiet << EOF config = { "_id" : "${rs}", "members" : [ @@ -49,12 +49,12 @@ EOF sleep 60 echo echo "configuring root user on primary $node replicaset $rs" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles on replicaset $rs" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -84,7 +84,7 @@ EOF EOF echo echo "creating pbm user for replicaset ${rs}" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -99,7 +99,7 @@ EOF EOF echo echo "creating pmm user for replicaset ${rs}" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_user}", pwd: "${pmm_pass}", @@ -118,7 +118,7 @@ EOF done echo "configuring configserver replicaset with members priorities" -docker-compose -f docker-compose-sharded.yaml exec -T rscfg01 mongo --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T rscfg01 mongo --quiet << EOF config = { "_id" : "rscfg", "members" : [ @@ -144,18 +144,18 @@ EOF sleep 60 echo echo "adding shards and creating global mongo user" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T mongos mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs1/rs101:27017,rs102:27017,rs103:27017" )' +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs1/rs101:27017,rs102:27017,rs103:27017" )' echo sleep 20 -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs2/rs201:27017,rs202:27017,rs203:27017" )' +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs2/rs201:27017,rs202:27017,rs203:27017" )' echo sleep 20 echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -185,7 +185,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -200,7 +200,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_user}", pwd: "${pmm_pass}", @@ -224,9 +224,9 @@ nodes="rs101 rs102 rs103 rs201 rs202 rs203 rscfg01 rscfg02 rscfg03" for node in $nodes do echo "congiguring pbm agent on $node" - docker-compose -f docker-compose-sharded.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker compose-sharded.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker-compose -f docker-compose-sharded.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker compose-sharded.yaml exec -T $node systemctl restart pbm-agent done echo echo "configuring pmm agents" @@ -236,26 +236,26 @@ for node in $nodes do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') - docker-compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup - docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup + docker compose -f docker compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker-compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 +docker compose -f docker compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 +docker compose -f docker compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 tests=${TESTS:-yes} if [ $tests != "no" ]; then echo "running tests" - docker-compose -f docker-compose-sharded.yaml run test pytest -s -x --verbose test.py - docker-compose -f docker-compose-sharded.yaml run test chmod -R 777 . + docker compose -f docker compose-sharded.yaml run test pytest -s -x --verbose test.py + docker compose -f docker compose-sharded.yaml run test chmod -R 777 . else echo "skipping tests" fi cleanup=${CLEANUP:-yes} if [ $cleanup != "no" ]; then echo "cleanup" - docker-compose -f docker-compose-sharded.yaml down -v --remove-orphans + docker compose -f docker compose-sharded.yaml down -v --remove-orphans else echo "skipping cleanup" fi diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index aad57ae..4622980 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -49,6 +49,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - 'psmdb-server-data:/data/db' privileged: true + cgroup: host ports: - "43000-43020:43000-43020" healthcheck: diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index a758ef9..58372ca 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -37,25 +37,25 @@ cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem find certs -type f -exec chmod 644 {} \; #Start setup -docker-compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans -docker-compose -f docker-compose-pmm-psmdb.yml build -docker-compose -f docker-compose-pmm-psmdb.yml up -d +docker compose -f docker compose-pmm-psmdb.yml down -v --remove-orphans +docker compose -f docker compose-pmm-psmdb.yml build +docker compose -f docker compose-pmm-psmdb.yml up -d #Add users -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js #Configure PBM -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent #Configure PMM set +e i=1 while [ $i -le 3 ]; do - output=$(docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) + output=$(docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) exit_code=$? if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then @@ -68,12 +68,12 @@ done #Add Mongo Service random_number=$RANDOM -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster tests=${TESTS:-yes} if [ $tests = "yes" ]; then echo "running tests" - docker-compose -f docker-compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py + docker compose -f docker compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py else echo "skipping tests" fi @@ -81,7 +81,7 @@ fi cleanup=${CLEANUP:-yes} if [ $cleanup = "yes" ]; then echo "cleanup" - docker-compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans + docker compose -f docker compose-pmm-psmdb.yml down -v --remove-orphans if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf fi From bb97e674f27bfe54817e169ec529113e5ba525b9 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:51:44 +0530 Subject: [PATCH 02/17] Updates fix docker-compose search replace (#80) --- pmm_psmdb-pbm_setup/configure-agents.sh | 16 +++--- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 12 ++--- pmm_psmdb-pbm_setup/configure-extra-psa.sh | 10 ++-- .../configure-extra-replset.sh | 10 ++-- pmm_psmdb-pbm_setup/configure-psa.sh | 10 ++-- pmm_psmdb-pbm_setup/configure-replset.sh | 10 ++-- pmm_psmdb-pbm_setup/start-rs-only.sh | 6 +-- pmm_psmdb-pbm_setup/start-rs.sh | 16 +++--- pmm_psmdb-pbm_setup/start-sharded.sh | 52 +++++++++---------- pmm_psmdb_diffauth_setup/test-auth.sh | 22 ++++---- 10 files changed, 82 insertions(+), 82 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 2498b03..107156f 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -13,14 +13,14 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pbm agent on $node" - docker compose -f docker compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker compose -f docker compose-rs.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent done if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node" - docker compose -f docker compose-rs.yaml exec -T rs103 systemctl stop pbm-agent + docker compose -f docker-compose-rs.yaml exec -T rs103 systemctl stop pbm-agent fi echo echo "configuring pmm agents" @@ -29,17 +29,17 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker compose -f docker compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup + docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo echo "adding some data" -docker compose -f docker compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF use students; db.students.insertMany([ { diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 10649ea..ce486d8 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -13,14 +13,14 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pbm agent on $node" - docker compose -f docker compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker compose -f docker compose-rs.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent done if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node rs203" - docker compose -f docker compose-rs.yaml exec -T rs203 systemctl stop pbm-agent + docker compose -f docker-compose-rs.yaml exec -T rs203 systemctl stop pbm-agent fi echo echo "configuring pmm agents" @@ -29,10 +29,10 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - docker compose -f docker compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup + docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then - docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done diff --git a/pmm_psmdb-pbm_setup/configure-extra-psa.sh b/pmm_psmdb-pbm_setup/configure-extra-psa.sh index 721029f..00bf29a 100755 --- a/pmm_psmdb-pbm_setup/configure-extra-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-psa.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring extra PSA replicaset with members priorities" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-extra-replset.sh b/pmm_psmdb-pbm_setup/configure-extra-replset.sh index 1f7e9c3..cade2cf 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-replset.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-replset.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring replicaset with members priorities" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-psa.sh b/pmm_psmdb-pbm_setup/configure-psa.sh index ce006e2..391b6df 100755 --- a/pmm_psmdb-pbm_setup/configure-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-psa.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring PSA replicaset with members priorities" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-replset.sh b/pmm_psmdb-pbm_setup/configure-replset.sh index 819ceeb..d89e571 100644 --- a/pmm_psmdb-pbm_setup/configure-replset.sh +++ b/pmm_psmdb-pbm_setup/configure-replset.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring replicaset with members priorities" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index 23f2442..d7f95df 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -13,9 +13,9 @@ docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} export MONGO_SETUP_TYPE=${mongo_setup_type} -docker compose -f docker compose-rs.yaml down -v --remove-orphans -docker compose -f docker compose-rs.yaml build --no-cache -docker compose -f docker compose-rs.yaml up -d +docker compose -f docker-compose-rs.yaml down -v --remove-orphans +docker compose -f docker-compose-rs.yaml build --no-cache +docker compose -f docker-compose-rs.yaml up -d echo echo "waiting 60 seconds for replica set members to start" sleep 60 diff --git a/pmm_psmdb-pbm_setup/start-rs.sh b/pmm_psmdb-pbm_setup/start-rs.sh index c6d3fb4..a4e651b 100755 --- a/pmm_psmdb-pbm_setup/start-rs.sh +++ b/pmm_psmdb-pbm_setup/start-rs.sh @@ -14,16 +14,16 @@ docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} export MONGO_SETUP_TYPE=${mongo_setup_type} -docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml down -v --remove-orphans -docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml build -docker compose -f docker compose-pmm.yaml -f docker compose-rs.yaml up -d +docker compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans +docker compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml build +docker compose -f docker-compose-pmm.yaml -f docker-compose-rs.yaml up -d echo echo "waiting 30 seconds for pmm-server to start" sleep 30 echo "configuring pmm-server" -docker compose -f docker compose-pmm.yaml exec -T pmm-server change-admin-password $pmm_server_admin_pass +docker compose -f docker-compose-pmm.yaml exec -T pmm-server change-admin-password $pmm_server_admin_pass echo "restarting pmm-server" -docker compose -f docker compose-pmm.yaml restart pmm-server +docker compose -f docker-compose-pmm.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 if [ $mongo_setup_type == "pss" ]; then @@ -36,8 +36,8 @@ tests=${TESTS:-yes} if [ $tests != "no" ]; then echo echo "running tests" - docker compose -f docker compose-pmm.yaml run test pytest -s -x --verbose test.py - docker compose -f docker compose-pmm.yaml run test chmod -R 777 . + docker compose -f docker-compose-pmm.yaml run test pytest -s -x --verbose test.py + docker compose -f docker-compose-pmm.yaml run test chmod -R 777 . else echo echo "skipping tests" @@ -46,7 +46,7 @@ cleanup=${CLEANUP:-yes} if [ $cleanup != "no" ]; then echo echo "cleanup" - docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml down -v --remove-orphans + docker compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans else echo echo "skipping cleanup" diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index dd4cbcb..4beac37 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -5,16 +5,16 @@ pmm_pass=${PMM_PASS:-pmmpass} pbm_user=${PBM_USER:-pbm} pbm_pass=${PBM_PASS:-pbmpass} -docker compose -f docker compose-sharded.yaml down -v --remove-orphans -docker compose -f docker compose-sharded.yaml build -docker compose -f docker compose-sharded.yaml up -d +docker compose -f docker-compose-sharded.yaml down -v --remove-orphans +docker compose -f docker-compose-sharded.yaml build +docker compose -f docker-compose-sharded.yaml up -d echo "waiting 30 seconds for pmm-server to start" sleep 30 echo "configuring pmm-server" -docker compose -f docker compose-sharded.yaml exec -T pmm-server change-admin-password password +docker compose -f docker-compose-sharded.yaml exec -T pmm-server change-admin-password password echo "restarting pmm-server" -docker compose -f docker compose-sharded.yaml restart pmm-server +docker compose -f docker-compose-sharded.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 @@ -23,7 +23,7 @@ for node in $nodes do rs=$(echo $node | awk -F "0" '{print $1}') echo "configuring replicaset ${rs} with members priorities" - docker compose -f docker compose-sharded.yaml exec -T $node mongo --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo --quiet << EOF config = { "_id" : "${rs}", "members" : [ @@ -49,12 +49,12 @@ EOF sleep 60 echo echo "configuring root user on primary $node replicaset $rs" - docker compose -f docker compose-sharded.yaml exec -T $node mongo --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles on replicaset $rs" - docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -84,7 +84,7 @@ EOF EOF echo echo "creating pbm user for replicaset ${rs}" - docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -99,7 +99,7 @@ EOF EOF echo echo "creating pmm user for replicaset ${rs}" - docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_user}", pwd: "${pmm_pass}", @@ -118,7 +118,7 @@ EOF done echo "configuring configserver replicaset with members priorities" -docker compose -f docker compose-sharded.yaml exec -T rscfg01 mongo --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T rscfg01 mongo --quiet << EOF config = { "_id" : "rscfg", "members" : [ @@ -144,18 +144,18 @@ EOF sleep 60 echo echo "adding shards and creating global mongo user" -docker compose -f docker compose-sharded.yaml exec -T mongos mongo --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs1/rs101:27017,rs102:27017,rs103:27017" )' +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs1/rs101:27017,rs102:27017,rs103:27017" )' echo sleep 20 -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs2/rs201:27017,rs202:27017,rs203:27017" )' +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs2/rs201:27017,rs202:27017,rs203:27017" )' echo sleep 20 echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -185,7 +185,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -200,7 +200,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_user}", pwd: "${pmm_pass}", @@ -224,9 +224,9 @@ nodes="rs101 rs102 rs103 rs201 rs202 rs203 rscfg01 rscfg02 rscfg03" for node in $nodes do echo "congiguring pbm agent on $node" - docker compose -f docker compose-sharded.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker-compose-sharded.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker compose -f docker compose-sharded.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker-compose-sharded.yaml exec -T $node systemctl restart pbm-agent done echo echo "configuring pmm agents" @@ -236,26 +236,26 @@ for node in $nodes do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') - docker compose -f docker compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup - docker compose -f docker compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup + docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker compose -f docker compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 +docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" -docker compose -f docker compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 +docker compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 tests=${TESTS:-yes} if [ $tests != "no" ]; then echo "running tests" - docker compose -f docker compose-sharded.yaml run test pytest -s -x --verbose test.py - docker compose -f docker compose-sharded.yaml run test chmod -R 777 . + docker compose -f docker-compose-sharded.yaml run test pytest -s -x --verbose test.py + docker compose -f docker-compose-sharded.yaml run test chmod -R 777 . else echo "skipping tests" fi cleanup=${CLEANUP:-yes} if [ $cleanup != "no" ]; then echo "cleanup" - docker compose -f docker compose-sharded.yaml down -v --remove-orphans + docker compose -f docker-compose-sharded.yaml down -v --remove-orphans else echo "skipping cleanup" fi diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 58372ca..41960ae 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -37,25 +37,25 @@ cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem find certs -type f -exec chmod 644 {} \; #Start setup -docker compose -f docker compose-pmm-psmdb.yml down -v --remove-orphans -docker compose -f docker compose-pmm-psmdb.yml build -docker compose -f docker compose-pmm-psmdb.yml up -d +docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans +docker compose -f docker-compose-pmm-psmdb.yml build +docker compose -f docker-compose-pmm-psmdb.yml up -d #Add users -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js #Configure PBM -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent #Configure PMM set +e i=1 while [ $i -le 3 ]; do - output=$(docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) + output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) exit_code=$? if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then @@ -68,12 +68,12 @@ done #Add Mongo Service random_number=$RANDOM -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster tests=${TESTS:-yes} if [ $tests = "yes" ]; then echo "running tests" - docker compose -f docker compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py + docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py else echo "skipping tests" fi @@ -81,7 +81,7 @@ fi cleanup=${CLEANUP:-yes} if [ $cleanup = "yes" ]; then echo "cleanup" - docker compose -f docker compose-pmm-psmdb.yml down -v --remove-orphans + docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf fi From e44d8fbdf89f0133324a9a4e63871057108bed89 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:21:18 +0530 Subject: [PATCH 03/17] Fix integration test issues (#83) fix cli integration tests --- pmm_qa/pxc_proxysql_setup.yml | 2 +- pmm_qa/setup_docker_client_images.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/pxc_proxysql_setup.yml b/pmm_qa/pxc_proxysql_setup.yml index 967c5d0..0e9b7f1 100644 --- a/pmm_qa/pxc_proxysql_setup.yml +++ b/pmm_qa/pxc_proxysql_setup.yml @@ -33,7 +33,7 @@ - name: Prepare Container for PXC and Proxysql shell: > docker run -d --name={{ pxc_container }} - -p 6032:6032 + -p 6033:6033 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pxc_container diff --git a/pmm_qa/setup_docker_client_images.sh b/pmm_qa/setup_docker_client_images.sh index f76d873..40cfb46 100755 --- a/pmm_qa/setup_docker_client_images.sh +++ b/pmm_qa/setup_docker_client_images.sh @@ -5,9 +5,9 @@ echo echo "Configuring Multiple Docker Images with PMM Server and Client" echo "Please wait...." docker network create docker-client-check || true -docker-compose -f docker-compose-clients.yaml down -v --remove-orphans -docker-compose -f docker-compose-clients.yaml build --no-cache -docker-compose -f docker-compose-clients.yaml up -d +docker compose -f docker-compose-clients.yaml down -v --remove-orphans +docker compose -f docker-compose-clients.yaml build --no-cache +docker compose -f docker-compose-clients.yaml up -d sleep 20 echo "Adding DB Clients to PMM Server" docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass --service-name=ps-8.0 --query-source=perfschema --host=ps-1 --port=3306 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true From 8cab71ade362ad75e6a1bab55066118281a41941 Mon Sep 17 00:00:00 2001 From: Puneet Kala Date: Tue, 5 Nov 2024 15:07:55 +0530 Subject: [PATCH 04/17] PMM-7 Fix PMM V3 mysql_tls_setup.yml (#84) * PMM-7 Fix PMM V3 mysql_tls_setup.yml * PMM-7 Fix Update postgresql_tls_setup.yml * PMM-7 mongodb_tls_setup.yml --- pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml | 4 ++-- pmm_qa/tls-ssl-setup/mysql_tls_setup.yml | 4 ++-- pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml b/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml index 6e8ad79..c2cb9f5 100644 --- a/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml +++ b/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml @@ -7,8 +7,8 @@ mongodb_version: "{{ lookup('vars', 'extra_mongodb_version', default=lookup('env','MONGODB_VERSION') | default('4.4', true) ) }}" mongodb_ssl_container: "{{ lookup('vars', 'extra_mongodb_ssl_container', default=lookup('env','MONGODB_SSL_CONTAINER') | default('mongodb_ssl', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" - admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('pmm3admin!', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" tasks: diff --git a/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml b/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml index 2f19f18..e1e07b2 100644 --- a/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml +++ b/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml @@ -11,8 +11,8 @@ mysql_version: "{{ lookup('vars', 'extra_mysql_version', default=lookup('env','MYSQL_VERSION') | default('8.0', true) ) }}" mysql_ssl_container: "{{ lookup('vars', 'extra_mysql_ssl_container', default=lookup('env','MYSQL_SSL_CONTAINER') | default('mysql_ssl', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" - admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('pmm3admin!', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" tasks: diff --git a/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml b/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml index c689f91..ff6bbc3 100644 --- a/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml +++ b/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml @@ -11,8 +11,8 @@ pgsql_version: "{{ lookup('vars', 'extra_pgsql_version', default=lookup('env','PGSQL_VERSION') | default('13', true) ) }}" pgsql_ssl_container: "{{ lookup('vars', 'extra_pgsql_ssl_container', default=lookup('env','PGSQL_SSL_CONTAINER') | default('pgsql_ssl', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" - admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('pmm3admin!', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" tasks: From 359159f1f3bbd323a8dbe82b8b37e5df2594d650 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:49:01 +0530 Subject: [PATCH 05/17] Update pmm-framework.py (#85) Fix envars override issue. This is a know issue in ansible runner. --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index a9ce849..2e3572c 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -86,7 +86,8 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook=playbook_path, inventory='127.0.0.1', cmdline='-l localhost, --connection=local', - envvars=env_vars + envvars=env_vars, + suppress_env_files=True ) print(f'{playbook_filename} playbook execution {r.status}') From 98b0f8e8bacc13bbcc3f2b70b029a75ff1a8acfd Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Sat, 9 Nov 2024 10:34:40 +0530 Subject: [PATCH 06/17] Pmm3 fix pdpgsql and psmdb versions (#86) * Fix versions for pdpgsql and psmdb --- pmm_qa/pdpgsql_pgsm_setup.yml | 4 ++-- pmm_qa/pg_stat_statements_setup.sh | 2 +- pmm_qa/pmm-framework.py | 18 +++++++++++------- pmm_qa/product_version_download_helper | 2 ++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 6fc4998..3046548 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -6,10 +6,10 @@ become: true become_method: sudo vars: - pdpgsql_version: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','PDPGSQL_VERSION') | default('14', true) ) }}" + pdpgsql_version: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','PDPGSQL_VERSION') | default('15', true) ) }}" pdpgsql_pgsm_container: "{{ lookup('vars', 'extra_pdpgsql_pgsm_container', default=lookup('env','PDPGSQL_PGSM_CONTAINER') | default('pdpgsql_pgsm', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('1.1.1', true) ) }}" diff --git a/pmm_qa/pg_stat_statements_setup.sh b/pmm_qa/pg_stat_statements_setup.sh index ed05a5f..550d813 100644 --- a/pmm_qa/pg_stat_statements_setup.sh +++ b/pmm_qa/pg_stat_statements_setup.sh @@ -13,7 +13,7 @@ done # If postgres server version is not provided then it will default to version 14. if [ -z "$pgsql_version" ] then - export pgsql_version=14 + export pgsql_version=15 fi # If distribution is not provided then it will default to percona distribution 'PPG' diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2e3572c..5673211 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -9,12 +9,12 @@ # Database configurations database_configs = { "PSMDB": { - "versions": ["4.4", "5.0", "6.0", "7.0", "latest"], + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", "TARBALL": ""} }, "SSL_PSMDB": { - "versions": ["4.4", "5.0", "6.0", "7.0", "latest"], + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", "TARBALL": ""} }, @@ -34,15 +34,15 @@ "TARBALL": ""} }, "PGSQL": { - "versions": ["11", "12", "13", "14", "15", "16"], + "versions": ["11", "12", "13", "14", "15", "16", "17"], "configurations": {"QUERY_SOURCE": "pgstatements", "CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, "PDPGSQL": { - "versions": ["11", "12", "13", "14", "15", "16"], + "versions": ["11", "12", "13", "14", "15", "16", "17"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, "SSL_PDPGSQL": { - "versions": ["11", "12", "13", "14", "15", "16"], + "versions": ["11", "12", "13", "14", "15", "16", "17"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, "PXC": { @@ -492,6 +492,10 @@ def mongo_sharding_setup(script_filename, args): def get_latest_psmdb_version(psmdb_version): if psmdb_version == "latest": return psmdb_version + # workaround till 8.0 is released. + elif psmdb_version in ("8.0", "8.0.1", "8.0.1-1"): + return "8.0.1-1" + # Define the data to be sent in the POST request data = { 'version': f'percona-server-mongodb-{psmdb_version}' @@ -640,7 +644,7 @@ def setup_ssl_psmdb(db_type, db_version=None, db_config=None, args=None): 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'COMPOSE_PROFILES': get_value('COMPOSE_PROFILES', db_type, args, db_config), 'MONGO_SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config), - 'TESTS': 'no', + '`TESTS`': 'no', 'CLEANUP': 'no' } @@ -752,7 +756,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): for db_type, options in database_configs.items(): db_parser = subparsers.add_parser(db_type.lower()) for config, value in options['configurations'].items(): - db_parser.add_argument(f'{config}',metavar='', help=f'{config} for {db_type} (default: {value})') + db_parser.add_argument(f'{config}', metavar='', help=f'{config} for {db_type} (default: {value})') # Add arguments parser.add_argument("--database", action='append', nargs=1, diff --git a/pmm_qa/product_version_download_helper b/pmm_qa/product_version_download_helper index d41657f..cb85dff 100644 --- a/pmm_qa/product_version_download_helper +++ b/pmm_qa/product_version_download_helper @@ -16,6 +16,8 @@ psmdb # 6.0.2 # https://downloads.percona.com/downloads/percona-distribution-mon psmdb # 7.0.2 # https://downloads.percona.com/downloads/percona-server-mongodb-7.0/percona-server-mongodb-7.0.2-1/binary/tarball/percona-server-mongodb-7.0.2-1-x86_64.glibc2.17.tar.gz +psmdb # 8.0.1 # https://downloads.percona.com/downloads/TESTING/psmdb-8.0.1/percona-server-mongodb-8.0.1-1-x86_64.jammy-minimal.tar.gz + ps # 8.0.33 # https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.33-25/binary/tarball/Percona-Server-8.0.33-25-Linux.x86_64.glibc2.17-minimal.tar.gz ps # 5.7.43 # https://downloads.percona.com/downloads/Percona-Server-5.7/Percona-Server-5.7.43-47/binary/tarball/Percona-Server-5.7.43-47-Linux.x86_64.glibc2.17-minimal.tar.gz From e1c40adda0f2d2ad2f50b9ce835c87d47cb1bf38 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Mon, 11 Nov 2024 10:46:48 +0530 Subject: [PATCH 07/17] Update product_version_download_helper (#87) Removed not required psmdb 8.0 binary from file --- pmm_qa/product_version_download_helper | 2 -- 1 file changed, 2 deletions(-) diff --git a/pmm_qa/product_version_download_helper b/pmm_qa/product_version_download_helper index cb85dff..d41657f 100644 --- a/pmm_qa/product_version_download_helper +++ b/pmm_qa/product_version_download_helper @@ -16,8 +16,6 @@ psmdb # 6.0.2 # https://downloads.percona.com/downloads/percona-distribution-mon psmdb # 7.0.2 # https://downloads.percona.com/downloads/percona-server-mongodb-7.0/percona-server-mongodb-7.0.2-1/binary/tarball/percona-server-mongodb-7.0.2-1-x86_64.glibc2.17.tar.gz -psmdb # 8.0.1 # https://downloads.percona.com/downloads/TESTING/psmdb-8.0.1/percona-server-mongodb-8.0.1-1-x86_64.jammy-minimal.tar.gz - ps # 8.0.33 # https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.33-25/binary/tarball/Percona-Server-8.0.33-25-Linux.x86_64.glibc2.17-minimal.tar.gz ps # 5.7.43 # https://downloads.percona.com/downloads/Percona-Server-5.7/Percona-Server-5.7.43-47/binary/tarball/Percona-Server-5.7.43-47-Linux.x86_64.glibc2.17-minimal.tar.gz From c8d056a5c520aff3e14196ce9efb24364b2d668d Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:26:10 +0530 Subject: [PATCH 08/17] PMM3 fix ssl qan (#88) * Updates for Mongo SSL QAN --- pmm_psmdb_diffauth_setup/conf/mongod.conf | 4 ++-- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 1 + pmm_psmdb_diffauth_setup/test-auth.sh | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/conf/mongod.conf b/pmm_psmdb_diffauth_setup/conf/mongod.conf index 4800c3f..0ddce7b 100644 --- a/pmm_psmdb_diffauth_setup/conf/mongod.conf +++ b/pmm_psmdb_diffauth_setup/conf/mongod.conf @@ -40,6 +40,6 @@ replication: replSetName: rs0 operationProfiling: - slowOpThresholdMs: 200 mode: all - rateLimit: 100 + slowOpThresholdMs: 1 + diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 4622980..38f0e5f 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -48,6 +48,7 @@ services: - ./certs:/mongodb_certs - /sys/fs/cgroup:/sys/fs/cgroup:rw - 'psmdb-server-data:/data/db' + - ../pmm_psmdb-pbm_setup/conf/datagen:/etc/datagen:ro privileged: true cgroup: host ports: diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 41960ae..915a101 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -69,6 +69,8 @@ done #Add Mongo Service random_number=$RANDOM docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +#Add some data +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem tests=${TESTS:-yes} if [ $tests = "yes" ]; then From 47bee28b27927ed512693c7f5898efe2313af21a Mon Sep 17 00:00:00 2001 From: Sandra Date: Tue, 16 Jul 2024 22:00:47 +0300 Subject: [PATCH 09/17] PSMDB. Ensure PSMDB related metrics are exported, adjust PSMDB tests for PMM3 and run tests for all major versions --- .github/workflows/PMM_PSMDB_PBM.yml | 10 --- .github/workflows/PMM_PSMDB_PBM_FULL.yml | 75 +++++++++++++++++++ pmm_psmdb-pbm_setup/Dockerfile | 4 +- pmm_psmdb-pbm_setup/test/expected_metrics.txt | 38 ++++++++++ pmm_psmdb-pbm_setup/test/test.py | 48 +++++++++--- pmm_psmdb_diffauth_setup/init/setup_psmdb.js | 2 +- pmm_psmdb_diffauth_setup/test-auth.sh | 7 +- .../test/expected_metrics.txt | 38 ++++++++++ pmm_psmdb_diffauth_setup/test/test.py | 24 +++--- 9 files changed, 212 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/PMM_PSMDB_PBM_FULL.yml create mode 100644 pmm_psmdb-pbm_setup/test/expected_metrics.txt create mode 100644 pmm_psmdb_diffauth_setup/test/expected_metrics.txt diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml index 9dd6d25..68d1d70 100644 --- a/.github/workflows/PMM_PSMDB_PBM.yml +++ b/.github/workflows/PMM_PSMDB_PBM.yml @@ -20,16 +20,6 @@ on: default: "perconalab/pmm-server:3-dev-latest" required: false - push: - branches: - - main - - v3 - - pull_request: - branches: - - main - - v3 - jobs: test_replica_set: runs-on: ubuntu-22.04 diff --git a/.github/workflows/PMM_PSMDB_PBM_FULL.yml b/.github/workflows/PMM_PSMDB_PBM_FULL.yml new file mode 100644 index 0000000..ef13949 --- /dev/null +++ b/.github/workflows/PMM_PSMDB_PBM_FULL.yml @@ -0,0 +1,75 @@ +name: PMM_PSMDB_PBM_FULL + +on: + workflow_dispatch: + inputs: + pmm_version: + description: "pmm2-client version" + default: "3-dev-latest" + required: false + pmm_image: + description: "pmm-server docker image" + default: "perconalab/pmm-server:3-dev-latest" + required: false + + push: + branches: + - main + - v3 + + pull_request: + branches: + - main + - v3 + +jobs: + test_replica_set: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + psmdb: ["6.0", "7.0", "8.0"] + env: + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} + steps: + - uses: actions/checkout@v2 + - name: Test RS with PSMDB ${{ matrix.psmdb }} + run: | + PSMDB_VERSION=${{ matrix.psmdb }} ./start-rs.sh + working-directory: ./pmm_psmdb-pbm_setup + + test_sharded_cluster: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + psmdb: ["6.0", "7.0", "8.0"] + env: + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} + steps: + - uses: actions/checkout@v2 + - name: Test sharded with PSMDB ${{ matrix.psmdb }} + run: | + PSMDB_VERSION=${{ matrix.psmdb }} ./start-sharded.sh + working-directory: ./pmm_psmdb-pbm_setup + + test_diff_auth: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + psmdb: ["6.0", "7.0", "8.0"] + env: + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} + steps: + - uses: actions/checkout@v3 + - name: Test auth with PSMDB ${{ matrix.psmdb }} + run: | + PSMDB_VERSION=${{ matrix.psmdb }} ./test-auth.sh + working-directory: ./pmm_psmdb_diffauth_setup diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index 0e54e18..dc28ff6 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -22,7 +22,7 @@ RUN set -ex; \ export GNUPGHOME="$(mktemp -d)"; \ export PERCONA_TELEMETRY_URL="https://check-dev.percona.com/v1/telemetry/GenericReport"; \ yum install -y gnupg2; \ - if [ $PSMDB_VERSION != "latest" ]; then export PSMDB_REPO="$(echo "${PSMDB_VERSION}" | awk -F '.' '{print "psmdb-"$1$2}')" ; else export PSMDB_REPO=psmdb-60 ; fi ; \ + if [ $PSMDB_VERSION != "latest" ]; then export PSMDB_REPO="$(echo "${PSMDB_VERSION}" | awk -F '.' '{print "psmdb-"$1$2}')" ; else export PSMDB_REPO=psmdb-80 ; fi ; \ gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 99DB70FAE1D7CE227FB6488205B555B38483C65D 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1; \ gpg --batch --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/PERCONA-PACKAGING-KEY; \ gpg --batch --export --armor 99DB70FAE1D7CE227FB6488205B555B38483C65D > ${GNUPGHOME}/RPM-GPG-KEY-centosofficial; \ @@ -38,7 +38,7 @@ RUN set -ex; \ percona-release enable pmm3-client ${PMM_REPO} && \ yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ yum update --refresh -y && \ - if [ $PSMDB_VERSION == "latest" ]; then \ + if [[ $PSMDB_VERSION == "latest" || "$PSMDB_VERSION" =~ ^[0-9]\.[0-9]$ ]] ; then \ yum -y install percona-server-mongodb \ percona-server-mongodb-tools \ percona-server-mongodb-server \ diff --git a/pmm_psmdb-pbm_setup/test/expected_metrics.txt b/pmm_psmdb-pbm_setup/test/expected_metrics.txt new file mode 100644 index 0000000..4a65db2 --- /dev/null +++ b/pmm_psmdb-pbm_setup/test/expected_metrics.txt @@ -0,0 +1,38 @@ +mongodb_up +mongodb_mongod_global_lock_client +mongodb_mongod_global_lock_current_queue +mongodb_mongod_instance_uptime_seconds +mongodb_mongod_locks_time_acquiring_global_microseconds_total +mongodb_mongod_metrics_cursor_open +mongodb_mongod_metrics_cursor_timed_out_total +mongodb_mongod_metrics_document_total +mongodb_mongod_metrics_get_last_error_wtime_num_total +mongodb_mongod_metrics_get_last_error_wtime_total_milliseconds +mongodb_mongod_metrics_get_last_error_wtimeouts_total +mongodb_mongod_metrics_operation_total +mongodb_mongod_metrics_query_executor_total +mongodb_mongod_metrics_ttl_deleted_documents_total +mongodb_mongod_metrics_ttl_passes_total +mongodb_mongod_op_counters_repl_total +mongodb_mongod_op_latencies_latency_total +mongodb_mongod_op_latencies_ops_total +mongodb_mongod_storage_engine +mongodb_mongod_wiredtiger_blockmanager_bytes_total +mongodb_mongod_wiredtiger_cache_bytes +mongodb_mongod_wiredtiger_cache_bytes_total +mongodb_mongod_wiredtiger_cache_evicted_total +mongodb_mongod_wiredtiger_cache_max_bytes +mongodb_mongod_wiredtiger_cache_overhead_percent +mongodb_mongod_wiredtiger_cache_pages +mongodb_mongod_wiredtiger_cache_pages_total +mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets +mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets +mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets +mongodb_mongod_wiredtiger_log_bytes_total +mongodb_mongod_wiredtiger_log_operations_total +mongodb_mongod_wiredtiger_log_records_scanned_total +mongodb_mongod_wiredtiger_log_records_total +mongodb_mongod_wiredtiger_session_open_sessions_total +mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds +mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds_total +mongodb_mongod_wiredtiger_transactions_total diff --git a/pmm_psmdb-pbm_setup/test/test.py b/pmm_psmdb-pbm_setup/test/test.py index a748fa8..660a5de 100755 --- a/pmm_psmdb-pbm_setup/test/test.py +++ b/pmm_psmdb-pbm_setup/test/test.py @@ -23,13 +23,13 @@ def test_pmm_services(): - req = requests.post(f"https://{pmm_server_url}/v1/inventory/Services/List", json={}, + req = requests.get(f"https://{pmm_server_url}/v1/inventory/services", json={}, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nGetting all mongodb services:') mongodb = req.json()['mongodb'] print(mongodb) assert mongodb - assert "service_id" in mongodb[0]['service_id'] + assert "service_id" in mongodb[0] for service in mongodb: assert "rs" or "mongos" in service['service_name'] if not "mongos" in service['service_name']: @@ -49,11 +49,11 @@ def test_pmm_add_location(): 'bucket_name': 'bcp' } } - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Locations/Add", json=data, + req = requests.post(f"https://{pmm_server_url}/v1/backups/locations", json=data, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nAdding new location:') print(req.json()) - assert "location_id" in req.json()['location_id'] + assert "location_id" in req.json() pytest.location_id = req.json()['location_id'] @@ -64,13 +64,15 @@ def test_pmm_logical_backup(): 'name': 'test', 'description': 'test', 'retries': 0, - 'data_model': 'LOGICAL' + 'data_model': 'DATA_MODEL_LOGICAL' } - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Backups/Start", json=data, + + print(data) + req = requests.post(f"https://{pmm_server_url}/v1/backups:start", json=data, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nCreating logical backup:') print(req.json()) - assert "artifact_id" in req.json()['artifact_id'] + assert "artifact_id" in req.json() pytest.artifact_id = req.json()['artifact_id'] @@ -78,7 +80,7 @@ def test_pmm_artifact(): backup_complete = False for i in range(600): done = False - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Artifacts/List", json={}, + req = requests.get(f"https://{pmm_server_url}/v1/backups/artifacts", json={}, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['artifacts'] for artifact in req.json()['artifacts']: @@ -118,11 +120,11 @@ def test_pmm_start_restore(): 'service_id': pytest.service_id, 'artifact_id': pytest.artifact_id } - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Backups/Restore", json=data, + req = requests.post(f"https://{pmm_server_url}/v1/backups/restores:start", json=data, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nRestoring logical backup:') print(req.json()) - assert "restore_id" in req.json()['restore_id'] + assert "restore_id" in req.json() pytest.restore_id = req.json()['restore_id'] @@ -132,7 +134,7 @@ def test_pmm_restore(): restore_complete = False for i in range(600): done = False - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/RestoreHistory/List", json={}, + req = requests.get(f"https://{pmm_server_url}/v1/backups/restores", json={}, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['items'] for item in req.json()['items']: @@ -166,3 +168,27 @@ def test_pbm_restore(): restore_complete = True assert restore_complete + +def test_metrics(): + pmm_admin_list = json.loads(docker_rs101.check_output('pmm-admin list --json', timeout=30)) + for agent in pmm_admin_list['agent']: + if agent['agent_type'] == 'AGENT_TYPE_MONGODB_EXPORTER': + agent_id = agent['agent_id'] + agent_port = agent['port'] + break + try: + command = f"curl -s http://pmm:{agent_id}@127.0.0.1:{agent_port}/metrics" + metrics = docker_rs101.run(command, timeout=30) + assert metrics.exit_status == 0, f"Curl command failed with exit status {metrics.exit_status}" + except Exception as e: + pytest.fail(f"Fail to get metrics from exporter") + + try: + with open("expected_metrics.txt", "r") as f: + expected_metrics = {line.strip() for line in f if line.strip()} + except FileNotFoundError: + pytest.fail("Expected metrics file not found") + + for metric in expected_metrics: + if metric not in metrics.stdout: + pytest.fail(f"Metric '{metric}' is missing from the exporter output") diff --git a/pmm_psmdb_diffauth_setup/init/setup_psmdb.js b/pmm_psmdb_diffauth_setup/init/setup_psmdb.js index f1ac4c1..8f1c8fe 100644 --- a/pmm_psmdb_diffauth_setup/init/setup_psmdb.js +++ b/pmm_psmdb_diffauth_setup/init/setup_psmdb.js @@ -1,4 +1,4 @@ -var db = connect("mongodb://dba:secret@localhost:27017/admin"); +var db = connect("mongodb://root:root@localhost:27017/admin"); db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 915a101..e4f6b3d 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -75,7 +75,7 @@ docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen - tests=${TESTS:-yes} if [ $tests = "yes" ]; then echo "running tests" - docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py + output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) else echo "skipping tests" fi @@ -90,3 +90,8 @@ if [ $cleanup = "yes" ]; then else echo "skipping cleanup" fi + +echo "$output" +if echo "$output" | grep -q "\bFAILED\b"; then + exit 1 +fi diff --git a/pmm_psmdb_diffauth_setup/test/expected_metrics.txt b/pmm_psmdb_diffauth_setup/test/expected_metrics.txt new file mode 100644 index 0000000..4a65db2 --- /dev/null +++ b/pmm_psmdb_diffauth_setup/test/expected_metrics.txt @@ -0,0 +1,38 @@ +mongodb_up +mongodb_mongod_global_lock_client +mongodb_mongod_global_lock_current_queue +mongodb_mongod_instance_uptime_seconds +mongodb_mongod_locks_time_acquiring_global_microseconds_total +mongodb_mongod_metrics_cursor_open +mongodb_mongod_metrics_cursor_timed_out_total +mongodb_mongod_metrics_document_total +mongodb_mongod_metrics_get_last_error_wtime_num_total +mongodb_mongod_metrics_get_last_error_wtime_total_milliseconds +mongodb_mongod_metrics_get_last_error_wtimeouts_total +mongodb_mongod_metrics_operation_total +mongodb_mongod_metrics_query_executor_total +mongodb_mongod_metrics_ttl_deleted_documents_total +mongodb_mongod_metrics_ttl_passes_total +mongodb_mongod_op_counters_repl_total +mongodb_mongod_op_latencies_latency_total +mongodb_mongod_op_latencies_ops_total +mongodb_mongod_storage_engine +mongodb_mongod_wiredtiger_blockmanager_bytes_total +mongodb_mongod_wiredtiger_cache_bytes +mongodb_mongod_wiredtiger_cache_bytes_total +mongodb_mongod_wiredtiger_cache_evicted_total +mongodb_mongod_wiredtiger_cache_max_bytes +mongodb_mongod_wiredtiger_cache_overhead_percent +mongodb_mongod_wiredtiger_cache_pages +mongodb_mongod_wiredtiger_cache_pages_total +mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets +mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets +mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets +mongodb_mongod_wiredtiger_log_bytes_total +mongodb_mongod_wiredtiger_log_operations_total +mongodb_mongod_wiredtiger_log_records_scanned_total +mongodb_mongod_wiredtiger_log_records_total +mongodb_mongod_wiredtiger_session_open_sessions_total +mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds +mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds_total +mongodb_mongod_wiredtiger_transactions_total diff --git a/pmm_psmdb_diffauth_setup/test/test.py b/pmm_psmdb_diffauth_setup/test/test.py index 8ac14ba..0cf308b 100755 --- a/pmm_psmdb_diffauth_setup/test/test.py +++ b/pmm_psmdb_diffauth_setup/test/test.py @@ -25,20 +25,26 @@ def run_test(add_db_command): pmm_admin_list = json.loads(docker_pmm_client.check_output('pmm-admin list --json', timeout=30)) for agent in pmm_admin_list['agent']: - if agent['agent_type'] == 'MONGODB_EXPORTER': + if agent['agent_type'] == 'AGENT_TYPE_MONGODB_EXPORTER': agent_id = agent['agent_id'] agent_port = agent['port'] break + try: + command = f"curl -s http://pmm:{agent_id}@127.0.0.1:{agent_port}/metrics" + metrics = docker_pmm_client.run(command, timeout=30) + assert metrics.exit_status == 0, f"Curl command failed with exit status {metrics.exit_status}" + except Exception as e: + pytest.fail(f"Fail to get metrics from exporter") - url = f'http://localhost:{agent_port}/metrics' try: - response = docker_pmm_client.check_output(f"curl --request GET --url {url} --header 'Content-Type: " - f"application/json' --user 'pmm:{agent_id}'") - pattern = r'mongodb_up (\d+)' - result = re.search(pattern, response) - assert result is not None, "MongoDB related data isn't exported" - except AssertionError: - pytest.fail(f"Connection to {url} failed") + with open("expected_metrics.txt", "r") as f: + expected_metrics = {line.strip() for line in f if line.strip()} + except FileNotFoundError: + pytest.fail("Expected metrics file not found") + + for metric in expected_metrics: + if metric not in metrics.stdout: + pytest.fail(f"Metric '{metric}' is missing from the exporter output") def test_simple_auth_wo_tls(): From a9a4b83ab0fb41092ac0507c8b4bf3e694fca8eb Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 13 Nov 2024 15:27:54 +0530 Subject: [PATCH 10/17] Update pmm-framework.py (#90) fix editor automatic changes, to avoid running tests and create more services --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5673211..5315337 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -644,7 +644,7 @@ def setup_ssl_psmdb(db_type, db_version=None, db_config=None, args=None): 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'COMPOSE_PROFILES': get_value('COMPOSE_PROFILES', db_type, args, db_config), 'MONGO_SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config), - '`TESTS`': 'no', + 'TESTS': 'no', 'CLEANUP': 'no' } From 92a2b96abf7645b01e93a074fbd6e73eab8352f7 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:30:09 +0530 Subject: [PATCH 11/17] Update pmm-framework.py (#92) Fix issue, to pick changed password --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5315337..cf5ea16 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -596,7 +596,7 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run(['sed', '-i', f's/password/{admin_password}/g', f'{compose_file_path}']) + subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) From f0c601dee62eb8049990c3a2fd87890a7f134a32 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:26:39 +0530 Subject: [PATCH 12/17] Update to expose minio port number on 9001 (#93) Fix minio port number, as we start pmm-sever on 9000 already in pipelines --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 ++ pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 2 +- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 9da2b1a..6bdee0f 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -186,6 +186,8 @@ services: image: minio/minio profiles: ["classic", "extra"] container_name: minio + ports: + - "9001:9000" depends_on: - build_member networks: diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index b223ad9..0123ecb 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -229,7 +229,7 @@ services: image: minio/minio container_name: minio ports: - - "9000:9000" + - "9001:9000" networks: - test-network volumes: diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 38f0e5f..5ac5ff6 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -99,6 +99,8 @@ services: minio: image: minio/minio container_name: minio + ports: + - "9001:9000" volumes: - /tmp/minio/backups:/backups environment: From 70755950f67a3421c42297979143b93d5a19e93c Mon Sep 17 00:00:00 2001 From: Sandra Romanchenko <53295797+sandraromanchenko@users.noreply.github.com> Date: Fri, 29 Nov 2024 18:12:46 +0200 Subject: [PATCH 13/17] PSMDB. Remove unnecessary pidFilePath option as forking is already disabled (#97) --- pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf | 4 ---- pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf | 4 ---- pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf | 4 ---- pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf | 4 ---- pmm_psmdb_diffauth_setup/conf/mongod.conf | 4 ---- 5 files changed, 20 deletions(-) diff --git a/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf index d69f78d..20e2c62 100644 --- a/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf +++ b/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf index 2dc676f..7ecaf25 100644 --- a/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf +++ b/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf index b2fa66f..34ad841 100644 --- a/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf +++ b/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf index e0ee0ea..8d7946f 100644 --- a/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf +++ b/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 diff --git a/pmm_psmdb_diffauth_setup/conf/mongod.conf b/pmm_psmdb_diffauth_setup/conf/mongod.conf index 0ddce7b..d83b3d8 100644 --- a/pmm_psmdb_diffauth_setup/conf/mongod.conf +++ b/pmm_psmdb_diffauth_setup/conf/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 From c063f46d8a938a2f84cda4942f2e50c1a08cae82 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:52:49 +0530 Subject: [PATCH 14/17] Updates fix gh triggers for PS and Proxysql (#101) * Updates fix gh triggers for PS and Proxysql * Updates PDPGSQL version default to 17 --- .github/workflows/PMM_PDPGSQL.yaml | 6 +++--- .../workflows/{PMM_PXC_PROXYSQL.yaml => PMM_PROXYSQL.yaml} | 2 +- .github/workflows/PMM_PS.yaml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename .github/workflows/{PMM_PXC_PROXYSQL.yaml => PMM_PROXYSQL.yaml} (99%) diff --git a/.github/workflows/PMM_PDPGSQL.yaml b/.github/workflows/PMM_PDPGSQL.yaml index e457459..097f2e5 100644 --- a/.github/workflows/PMM_PDPGSQL.yaml +++ b/.github/workflows/PMM_PDPGSQL.yaml @@ -3,9 +3,9 @@ on: workflow_dispatch: inputs: pdpgsql_version: - description: "PDPGSQL Version available Image versions on github, example: 14, 15, 16" + description: "PDPGSQL Version available Image versions on github, example: 15, 16, 17" required: true - default: '16' + default: '17' type: string pmm_qa_branch: description: "Branch for qa-integration to checkout" @@ -39,7 +39,7 @@ jobs: timeout-minutes: 20 env: ADMIN_PASSWORD: 'admin' - PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || '16' }} + PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || '17' }} PMM_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }} PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }} PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }} diff --git a/.github/workflows/PMM_PXC_PROXYSQL.yaml b/.github/workflows/PMM_PROXYSQL.yaml similarity index 99% rename from .github/workflows/PMM_PXC_PROXYSQL.yaml rename to .github/workflows/PMM_PROXYSQL.yaml index b456521..2379c9b 100644 --- a/.github/workflows/PMM_PXC_PROXYSQL.yaml +++ b/.github/workflows/PMM_PROXYSQL.yaml @@ -1,4 +1,4 @@ -name: PMM_PXC_PROXYSQL +name: PMM_PROXYSQL on: workflow_dispatch: inputs: diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index c56fc3b..05408bc 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -7,8 +7,8 @@ on: default: 'https://github.com/Percona-QA/package-testing/blob/master/VERSIONS' required: true ps_glibc: - description: "ps glibc version, example: 2.28" - default: '2.28' + description: "ps glibc version, example: 2.35" + default: '2.35' required: false pmm_qa_branch: description: "Branch for qa-integration to checkout" @@ -42,7 +42,7 @@ jobs: env: ADMIN_PASSWORD: 'admin' PS_VERSION: ${{ github.event.inputs.ps_version || '8.0' }} - PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.28' }} + PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.35' }} PMM_SERVER_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }} PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }} PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }} From 19ea78648e1d2ce0e1e33f625e00078d029c5cb2 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:33:16 +0530 Subject: [PATCH 15/17] Update PMM_PS.yaml (#102) --- .github/workflows/PMM_PS.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index 05408bc..ae57e8e 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -29,11 +29,11 @@ on: type: string push: branches: - - main + - v3 pull_request: branches: - - main + - v3 jobs: PMM_PS_TEST: From 204660fc4bcb2f8bab2cfbef37c883230e887ccf Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:35:22 +0530 Subject: [PATCH 16/17] Updates to fix PGSQL for PGSM (#103) --- pmm_qa/pdpgsql_pgsm_setup.yml | 5 +++-- pmm_qa/pg_stat_monitor_setup.sh | 2 +- pmm_qa/pmm-framework.py | 9 ++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 3046548..fd3ef2d 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -12,9 +12,10 @@ client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" - pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('1.1.1', true) ) }}" + pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('2.1.0', true) ) }}" pgstat_monitor_repo: "{{ lookup('vars', 'extra_pgstat_monitor_repo', default=lookup('env','PGSTAT_MONITOR_REPO') | default('percona/pg_stat_monitor', true) ) }}" use_socket: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','USE_SOCKET') | default('', true) ) }}" + distribution: "{{ lookup('vars', 'extra_pdpgsql_distribution', default=lookup('env','DISTRIBUTION') | default('PPG', true) ) }}" tasks: - name: cleanup container for client and DB setup @@ -45,7 +46,7 @@ - name: Execute Setup script inside the pdpgsql pdpgsql_pgsm_container shell: "{{ item }}" with_items: - - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution PPG --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_pdpgsql_pgsm_{{ pdpgsql_version }}.log + - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution {{ distribution }} --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_pdpgsql_pgsm_{{ pdpgsql_version }}.log - name: Install pmm2-client on the pdpgsql_pgsm_container shell: "{{ item }}" diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index c3f5d6c..cc6fb54 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -19,7 +19,7 @@ fi # If branch/tag is not provided then it will default to main branch if [ -z "$pgstat_monitor_branch" ] then - export pgstat_monitor_branch="1.1.1" + export pgstat_monitor_branch="2.1.0" fi # If repo is not provided then it will default to percona PGSM repository diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index cf5ea16..129d560 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -284,6 +284,7 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'DISTRIBUTION': '', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } @@ -335,17 +336,19 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): # Define environment variables for playbook env_vars = { - 'PGSQL_VERSION': pgsql_version, + 'PDPGSQL_VERSION': pgsql_version, + 'PGSTAT_MONITOR_BRANCH': 'main', 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PGSQL_PGSS_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), + 'PDPGSQL_PGSM_CONTAINER': 'pgsql_pgsm_pmm_' + str(pgsql_version), 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'DISTRIBUTION': 'PGDG', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename - playbook_filename = 'pgsql_pgss_setup.yml' + playbook_filename = 'pdpgsql_pgsm_setup.yml' # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) From 136d1729ca5fd332630708a40ac22d3175dd3150 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:44:51 +0530 Subject: [PATCH 17/17] Pmm3 pgsql portfix (#106) * Updates to fix PGSQL port --- pmm_qa/pdpgsql_pgsm_setup.yml | 5 +++-- pmm_qa/pmm-framework.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index fd3ef2d..3672241 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -15,6 +15,7 @@ pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('2.1.0', true) ) }}" pgstat_monitor_repo: "{{ lookup('vars', 'extra_pgstat_monitor_repo', default=lookup('env','PGSTAT_MONITOR_REPO') | default('percona/pg_stat_monitor', true) ) }}" use_socket: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','USE_SOCKET') | default('', true) ) }}" + port: "{{ lookup('vars', 'extra_pdpgsql_port', default=lookup('env','PORT') | default(5447, true) ) }}" distribution: "{{ lookup('vars', 'extra_pdpgsql_distribution', default=lookup('env','DISTRIBUTION') | default('PPG', true) ) }}" tasks: @@ -32,7 +33,7 @@ - name: Prepare Container for PostgreSQL shell: > sudo docker run -d --name={{ pdpgsql_pgsm_container }} - -p 5447:5432 + -p {{ port }}:5432 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pdpgsql_pgsm_container @@ -46,7 +47,7 @@ - name: Execute Setup script inside the pdpgsql pdpgsql_pgsm_container shell: "{{ item }}" with_items: - - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution {{ distribution }} --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_pdpgsql_pgsm_{{ pdpgsql_version }}.log + - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution {{ distribution }} --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_{{ pdpgsql_pgsm_container }}.log - name: Install pmm2-client on the pdpgsql_pgsm_container shell: "{{ item }}" diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 129d560..b9129b4 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -284,6 +284,7 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PORT': 5447, 'DISTRIBUTION': '', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } @@ -343,6 +344,7 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PORT': 5448, 'DISTRIBUTION': 'PGDG', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' }