PMM-8641 PMM v3 API breaking changes

.devcontainer/setup.py

@@ -26,7 +26,7 @@ def install_packages():
     run_commands([
         "dnf install -y gcc git make pkgconfig \
             vim \
-            mc tmux psmisc lsof which iproute \
+            mc tmux psmisc lsof which iproute diffutils \
             bash-completion \
             man man-pages \
             openssl-devel \
@@ -47,16 +47,7 @@ def install_go():
 
     go_version = str(subprocess.check_output("gimme -r " + GO_VERSION, shell=True).strip().decode())
 
-    if GO_VERSION == "tip":
-        run_commands([
-            "mkdir $HOME/git_source",
-            "wget https://github.com/git/git/archive/refs/tags/v2.34.4.tar.gz -O $HOME/git.tar.gz",
-            "tar -xzf $HOME/git.tar.gz -C $HOME/git_source --strip-components 1",
-            "cd $HOME/git_source && make configure && ./configure --prefix=/usr && make all && make install",
-        ])
-        gimme_go_dir = "go"
-    else:
-        gimme_go_dir = "go{go_version}.linux.amd64".format(go_version=go_version)
+    gimme_go_dir = "go{go_version}.linux.amd64".format(go_version=go_version)
 
     run_commands([
         "gimme " + go_version,

.github/workflows/agent.yml

@@ -27,13 +27,13 @@ jobs:
     strategy:
       matrix:
         images:
-          - { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:10', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mysql:5.6', mongo: 'mongo:4.4', postgres: 'postgres:10', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
           - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:11', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
           - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
           - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
 
           # Percona + latest PMM Server release
-          - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:11', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:11', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
           - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:12.8-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
           - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:13.5-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
 
@@ -114,8 +114,8 @@ jobs:
       - name: Run debug commands on failure
         if: ${{ failure() }}
         run: |
-          env
-          go version
-          go env
-          pwd
+          echo "--- Environment variables ---"
+          env | sort
+          echo "--- GO Environment ---"
+          go env | sort
           git status

.github/workflows/api-tests.yml

@@ -132,7 +132,7 @@ jobs:
         run: |
           docker run -e PMM_SERVER_URL=${{env.PMM_URL}} \
             -e PMM_RUN_UPDATE_TEST=0 \
-            -e PMM_RUN_STT_TESTS=0 \
+            -e PMM_RUN_ADVISOR_TESTS=0 \
             --name pmm-api-tests \
             --network host \
             percona/pmm-api-tests

.github/workflows/clean.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Configure git for private modules
         env:
           ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-        run: git config --global url."https://percona-platform-robot:${ROBOT_TOKEN}@github.com".insteadOf "https://github.com"
+        run: echo "machine github.com login percona-robot password ${{ secrets.ROBOT_TOKEN }}" > $HOME/.netrc
 
       - name: Set up Go release
         if: matrix.go.version != 'tip'

.github/workflows/main.yml

@@ -79,7 +79,26 @@ jobs:
       - name: Run go-sumtype
         run: bin/go-sumtype ./...
 
-      - name: Run linters
+      - name: Run API linter
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.ROBOT_TOKEN || secrets.GITHUB_TOKEN }}
+        run: |
+          if out=$(bin/buf lint -v api); code="$?"; test "$code" -eq 0; then
+            echo "$out"
+            exit 0
+          fi
+
+          echo "API linter exited with code: $code"
+          echo "$out"
+          ## buf uses exit code 100 for linter warnings
+          if [ "$code" -ne 100 ] || ${{ github.event.pull_request == null }}; then
+            exit $code
+          fi
+
+          # One may need to suppress passing to reviewdog because of https://github.com/reviewdog/reviewdog/issues/1696
+          echo "$out" | bin/reviewdog -f=buf -reporter=github-pr-review -fail-on-error=true
+
+      - name: Run code linters
         uses: reviewdog/action-golangci-lint@v2
         with:
           github_token: ${{ secrets.ROBOT_TOKEN || secrets.GITHUB_TOKEN }}
@@ -92,20 +111,20 @@ jobs:
 
       - name: Run go-consistent
         env:
-          COMMAND: 'bin/go-consistent -pedantic -exclude "tests" ./...'
           REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.ROBOT_TOKEN || secrets.GITHUB_TOKEN }}
         run: |
-          if out=$( ${{ env.COMMAND }} ); exit_code=$?; [ $exit_code -ne 0 ]; then
-            if [ $exit_code -gt 1 ] || ${{ github.event.pull_request == null }}; then
-              echo "$out"
-              exit $exit_code
-            else
-              echo "$out" | bin/reviewdog -f=go-consistent -reporter=github-pr-review -fail-on-error=true
-            fi
-          else
+          if out=$(bin/go-consistent -pedantic -exclude "tests" ./...); exit_code=$?; [ $exit_code -eq 0 ]; then
+            echo "$out"
+            exit 0
+          fi
+
+          if [ $exit_code -gt 1 ] || ${{ github.event.pull_request == null }}; then
             echo "$out"
+            exit $exit_code
           fi
 
+          echo "$out" | bin/reviewdog -f=go-consistent -reporter=github-pr-review -fail-on-error=true
+
       - name: Test common API
         run: make test-common
 

.github/workflows/managed.yml

@@ -43,6 +43,7 @@ jobs:
       - name: Launch PMM Server (see docker-compose.yml)
         run: |
           make env-compose-up # the container workdir is /root/go/src/github.com/percona/pmm
+          docker exec -t pmm-server id -Z || :
           docker logs pmm-server
 
       - name: Mark the root directory of pmm as safe
@@ -123,7 +124,8 @@ jobs:
           docker exec pmm-server go env | sort
           docker exec pmm-server supervisorctl status || true
           services=$(docker exec pmm-server supervisorctl status | awk '{print $1}')
-          while IFS= read -r service; do
+          services=(`echo ${services}`)
+          for service in "${services[@]}"; do
             echo "Logs for $service:"
             docker exec pmm-server supervisorctl tail $service
-          done <<< "$services"
+          done

.golangci.yml

@@ -6,6 +6,10 @@ linters-settings:
   cyclop:
     max-complexity: 30
 
+  inamedparam:
+    # Skips check for interface methods with only a single parameter.
+    skip-single-param: true    
+
   depguard:
     rules:
       main:
@@ -81,11 +85,11 @@ linters:
   enable-all: true
   disable:
     # keep the rules sorted alpahbetically
+    - deadcode         # unmaintained, we leverage `unused`
+    - dupl             # we can't avoid duplicating code
     - execinquery      # false positives only
     - exhaustivestruct # too annoying
     - exhaustruct      # too many files to fix/nolint
-    - deadcode         # unmaintained, we leverage `unused`
-    - dupl             # we can't avoid duplicating code
     - funlen           # useless
     - gochecknoglobals # mostly useless
     - gochecknoinits   # we use init functions
@@ -100,7 +104,7 @@ linters:
     - maligned         # deprecated
     - nlreturn         # too annoying
     - nosnakecase      # deprecated
-    - protogetter      # mostly useless
+    - protogetter      # we need direct access to proto fields
     - rowserrcheck     # disabled because of generics
     - scopelint        # too many false positives
     - structcheck      # replaced by unused
@@ -145,6 +149,7 @@ issues:
         - forcetypeassert  # for tests' brevity sake
         - funlen           # tests may be long
         - gocognit         # triggered by subtests
+        - goconst          # not critical for tests
         - gomnd            # tests are full of magic numbers
         - ireturn          # we have exceptions, so need to silence them in tests
         - lll              # tests often require long lines

.mockery.yaml

@@ -38,7 +38,6 @@ packages:
     interfaces:
       agentsRegistry:
       agentsStateUpdater:
-      authProvider:
       checksService:
       connectionChecker:
       grafanaClient:
@@ -47,7 +46,7 @@ packages:
       serviceInfoBroker:
       versionCache:
       victoriaMetricsClient:
-  github.com/percona/pmm/managed/services/management/alerting:
+  github.com/percona/pmm/managed/services/alerting:
     interfaces:
       grafanaClient:
   github.com/percona/pmm/managed/services/management/backup:

Makefile.include

@@ -48,7 +48,22 @@ gen-api:            ## Generate PMM API
 
 	bin/buf generate -v api
 
-	for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/alerting api/managementpb/backup api/managementpb/dump api/managementpb/azure api/managementpb/role api/qanpb api/managementpb/agent api/managementpb/node api/managementpb/service api/platformpb api/userpb; do \
+	SPECS="\
+		api/agentlocal/v1 \
+		api/server/v1 \
+		api/user/v1 \
+		api/inventory/v1 \
+		api/management/v1 \
+		api/management/v1/service \
+		api/actions/v1 \
+		api/advisors/v1 \
+		api/alerting/v1 \
+		api/backup/v1 \
+		api/dump/v1beta1 \
+		api/accesscontrol/v1beta1 \
+		api/qan/v1 \
+		api/platform/v1"; \
+	for API in $$SPECS; do \
 		set -x ; \
 		bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
 		bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
@@ -67,37 +82,40 @@ gen-api:            ## Generate PMM API
 			--additional-initialism=ha ; \
 	done
 
-	# generate public API spec, omit agentlocalpb (always private),
+	# generate public API spec, omit agentlocal and inventory (always private),
 	# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
 	bin/swagger mixin --output=api/swagger/swagger.json \
 		api/swagger/header.json \
-		api/serverpb/json/serverpb.json \
-		api/userpb/json/userpb.json \
-		api/inventorypb/json/inventorypb.json \
-		api/managementpb/json/managementpb.json \
-		api/managementpb/backup/json/backup.json \
-		api/managementpb/alerting/json/alerting.json
+		api/server/v1/json/v1.json \
+		api/user/v1/json/v1.json \
+		api/inventory/v1/json/v1.json \
+		api/management/v1/json/v1.json \
+		api/actions/v1/json/v1.json \
+		api/alerting/v1/json/v1.json \
+		api/advisors/v1/json/v1.json \
+		api/backup/v1/json/v1.json \
+		api/qan/v1/json/v1.json \
+		api/platform/v1/json/v1.json
 	bin/swagger validate api/swagger/swagger.json
 
 	bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json
 
-	# generate API spec with all PMM Server APIs (omit agentlocalpb)
+	# generate API spec with all PMM Server APIs (omit agentlocal)
 	bin/swagger mixin --output=api/swagger/swagger-dev.json \
 		api/swagger/header-dev.json \
-		api/serverpb/json/serverpb.json \
-		api/userpb/json/userpb.json \
-		api/inventorypb/json/inventorypb.json \
-		api/managementpb/json/managementpb.json \
-		api/managementpb/alerting/json/alerting.json \
-		api/managementpb/backup/json/backup.json \
-		api/managementpb/dump/json/dump.json \
-		api/managementpb/azure/json/azure.json \
-		api/managementpb/role/json/role.json \
-		api/managementpb/agent/json/agent.json \
-		api/managementpb/node/json/node.json \
-		api/managementpb/service/json/service.json \
-		api/qanpb/json/qanpb.json \
-		api/platformpb/json/platformpb.json
+		api/server/v1/json/v1.json \
+		api/user/v1/json/v1.json \
+		api/inventory/v1/json/v1.json \
+		api/management/v1/json/v1.json \
+		api/actions/v1/json/v1.json \
+		api/alerting/v1/json/v1.json \
+		api/advisors/v1/json/v1.json \
+		api/backup/v1/json/v1.json \
+		api/dump/v1beta1/json/v1beta1.json \
+		api/accesscontrol/v1beta1/json/v1beta1.json \
+		api/qan/v1/json/v1.json \
+		api/platform/v1/json/v1.json
+
 	bin/swagger validate api/swagger/swagger-dev.json
 
 	bin/swagger-order --output=api/swagger/swagger-dev.json api/swagger/swagger-dev.json
@@ -124,7 +142,21 @@ clean: clean_swagger  ## Remove generated files
 	find api -name '*.pb.gw.go' -print -delete
 	find api -name '*.validate.go' -print -delete
 
-	for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/alerting api/managementpb/backup api/management/dump api/managementpb/role api/managementpb/agent api/managementpb/node api/managementpb/service api/qanpb api/platformpb ; do \
+	SPECS="\
+		api/agentlocal/v1 \
+		api/server/v1 \
+		api/user/v1 \
+		api/inventory/v1 \
+		api/management/v1 \
+		api/actions/v1 \
+		api/alerting/v1 \
+		api/advisors/v1 \
+		api/backup/v1 \
+		api/dump/v1beta1 \
+		api/accesscontrol/v1beta1 \
+		api/qan/v1 \
+		api/platform/v1"; \
+	for API in $$SPECS; do \
 		rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
 	done
 	rm -f api/swagger/swagger.json api/swagger/swagger-dev.json
@@ -136,8 +168,9 @@ api-test:             ## Run API tests on dev env.
 	go test -count=1 -race -p 1 -v ./api-tests/... -pmm.server-insecure-tls
 
 check:                ## Run required checkers and linters
-	LOG_LEVEL=error bin/golangci-lint run ; \
-	bin/go-sumtype ./... ; \
+	bin/buf lint -v api
+	LOG_LEVEL=error bin/golangci-lint run
+	bin/go-sumtype ./...
 	bin/go-consistent -pedantic ./...
 
 check-license:          ## Run license header checks against source files

admin/README.md

@@ -9,7 +9,7 @@ git, make, curl, go, gcc, pmm-server, pmm-agent
 
 ## Local setup
 ### To run pmm-admin commands
-- Run [pmm-server docker container](https://hub.docker.com/r/percona/pmm-server) or [pmm-managed](https://github.com/percona/pmm-managed).  
+- Run [pmm-server docker container](https://hub.docker.com/r/percona/pmm-server) or [pmm-managed](https://github.com/percona/pmm/tree/main/managed).  
 - Run pmm-agent `cd ../agent`.
 - Run pmm-admin commands.
     ```shell script
@@ -18,8 +18,9 @@ git, make, curl, go, gcc, pmm-server, pmm-agent
 
 You should see something like this
  ```shell script
-Agent ID: /agent_id/fcbe3cb4-a95a-43f4-aef5-c3494caa5132
-Node ID : /node_id/77be6b4d-a1d9-4687-8fae-7acbaee7db47
+Agent ID: fcbe3cb4-a95a-43f4-aef5-c3494caa5132
+Node ID : 77be6b4d-a1d9-4687-8fae-7acbaee7db47
+Node name: mysql-server-test-1
 
 PMM Server:
         URL    : https://127.0.0.1:443/
@@ -31,10 +32,10 @@ PMM-agent:
         Latency   : 211.026µs
 
 Agents:
-        /agent_id/3329a405-8a5d-4414-9890-b6ae4209e0cc NODE_EXPORTER RUNNING
+        3329a405-8a5d-4414-9890-b6ae4209e0cc NODE_EXPORTER RUNNING 40001
 ```
 It means that everything works.
 
 ## Testing
-pmm-admin doesn't require setting-up environment.  
+pmm-admin doesn't require setting-up an environment.  
 Run `make test` to run tests. 

admin/agentlocal/agentlocal.go

@@ -27,8 +27,8 @@ import (
 	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/sirupsen/logrus"
 
-	"github.com/percona/pmm/api/agentlocalpb/json/client"
-	agentlocal "github.com/percona/pmm/api/agentlocalpb/json/client/agent_local"
+	"github.com/percona/pmm/api/agentlocal/v1/json/client"
+	agentlocal "github.com/percona/pmm/api/agentlocal/v1/json/client/agent_local_service"
 )
 
 // SetTransport configures transport for accessing local pmm-agent API.
@@ -100,7 +100,7 @@ func GetRawStatus(ctx context.Context, requestNetworkInfo NetworkInfo) (*agentlo
 		Context: ctx,
 	}
 
-	res, err := client.Default.AgentLocal.Status(params)
+	res, err := client.Default.AgentLocalService.Status(params)
 	if err != nil {
 		if res == nil {
 			return nil, err

admin/cmd/bootstrap.go

@@ -131,7 +131,7 @@ func getDefaultKongOptions(appName string) []kong.Option {
 
 	var defaultMachineID string
 	if nodeinfo.MachineID != "" {
-		defaultMachineID = "/machine_id/" + nodeinfo.MachineID
+		defaultMachineID = nodeinfo.MachineID
 	}
 
 	mysqlQuerySources := []string{