Merge branch 'v3' into PMM-10684-clickhouse-v2

.github/workflows/admin.yml

@@ -64,19 +64,18 @@ jobs:
         run: make test-cover
 
       - name: Upload coverage results
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: cover.out
           flags: admin
           fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Run debug commands on failure
         if: ${{ failure() }}
         run: |
-          env
-          go version
-          go env
-          pwd
+          env | sort
+          go env | sort
           git status
 
   cli-test:
@@ -160,9 +159,9 @@ jobs:
       - name: Run debug commands on failure
         if: ${{ failure() }}
         run: |
-          env
-          go version
-          go env
-          node --version
-          pwd
+          env | sort
+          go env | sort
           git status
+          node --version
+          npx --version
+          npx playwright --version

.github/workflows/agent.yml

@@ -94,7 +94,7 @@ jobs:
       - name: Build and install
         run: make install
 
-      - name: Docker-compose
+      - name: Launch containers
         env:
           ENV_UP_FLAGS: "--detach"
         run: make env-up
@@ -103,12 +103,13 @@ jobs:
         run: make test-cover
 
       - name: Upload coverage results
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: cover.out
           flags: agent
           env_vars: MYSQL_IMAGE,MONGO_IMAGE,POSTGRES_IMAGE,PMM_SERVER_IMAGE
           fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Run debug commands on failure
         if: ${{ failure() }}

.github/workflows/api-tests.yml

@@ -94,9 +94,8 @@ jobs:
               container_name: pmm-agent_pmm-server
               environment:
                 - PMM_DEBUG=1
-                - PERCONA_TEST_CHECKS_INTERVAL=10s
-                - PERCONA_TEST_PLATFORM_ADDRESS=https://check-dev.percona.com
-                - PERCONA_TEST_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX
+                - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check-dev.percona.com
+                - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX
               ports:
                 - 80:8080
                 - 443:8443

.github/workflows/clean.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         go:
-          - version: 1.21.x
+          - version: 1.22.x
             may-fail: false
           - version: tip
             may-fail: true

.github/workflows/dependabot.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1
+        uses: dependabot/fetch-metadata@v2
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/managed.yml

@@ -29,8 +29,8 @@ jobs:
       PMM_SERVER_IMAGE: perconalab/pmm-server:3-dev-latest
       AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
       AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
-      OAUTH_PMM_CLIENT_ID: ${{ secrets.OAUTH_PMM_CLIENT_ID }}
-      OAUTH_PMM_CLIENT_SECRET: ${{ secrets.OAUTH_PMM_CLIENT_SECRET }}
+      PMM_DEV_OAUTH_CLIENT_ID: ${{ secrets.OAUTH_PMM_CLIENT_ID }}
+      PMM_DEV_OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_PMM_CLIENT_SECRET }}
       DEVCONTAINER_CACHE_ENABLED: false
 
     steps:
@@ -98,12 +98,13 @@ jobs:
           # docker exec -i pmm-server make -C managed test-update
 
       - name: Upload coverage results
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: managed/cover.out
           flags: managed
           env_vars: PMM_SERVER_IMAGE
           fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Cache
         if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}

.github/workflows/update.yml

@@ -62,8 +62,10 @@ jobs:
         run: docker exec pmm-update-server make -C /root/go/src/github.com/percona/pmm/update run-race-cover RUN_FLAGS='-debug -check'
 
       - name: Upload coverage results
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: cover.out
           flags: update
           fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
+

.github/workflows/vmproxy.yml

@@ -66,17 +66,16 @@ jobs:
         run: make test-cover
 
       - name: Upload coverage results
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: cover.out
           flags: vmproxy
           fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Run debug commands on failure
         if: ${{ failure() }}
         run: |
-          env
-          go version
-          go env
-          pwd
+          env | sort
+          go env | sort
           git status

.mockery.yaml

@@ -38,7 +38,7 @@ packages:
     interfaces:
       agentsRegistry:
       agentsStateUpdater:
-      apiKeyProvider:
+      authProvider:
       checksService:
       connectionChecker:
       grafanaClient:

Dockerfile

@@ -1,6 +1,6 @@
 # This Dockerfile is used only for API tests.
 
-FROM golang:1.21
+FROM golang:1.22
 
 RUN export GOPATH=$(go env GOPATH) && \
     mkdir -p $GOPATH/src/github.com/percona/pmm

admin/Makefile

@@ -51,8 +51,8 @@ test-cover:                     ## Run tests and collect cross-package coverage
 	go test $(TEST_FLAGS) -race -coverprofile=cover.out -covermode=atomic -coverpkg=./... ./...
 
 env-up:                         ## Start development environment
-	docker-compose up --force-recreate --abort-on-container-exit --renew-anon-volumes --remove-orphans
+	docker compose up --force-recreate --abort-on-container-exit --renew-anon-volumes --remove-orphans
 
 env-down:                       ## Stop development environment
-	docker-compose down --volumes --remove-orphans
+	docker compose down --volumes --remove-orphans
 

admin/commands/base/setup.go

@@ -88,8 +88,13 @@ func SetupClients(ctx context.Context, globalFlags *flags.GlobalFlags) {
 	// use JSON APIs over HTTP/1.1
 	transport := httptransport.New(globalFlags.ServerURL.Host, globalFlags.ServerURL.Path, []string{globalFlags.ServerURL.Scheme})
 	if u := globalFlags.ServerURL.User; u != nil {
+		user := u.Username()
 		password, _ := u.Password()
-		transport.DefaultAuthentication = httptransport.BasicAuth(u.Username(), password)
+		if user == "service_token" || user == "api_key" {
+			transport.DefaultAuthentication = httptransport.BearerToken(password)
+		} else {
+			transport.DefaultAuthentication = httptransport.BasicAuth(user, password)
+		}
 	}
 	transport.SetLogger(logrus.WithField("component", "server-transport"))
 	transport.SetDebug(globalFlags.EnableDebug || globalFlags.EnableTrace)

admin/commands/management/unregister.go

@@ -16,12 +16,15 @@ package management
 
 import (
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 
 	"github.com/percona/pmm/admin/agentlocal"
 	"github.com/percona/pmm/admin/commands"
 	"github.com/percona/pmm/admin/helpers"
-	"github.com/percona/pmm/api/inventorypb/json/client"
+	inventoryClient "github.com/percona/pmm/api/inventorypb/json/client"
 	"github.com/percona/pmm/api/inventorypb/json/client/nodes"
+	"github.com/percona/pmm/api/managementpb/json/client"
+	"github.com/percona/pmm/api/managementpb/json/client/node"
 )
 
 type unregisterResult struct {
@@ -63,7 +66,7 @@ func (cmd *UnregisterCommand) RunCmd() (commands.Result, error) {
 		}
 
 		nodeID = status.NodeID
-		node, err := client.Default.Nodes.GetNode(&nodes.GetNodeParams{
+		node, err := inventoryClient.Default.Nodes.GetNode(&nodes.GetNodeParams{
 			Context: commands.Ctx,
 			Body: nodes.GetNodeBody{
 				NodeID: nodeID,
@@ -78,27 +81,31 @@ func (cmd *UnregisterCommand) RunCmd() (commands.Result, error) {
 		}
 	}
 
-	params := &nodes.RemoveNodeParams{
-		Body: nodes.RemoveNodeBody{
+	params := &node.UnregisterNodeParams{
+		Body: node.UnregisterNodeBody{
 			NodeID: nodeID,
 			Force:  cmd.Force,
 		},
 		Context: commands.Ctx,
 	}
 
-	_, err = client.Default.Nodes.RemoveNode(params)
+	res, err := client.Default.Node.UnregisterNode(params)
 	if err != nil {
 		return nil, err
 	}
 
+	if res.Payload.Warning != "" {
+		logrus.Warning(res.Payload.Warning)
+	}
+
 	return &unregisterResult{
 		NodeID:   nodeID,
 		NodeName: nodeName,
 	}, nil
 }
 
 func nodeIDFromNodeName(nodeName string) (string, error) {
-	listNodes, err := client.Default.Nodes.ListNodes(nil)
+	listNodes, err := inventoryClient.Default.Nodes.ListNodes(nil)
 	if err != nil {
 		return "", err
 	}

agent/Makefile

@@ -110,10 +110,10 @@ env-up:                         ## Start development environment
 	mkdir -p testdata/mysql/slowlogs
 	chmod -R 0777 testdata
 
-	docker-compose up $(ENV_UP_FLAGS)
+	docker compose up $(ENV_UP_FLAGS)
 
 env-down:                       ## Stop development environment
-	docker-compose down --volumes --remove-orphans
+	docker compose down --volumes --remove-orphans
 
 setup-dev: install              ## Run pmm-agent setup in development environment
 	pmm-agent setup $(RUN_FLAGS) --server-insecure-tls --server-address=127.0.0.1:${PMM_DEV_SERVER_PORT} --server-username=admin --server-password=admin --paths-exporters_base=$(GOPATH)/bin --force
@@ -128,13 +128,13 @@ env-psql:                       ## Run psql client
 	docker exec -ti pmm-agent_postgres env PGPASSWORD=pmm-agent-password psql --username=pmm-agent
 
 env-sysbench-prepare:
-	docker-compose exec --workdir=/sysbench/sysbench-tpcc sysbench ./tpcc.lua \
+	docker compose exec --workdir=/sysbench/sysbench-tpcc sysbench ./tpcc.lua \
 		--db-driver=pgsql --pgsql-host=postgres --pgsql-user=pmm-agent --pgsql-password=pmm-agent-password --pgsql-db=pmm-agent \
 		--threads=1 --time=0 --report-interval=10 \
 		--tables=1 --scale=10  --use_fk=0 --enable_purge=yes prepare
 
 env-sysbench-run:
-	docker-compose exec --workdir=/sysbench/sysbench-tpcc sysbench ./tpcc.lua \
+	docker compose exec --workdir=/sysbench/sysbench-tpcc sysbench ./tpcc.lua \
 		--db-driver=pgsql --pgsql-host=postgres --pgsql-user=pmm-agent --pgsql-password=pmm-agent-password --pgsql-db=pmm-agent \
 		--threads=4 --time=0 --rate=10 --report-interval=10 --percentile=99 \
 		--tables=1 --scale=10  --use_fk=0 --enable_purge=yes run

agent/agents/mongodb/internal/profiler/profiler_test.go

@@ -222,7 +222,9 @@ func testProfiler(t *testing.T, url string) {
 			Query: findBucket.Common.Example,
 		}
 
-		ex := actions.NewMongoDBExplainAction(id, 5*time.Second, params, os.TempDir())
+		ex, err := actions.NewMongoDBExplainAction(id, 5*time.Second, params, os.TempDir())
+		require.NoError(t, err)
+
 		ctx, cancel := context.WithTimeout(context.Background(), ex.Timeout())
 		defer cancel()
 		res, err := ex.Run(ctx)

agent/agents/mysql/perfschema/perfschema.go

@@ -129,7 +129,7 @@ func getPerfschemaHistorySize(q reform.Querier, l *logrus.Entry) uint {
 // New creates new PerfSchema QAN service.
 func New(params *Params, l *logrus.Entry) (*PerfSchema, error) {
 	if params.TextFiles != nil {
-		err := tlshelpers.RegisterMySQLCerts(params.TextFiles.Files)
+		err := tlshelpers.RegisterMySQLCerts(params.TextFiles.Files, params.TLSSkipVerify)
 		if err != nil {
 			return nil, err
 		}

agent/agents/mysql/slowlog/slowlog.go

@@ -17,7 +17,9 @@ package slowlog
 
 import (
 	"context"
+	"crypto/md5" //nolint:gosec
 	"database/sql"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"math"
@@ -82,7 +84,7 @@ type slowLogInfo struct {
 // New creates new SlowLog QAN service.
 func New(params *Params, l *logrus.Entry) (*SlowLog, error) {
 	if params.TextFiles != nil {
-		err := tlshelpers.RegisterMySQLCerts(params.TextFiles.Files)
+		err := tlshelpers.RegisterMySQLCerts(params.TextFiles.Files, params.TLSSkipVerify)
 		if err != nil {
 			return nil, err
 		}
@@ -370,7 +372,7 @@ func (s *SlowLog) processFile(ctx context.Context, file string, outlierTime floa
 
 			s.l.Tracef("Parsed slowlog event: %+v.", e)
 			fingerprint := query.Fingerprint(e.Query)
-			digest := query.Id(fingerprint)
+			digest := hashIntoQueryID(fingerprint)
 			aggregator.AddEvent(e, digest, e.User, e.Host, e.Db, e.Server, e.Query)
 
 		case <-t.C:
@@ -391,6 +393,18 @@ func (s *SlowLog) processFile(ctx context.Context, file string, outlierTime floa
 	}
 }
 
+// hashIntoQueryID returns slowlog query ID hashed by MD5 from given fingerprint.
+func hashIntoQueryID(fingerprint string) string {
+	// MD5 is used only to hash fingerprint into query ID, so there is no risk.
+	// It is ideal due to its length (32 chars) and it corresponds to Perfschema query ID length.
+	id := md5.New() //nolint:gosec
+	_, err := io.WriteString(id, fingerprint)
+	if err != nil {
+		logrus.Debugf("cannot hash fingerprint into query ID: %s", err.Error())
+	}
+	return strings.ToUpper(hex.EncodeToString(id.Sum(nil)))
+}
+
 // makeBuckets is a pure function for easier testing.
 //
 //nolint:cyclop,maintidx

agent/client/basic_auth.go

@@ -17,6 +17,7 @@ package client
 import (
 	"context"
 	"encoding/base64"
+	"fmt"
 
 	"google.golang.org/grpc/credentials"
 )
@@ -31,7 +32,7 @@ func (b *basicAuth) GetRequestMetadata(ctx context.Context, uri ...string) (map[
 	auth := b.username + ":" + b.password
 	enc := base64.StdEncoding.EncodeToString([]byte(auth))
 	return map[string]string{
-		"authorization": "Basic " + enc,
+		"Authorization": fmt.Sprintf("Basic %s", enc),
 	}, nil
 }