Merge branch 'v3' into PMM-13129-encryption

api-tests/helpers.go

@@ -145,6 +145,7 @@ func UnregisterNodes(t TestingT, nodeIDs ...string) {
 		params := &node.UnregisterNodeParams{
 			Body: node.UnregisterNodeBody{
 				NodeID: nodeID,
+				Force:  true,
 			},
 			Context: context.Background(),
 		}

api-tests/management/nodes_test.go

@@ -150,7 +150,6 @@ func TestNodeRegister(t *testing.T) {
 			})
 			assert.NotEmpty(t, nodeID)
 			assert.NotEmpty(t, pmmAgentID)
-			pmmapitests.UnregisterNodes(t, nodeID)
 
 			body := node.RegisterNodeBody{
 				NodeName:   nodeName + "_new",

docker-compose.yml

@@ -33,8 +33,6 @@ services:
       # - PMM_DEV_TELEMETRY_RETRY_BACKOFF=10s
       # - PMM_CLICKHOUSE_ADDR=127.0.0.1:9000
       # - PMM_CLICKHOUSE_DATABASE=pmm
-      # - PMM_CLICKHOUSE_BLOCK_SIZE=10000
-      # - PMM_CLICKHOUSE_POOL_SIZE=2
       # - PMM_DEBUG=1
       # - PMM_DEV_ADVISOR_CHECKS_FILE=/srv/checks/local-checks.yml
       # - PMM_POSTGRES_ADDR=pg
@@ -232,8 +230,6 @@ services:
       # - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1
       - PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000
       - PMM_CLICKHOUSE_DATABASE=pmm
-      - PMM_CLICKHOUSE_BLOCK_SIZE=10000
-      - PMM_CLICKHOUSE_POOL_SIZE=2
       - PMM_DISABLE_BUILTIN_CLICKHOUSE=1
       - PMM_POSTGRES_ADDR=pg:5432
       - PMM_POSTGRES_USERNAME=pmm-managed
@@ -315,8 +311,6 @@ services:
       # - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1
       - PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000
       - PMM_CLICKHOUSE_DATABASE=pmm
-      - PMM_CLICKHOUSE_BLOCK_SIZE=10000
-      - PMM_CLICKHOUSE_POOL_SIZE=2
       - PMM_DISABLE_BUILTIN_CLICKHOUSE=1
       - PMM_POSTGRES_ADDR=pg:5432
       - PMM_POSTGRES_USERNAME=pmm-managed
@@ -397,8 +391,6 @@ services:
       # - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1
       - PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000
       - PMM_CLICKHOUSE_DATABASE=pmm
-      - PMM_CLICKHOUSE_BLOCK_SIZE=10000
-      - PMM_CLICKHOUSE_POOL_SIZE=2
       - PMM_DISABLE_BUILTIN_CLICKHOUSE=1
       - PMM_POSTGRES_ADDR=pg:5432
       - PMM_POSTGRES_USERNAME=pmm-managed

docs/process/v2_to_v3_environment_variables.md

@@ -26,11 +26,11 @@ Below is a list of affected variables and their new names.
 | `PERCONA_TEST_CHECKS_PUBLIC_KEY`              |                                            | Removed in PMM v3, use `PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY` |
 | `PERCONA_TEST_NICER_API`                      | `PMM_NICER_API`                            |                                                              |
 | `PERCONA_TEST_PMM_CLICKHOUSE_ADDR`            | `PMM_CLICKHOUSE_ADDR`                      |                                                              |
-| `PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE`      | `PMM_CLICKHOUSE_BLOCK_SIZE`                |                                                              |
+| `PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE`      |                                            | Removed in PMM v3, because of new clickhouse version.        |
 | `PERCONA_TEST_PMM_CLICKHOUSE_DATABASE`        | `PMM_CLICKHOUSE_DATABASE`                  |                                                              |
 | `PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE`      | `PMM_CLICKHOUSE_DATASOURCE`                |                                                              |
 | `PERCONA_TEST_PMM_CLICKHOUSE_HOST`            | `PMM_CLICKHOUSE_HOST`                      |                                                              |
-| `PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE`       | `PMM_CLICKHOUSE_POOL_SIZE`                 |                                                              |
+| `PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE`       |                                            | Removed in PMM v3, because of new clickhouse version.        |
 | `PERCONA_TEST_PMM_CLICKHOUSE_PORT`            | `PMM_CLICKHOUSE_PORT`                      |                                                              |
 | `PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE` | `PMM_DISABLE_BUILTIN_CLICKHOUSE`           |                                                              |
 | `PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES`   | `PMM_DISABLE_BUILTIN_POSTGRES`             |                                                              |

go.mod

@@ -15,11 +15,8 @@ replace gopkg.in/alecthomas/kingpin.v2 => github.com/Percona-Lab/kingpin v2.2.6-
 
 replace golang.org/x/crypto => github.com/percona-lab/crypto v0.0.0-20231108144114-756dfb24eaf2
 
-replace github.com/ClickHouse/clickhouse-go/151 => github.com/ClickHouse/clickhouse-go v1.5.1 // clickhouse-go/v2 cannot work with 1.5.1 which we need for QAN-API
-
 require (
 	github.com/AlekSi/pointer v1.2.0
-	github.com/ClickHouse/clickhouse-go/151 v0.0.0-00010101000000-000000000000
 	github.com/ClickHouse/clickhouse-go/v2 v2.23.0
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/alecthomas/kong v0.9.0
@@ -124,7 +121,6 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
 	github.com/ClickHouse/ch-go v0.61.5 // indirect
-	github.com/ClickHouse/clickhouse-go v1.5.4 // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
@@ -133,7 +129,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/charmbracelet/harmonica v0.2.0 // indirect
-	github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
 	github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect

go.sum

@@ -19,8 +19,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
 github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
-github.com/ClickHouse/clickhouse-go v1.5.1 h1:I8zVFZTz80crCs0FFEBJooIxsPcV0xfthzK1YrkpJTc=
-github.com/ClickHouse/clickhouse-go v1.5.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
 github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=
 github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
 github.com/ClickHouse/clickhouse-go/v2 v2.23.0 h1:srmRrkS0BR8gEut87u8jpcZ7geOob6nGj9ifrb+aKmg=
@@ -70,8 +68,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
-github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8=
@@ -178,7 +174,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr
 github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
 github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
 github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
-github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
@@ -293,7 +288,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -332,7 +326,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
@@ -356,7 +349,6 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
@@ -430,9 +422,6 @@ github.com/percona/promconfig v0.2.5 h1:f/HN/CbECQs7d9RIB6MKVkuXstsrsqEDxRvf6yig
 github.com/percona/promconfig v0.2.5/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
 github.com/pganalyze/pg_query_go/v2 v2.2.0 h1:OW+reH+ZY7jdEuPyuLGlf1m7dLbE+fDudKXhLs0Ttpk=
 github.com/pganalyze/pg_query_go/v2 v2.2.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA=
-github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
-github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=

managed/cmd/pmm-managed/main.go

@@ -264,7 +264,8 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) {
 	inventorypb.RegisterServicesServer(gRPCServer, inventorygrpc.NewServicesServer(servicesSvc, mgmtServices))
 	inventorypb.RegisterAgentsServer(gRPCServer, inventorygrpc.NewAgentsServer(agentsSvc))
 
-	nodeSvc := management.NewNodeService(deps.db, deps.grafanaClient)
+	nodeSvc := management.NewNodeService(deps.db, deps.grafanaClient, deps.agentsRegistry, deps.agentsStateUpdater,
+		deps.vmdb)
 	agentSvc := management.NewAgentService(deps.db, deps.agentsRegistry)
 	serviceSvc := management.NewServiceService(deps.db, deps.agentsRegistry, deps.agentsStateUpdater, deps.vmdb)
 	mysqlSvc := management.NewMySQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck, deps.serviceInfoBroker, deps.versionCache)

managed/services/management/node.go

@@ -19,6 +19,7 @@ import (
 	"context"
 
 	"github.com/AlekSi/pointer"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -33,23 +34,30 @@ import (
 
 // NodeService represents service for working with nodes.
 type NodeService struct {
-	db *reform.DB
-	ap authProvider
-	l  *logrus.Entry
+	db    *reform.DB
+	ap    authProvider
+	l     *logrus.Entry
+	r     agentsRegistry
+	state agentsStateUpdater
+	vmdb  prometheusService
 }
 
 // NewNodeService creates NodeService instance.
-func NewNodeService(db *reform.DB, ap authProvider) *NodeService {
+func NewNodeService(db *reform.DB, ap authProvider, r agentsRegistry, state agentsStateUpdater, vmdb prometheusService) *NodeService {
 	return &NodeService{
-		db: db,
-		ap: ap,
-		l:  logrus.WithField("component", "node"),
+		db:    db,
+		ap:    ap,
+		r:     r,
+		state: state,
+		vmdb:  vmdb,
+		l:     logrus.WithField("component", "node"),
 	}
 }
 
 // Register do registration of the new node.
 func (s *NodeService) Register(ctx context.Context, req *managementpb.RegisterNodeRequest) (*managementpb.RegisterNodeResponse, error) {
 	res := &managementpb.RegisterNodeResponse{}
+
 	e := s.db.InTransaction(func(tx *reform.TX) error {
 		node, err := models.FindNodeByName(tx.Querier, req.NodeName)
 		switch status.Code(err) { //nolint:exhaustive
@@ -147,16 +155,60 @@ func (s *NodeService) Register(ctx context.Context, req *managementpb.RegisterNo
 
 // Unregister do unregistration of the node.
 func (s *NodeService) Unregister(ctx context.Context, req *managementpb.UnregisterNodeRequest) (*managementpb.UnregisterNodeResponse, error) {
+	idsToKick := make(map[string]struct{})
+	idsToSetState := make(map[string]struct{})
+
 	node, err := models.FindNodeByID(s.db.Querier, req.NodeId)
 	if err != nil {
 		return nil, err
 	}
 
-	err = s.db.InTransaction(func(tx *reform.TX) error {
-		return models.RemoveNode(tx.Querier, req.NodeId, models.RemoveCascade)
-	})
-	if err != nil {
-		return nil, err
+	if e := s.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error {
+		mode := models.RemoveRestrict
+		if req.Force {
+			mode = models.RemoveCascade
+
+			agents, err := models.FindPMMAgentsRunningOnNode(tx.Querier, node.NodeID)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			for _, a := range agents {
+				idsToKick[a.AgentID] = struct{}{}
+			}
+
+			agents, err = models.FindAgents(tx.Querier, models.AgentFilters{NodeID: node.NodeID})
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			for _, a := range agents {
+				if a.PMMAgentID != nil {
+					idsToSetState[pointer.GetString(a.PMMAgentID)] = struct{}{}
+				}
+			}
+
+			agents, err = models.FindPMMAgentsForServicesOnNode(tx.Querier, node.NodeID)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			for _, a := range agents {
+				idsToSetState[a.AgentID] = struct{}{}
+			}
+		}
+		return models.RemoveNode(tx.Querier, node.NodeID, mode)
+	}); e != nil {
+		return nil, e
+	}
+
+	for id := range idsToSetState {
+		s.state.RequestStateUpdate(ctx, id)
+	}
+	for id := range idsToKick {
+		s.r.Kick(ctx, id)
+	}
+
+	if req.Force {
+		// It's required to regenerate victoriametrics config file for the agents which aren't run by pmm-agent.
+		s.vmdb.RequestConfigurationUpdate()
 	}
 
 	warning, err := s.ap.DeleteServiceAccount(ctx, node.NodeName, req.Force)