Merge remote-tracking branch 'origin/main' into v3

# Conflicts:
#	api/managementpb/dbaas/components.pb.go
#	api/managementpb/dbaas/db_clusters.pb.go
#	api/managementpb/dbaas/dbaas.pb.go
#	api/managementpb/dbaas/kubernetes.pb.go
#	api/managementpb/dbaas/logs.pb.go
#	api/managementpb/dbaas/psmdb_clusters.pb.go
#	api/managementpb/dbaas/pxc_clusters.pb.go
#	api/managementpb/dbaas/templates.pb.go
#	api/managementpb/ia/alerts.pb.go
#	api/managementpb/ia/channels.pb.go
#	api/managementpb/ia/rules.pb.go
#	api/managementpb/ia/status.pb.go
#	api/serverpb/json/client/server/test_email_alerting_settings_responses.go
#	api/serverpb/json/serverpb.json
#	api/swagger/swagger-dev.json
#	api/swagger/swagger.json
#	descriptor.bin
#	docker-compose.yml
#	managed/models/database.go
#	managed/services/management/node.go

.github/workflows/dependabot.yml

@@ -21,4 +21,4 @@ jobs:
         run: gh pr merge --auto --squash "$PR_URL"
         env:
           PR_URL: ${{github.event.pull_request.html_url}}
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -38,3 +38,6 @@ compose.yml
 
 cli-tests/node_modules/
 cli-tests/playwright-report/
+
+api-tests/pmm-api-tests-output.txt
+api-tests/pmm-api-tests-junit-report.xml

admin/commands/inventory/add_agent_mysqld_exporter.go

@@ -62,7 +62,7 @@ func (res *addAgentMysqldExporterResult) TablestatStatus() string {
 	}
 
 	switch {
-	case res.Agent.TablestatsGroupTableLimit == 0: // no limit
+	case res.Agent.TablestatsGroupTableLimit == 0: // server defined
 		s += " (the table count limit is not set)."
 	case res.Agent.TablestatsGroupTableLimit < 0: // always disabled
 		s += " (always)."
@@ -94,8 +94,9 @@ type AddAgentMysqldExporterCommand struct {
 	TLSCAFile                 string            `name:"tls-ca" help:"Path to certificate authority certificate file"`
 	TLSCertFile               string            `name:"tls-cert" help:"Path to client certificate file"`
 	TLSKeyFile                string            `name:"tls-key" help:"Path to client key file"`
-	TablestatsGroupTableLimit int32             `placeholder:"number" help:"Tablestats group collectors will be disabled if there are more than that number of tables (default: 0 - always enabled; negative value - always disabled)"`
+	TablestatsGroupTableLimit int32             `placeholder:"number" help:"Tablestats group collectors will be disabled if there are more than that number of tables (default: server-defined, -1: always disabled)"`
 	PushMetrics               bool              `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
+	ExposeExporter            bool              `help:"Expose the address of the exporter publicly on 0.0.0.0"`
 	DisableCollectors         []string          `help:"Comma-separated list of collector names to exclude from exporter"`
 	LogLevel                  string            `enum:"debug,info,warn,error" default:"warn" help:"Service logging level. One of: [debug, info, warn, error]"`
 }
@@ -140,6 +141,7 @@ func (cmd *AddAgentMysqldExporterCommand) RunCmd() (commands.Result, error) {
 			TLSKey:                    tlsKey,
 			TablestatsGroupTableLimit: cmd.TablestatsGroupTableLimit,
 			PushMetrics:               cmd.PushMetrics,
+			ExposeExporter:            cmd.ExposeExporter,
 			DisableCollectors:         commands.ParseDisableCollectors(cmd.DisableCollectors),
 			LogLevel:                  &cmd.LogLevel,
 		},

admin/commands/inventory/add_agent_node_exporter.go

@@ -46,6 +46,7 @@ type AddAgentNodeExporterCommand struct {
 	PMMAgentID        string            `arg:"" help:"The pmm-agent identifier which runs this instance"`
 	CustomLabels      map[string]string `mapsep:"," help:"Custom user-assigned labels"`
 	PushMetrics       bool              `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
+	ExposeExporter    bool              `help:"Expose the address of the exporter publicly on 0.0.0.0"`
 	DisableCollectors []string          `help:"Comma-separated list of collector names to exclude from exporter"`
 	LogLevel          string            `enum:"debug,info,warn,error" default:"warn" help:"Service logging level. One of: [debug, info, warn, error]"`
 }
@@ -57,6 +58,7 @@ func (cmd *AddAgentNodeExporterCommand) RunCmd() (commands.Result, error) {
 			PMMAgentID:        cmd.PMMAgentID,
 			CustomLabels:      customLabels,
 			PushMetrics:       cmd.PushMetrics,
+			ExposeExporter:    cmd.ExposeExporter,
 			DisableCollectors: commands.ParseDisableCollectors(cmd.DisableCollectors),
 			LogLevel:          &cmd.LogLevel,
 		},

admin/commands/inventory/add_agent_postgres_exporter.go

@@ -46,6 +46,8 @@ func (res *addAgentPostgresExporterResult) String() string {
 }
 
 // AddAgentPostgresExporterCommand is used by Kong for CLI flags and commands.
+//
+//nolint:lll
 type AddAgentPostgresExporterCommand struct {
 	PMMAgentID          string            `arg:"" help:"The pmm-agent identifier which runs this instance"`
 	ServiceID           string            `arg:"" help:"Service identifier"`
@@ -55,13 +57,15 @@ type AddAgentPostgresExporterCommand struct {
 	CustomLabels        map[string]string `mapsep:"," help:"Custom user-assigned labels"`
 	SkipConnectionCheck bool              `help:"Skip connection check"`
 	PushMetrics         bool              `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
+	ExposeExporter      bool              `help:"Expose the address of the exporter publicly on 0.0.0.0"`
 	DisableCollectors   []string          `help:"Comma-separated list of collector names to exclude from exporter"`
 	TLS                 bool              `help:"Use TLS to connect to the database"`
 	TLSSkipVerify       bool              `help:"Skip TLS certificates validation"`
 	TLSCAFile           string            `help:"TLS CA certificate file"`
 	TLSCertFile         string            `help:"TLS certificate file"`
 	TLSKeyFile          string            `help:"TLS certificate key file"`
 	LogLevel            string            `enum:"debug,info,warn,error" default:"warn" help:"Service logging level. One of: [debug, info, warn, error]"`
+	AutoDiscoveryLimit  int32             `default:"0" placeholder:"NUMBER" help:"Auto-discovery will be disabled if there are more than that number of databases (default: server-defined, -1: always disabled)"`
 }
 
 func (cmd *AddAgentPostgresExporterCommand) RunCmd() (commands.Result, error) {
@@ -98,7 +102,9 @@ func (cmd *AddAgentPostgresExporterCommand) RunCmd() (commands.Result, error) {
 			CustomLabels:        customLabels,
 			SkipConnectionCheck: cmd.SkipConnectionCheck,
 			PushMetrics:         cmd.PushMetrics,
+			ExposeExporter:      cmd.ExposeExporter,
 			DisableCollectors:   commands.ParseDisableCollectors(cmd.DisableCollectors),
+			AutoDiscoveryLimit:  cmd.AutoDiscoveryLimit,
 
 			TLS:           cmd.TLS,
 			TLSSkipVerify: cmd.TLSSkipVerify,

admin/commands/inventory/add_agent_proxysql_exporter.go

@@ -57,6 +57,7 @@ type AddAgentProxysqlExporterCommand struct {
 	TLS                 bool              `help:"Use TLS to connect to the database"`
 	TLSSkipVerify       bool              `help:"Skip TLS certificates validation"`
 	PushMetrics         bool              `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
+	ExposeExporter      bool              `help:"Expose the address of the exporter publicly on 0.0.0.0"`
 	DisableCollectors   []string          `help:"Comma-separated list of collector names to exclude from exporter"`
 	LogLevel            string            `enum:"debug,info,warn,error,fatal" default:"warn" help:"Service logging level. One of: [debug, info, warn, error, fatal]"`
 }
@@ -75,6 +76,7 @@ func (cmd *AddAgentProxysqlExporterCommand) RunCmd() (commands.Result, error) {
 			TLS:                 cmd.TLS,
 			TLSSkipVerify:       cmd.TLSSkipVerify,
 			PushMetrics:         cmd.PushMetrics,
+			ExposeExporter:      cmd.ExposeExporter,
 			DisableCollectors:   commands.ParseDisableCollectors(cmd.DisableCollectors),
 			LogLevel:            &cmd.LogLevel,
 		},

admin/commands/management/add_mongodb.go

@@ -82,6 +82,7 @@ type AddMongoDBCommand struct {
 	DisableCollectors             []string          `help:"Comma-separated list of collector names to exclude from exporter"`
 	StatsCollections              []string          `help:"Collections for collstats & indexstats"`
 	CollectionsLimit              int32             `name:"max-collections-limit" default:"-1" help:"Disable collstats, dbstats, topmetrics and indexstats if there are more than <n> collections. 0: No limit. Default is -1, which let PMM automatically set this value"`
+	ExposeExporter                bool              `name:"expose-exporter" help:"Optionally expose the address of the exporter publicly on 0.0.0.0"`
 
 	AddCommonFlags
 	AddLogLevelFatalFlags
@@ -157,8 +158,9 @@ func (cmd *AddMongoDBCommand) RunCmd() (commands.Result, error) {
 			NodeID:         cmd.NodeID,
 			ServiceName:    serviceName,
 			Address:        host,
-			Port:           int64(port),
 			Socket:         socket,
+			Port:           int64(port),
+			ExposeExporter: cmd.ExposeExporter,
 			PMMAgentID:     cmd.PMMAgentID,
 			Environment:    cmd.Environment,
 			Cluster:        cmd.Cluster,

admin/commands/management/add_mysql.go

@@ -71,7 +71,7 @@ func (res *addMySQLResult) TablestatStatus() string {
 	s := "Table statistics collection " + status
 
 	switch {
-	case res.MysqldExporter.TablestatsGroupTableLimit == 0: // no limit
+	case res.MysqldExporter.TablestatsGroupTableLimit == 0: // server defined
 		s += " (the table count limit is not set)."
 	case res.MysqldExporter.TablestatsGroupTableLimit < 0: // always disabled
 		s += " (always)."
@@ -120,6 +120,7 @@ type AddMySQLCommand struct {
 	CreateUser             bool              `hidden:"" help:"Create pmm user"`
 	MetricsMode            string            `enum:"${metricsModesEnum}" default:"auto" help:"Metrics flow mode, can be push - agent will push metrics, pull - server scrape metrics from agent or auto - chosen by server"`
 	DisableCollectors      []string          `help:"Comma-separated list of collector names to exclude from exporter"`
+	ExposeExporter         bool              `name:"expose-exporter" help:"Optionally expose the address of the exporter publicly on 0.0.0.0"`
 
 	AddCommonFlags
 	AddLogLevelNoFatalFlags
@@ -209,6 +210,7 @@ func (cmd *AddMySQLCommand) RunCmd() (commands.Result, error) {
 			Address:        host,
 			Socket:         socket,
 			Port:           int64(port),
+			ExposeExporter: cmd.ExposeExporter,
 			PMMAgentID:     cmd.PMMAgentID,
 			Environment:    cmd.Environment,
 			Cluster:        cmd.Cluster,

admin/commands/management/add_postgresql.go

@@ -73,6 +73,8 @@ type AddPostgreSQLCommand struct {
 	DisableQueryExamples bool              `name:"disable-queryexamples" help:"Disable collection of query examples"`
 	MetricsMode          string            `enum:"${metricsModesEnum}" default:"auto" help:"Metrics flow mode, can be push - agent will push metrics, pull - server scrape metrics from agent or auto - chosen by server"`
 	DisableCollectors    []string          `help:"Comma-separated list of collector names to exclude from exporter"`
+	ExposeExporter       bool              `name:"expose-exporter" help:"Optionally expose the address of the exporter publicly on 0.0.0.0"`
+	AutoDiscoveryLimit   int32             `default:"0" placeholder:"NUMBER" help:"Auto-discovery will be disabled if there are more than that number of databases (default: server-defined, -1: always disabled)"`
 
 	AddCommonFlags
 	AddLogLevelNoFatalFlags
@@ -171,16 +173,16 @@ func (cmd *AddPostgreSQLCommand) RunCmd() (commands.Result, error) {
 
 	params := &postgresql.AddPostgreSQLParams{
 		Body: postgresql.AddPostgreSQLBody{
-			NodeID:      cmd.NodeID,
-			ServiceName: serviceName,
-
+			NodeID:                 cmd.NodeID,
+			ServiceName:            serviceName,
 			Address:                host,
+			Socket:                 socket,
 			Port:                   int64(port),
+			ExposeExporter:         cmd.ExposeExporter,
 			Username:               cmd.Username,
 			Password:               cmd.Password,
 			Database:               cmd.Database,
 			AgentPassword:          cmd.AgentPassword,
-			Socket:                 socket,
 			SkipConnectionCheck:    cmd.SkipConnectionCheck,
 			DisableCommentsParsing: disableCommentsParsing,
 
@@ -203,6 +205,7 @@ func (cmd *AddPostgreSQLCommand) RunCmd() (commands.Result, error) {
 			DisableQueryExamples: cmd.DisableQueryExamples,
 			MetricsMode:          pointer.ToString(strings.ToUpper(cmd.MetricsMode)),
 			DisableCollectors:    commands.ParseDisableCollectors(cmd.DisableCollectors),
+			AutoDiscoveryLimit:   cmd.AutoDiscoveryLimit,
 			LogLevel:             &cmd.AddLogLevel,
 		},
 		Context: commands.Ctx,

admin/commands/management/add_proxysql.go

@@ -64,6 +64,7 @@ type AddProxySQLCommand struct {
 	TLSSkipVerify       bool              `help:"Skip TLS certificates validation"`
 	MetricsMode         string            `enum:"${metricsModesEnum}" default:"auto" help:"Metrics flow mode, can be push - agent will push metrics, pull - server scrape metrics from agent or auto - chosen by server"`
 	DisableCollectors   []string          `help:"Comma-separated list of collector names to exclude from exporter"`
+	ExposeExporter      bool              `name:"expose-exporter" help:"Optionally expose the address of the exporter publicly on 0.0.0.0"`
 
 	AddCommonFlags
 	AddLogLevelFatalFlags
@@ -130,8 +131,9 @@ func (cmd *AddProxySQLCommand) RunCmd() (commands.Result, error) {
 			NodeID:         cmd.NodeID,
 			ServiceName:    serviceName,
 			Address:        host,
-			Port:           int64(port),
 			Socket:         socket,
+			Port:           int64(port),
+			ExposeExporter: cmd.ExposeExporter,
 			PMMAgentID:     cmd.PMMAgentID,
 			Environment:    cmd.Environment,
 			Cluster:        cmd.Cluster,

admin/commands/management/add_test.go

@@ -122,11 +122,11 @@ func TestManagementGlobalFlags(t *testing.T) {
 				},
 			}
 
-			serviceName, socket, address, port, err := processGlobalAddFlagsWithSocket(cmd, cmd.AddCommonFlags)
+			serviceName, socket, host, port, err := processGlobalAddFlagsWithSocket(cmd, cmd.AddCommonFlags)
 
 			assert.NoError(t, err)
 			assert.Equal(t, serviceName, test.wantServiceName)
-			assert.Equal(t, address, test.wantHost)
+			assert.Equal(t, host, test.wantHost)
 			assert.Equal(t, int(port), int(test.wantPort))
 			assert.Equal(t, socket, test.wantSocket)
 		})

admin/commands/management/register.go

@@ -28,12 +28,16 @@ var registerResultT = commands.ParseTemplate(`
 pmm-agent registered.
 pmm-agent ID: {{ .PMMAgent.AgentID }}
 Node ID     : {{ .PMMAgent.RunsOnNodeID }}
+{{ if .Warning }}
+Warning: {{ .Warning }}
+{{- end -}}
 `)
 
 type registerResult struct {
 	GenericNode   *node.RegisterNodeOKBodyGenericNode   `json:"generic_node"`
 	ContainerNode *node.RegisterNodeOKBodyContainerNode `json:"container_node"`
 	PMMAgent      *node.RegisterNodeOKBodyPMMAgent      `json:"pmm_agent"`
+	Warning       string                                `json:"warning"`
 }
 
 func (res *registerResult) Result() {}
@@ -96,5 +100,6 @@ func (cmd *RegisterCommand) RunCmd() (commands.Result, error) {
 		GenericNode:   resp.Payload.GenericNode,
 		ContainerNode: resp.Payload.ContainerNode,
 		PMMAgent:      resp.Payload.PMMAgent,
+		Warning:       resp.Payload.Warning,
 	}, nil
 }

admin/commands/management/register_test.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package management
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/percona/pmm/api/managementpb/json/client/node"
+)
+
+func TestRegisterResult(t *testing.T) {
+	tests := []struct {
+		name   string
+		result registerResult
+		want   string
+	}{
+		{
+			name: "Success",
+			result: registerResult{
+				PMMAgent: &node.RegisterNodeOKBodyPMMAgent{
+					AgentID:      "/agent_id/new_id",
+					RunsOnNodeID: "/node_id/second_id",
+				},
+				Warning: "",
+			},
+			want: `pmm-agent registered.
+pmm-agent ID: /agent_id/new_id
+Node ID     : /node_id/second_id
+`,
+		},
+		{
+			name: "With warning",
+			result: registerResult{
+				PMMAgent: &node.RegisterNodeOKBodyPMMAgent{
+					AgentID:      "/agent_id/warning",
+					RunsOnNodeID: "/node_id/warning_node",
+				},
+				Warning: "Couldn't create Admin API Key",
+			},
+			want: `pmm-agent registered.
+pmm-agent ID: /agent_id/warning
+Node ID     : /node_id/warning_node
+
+Warning: Couldn't create Admin API Key
+`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, tt.result.String(), "String()")
+		})
+	}
+}

agent/agents/mysql/perfschema/perfschema_test.go

@@ -304,8 +304,8 @@ func TestPerfSchema(t *testing.T) {
 
 	case "10.4-mariadb":
 		digests = map[string]string{
-			"SELECT `sleep` (?)":   "ce5b40e78030bb319c84965637255c18",
-			"SELECT * FROM `city`": "978a3813c9f566d7a72d65b88a9149d9",
+			"SELECT `sleep` (?)":   "0a01e0e8325cdd1db9a0746270ab8ce9",
+			"SELECT * FROM `city`": "a65e76b1643273fa3206b11c4f4d8739",
 		}
 
 	default:

agent/agents/postgres/pgstatstatements/pgstatstatements.go

@@ -236,10 +236,9 @@ func (m *PGStatStatementsQAN) getStatStatementsExtended(
 	databases := queryDatabases(q)
 	usernames := queryUsernames(q)
 
-	rows, e := rowsByVersion(q, "WHERE queryid IS NOT NULL AND query IS NOT NULL")
-	if e != nil {
-		err = e
-		return nil, nil, err
+	rows, err := rowsByVersion(q, "WHERE queryid IS NOT NULL AND query IS NOT NULL")
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "couldn't get rows from pg_stat_statements")
 	}
 	defer rows.Close() //nolint:errcheck
 

agent/commands/clients.go

@@ -197,6 +197,7 @@ func serverRegister(cfgSetup *config.Setup) (agentID, token string, _ error) { /
 			Reregister:        cfgSetup.Force,
 			MetricsMode:       pointer.ToString(strings.ToUpper(cfgSetup.MetricsMode)),
 			DisableCollectors: disableCollectors,
+			ExposeExporter:    cfgSetup.ExposeExporter,
 		},
 		Context: context.Background(),
 	})

agent/config/config.go

@@ -139,6 +139,7 @@ type Setup struct {
 
 	Force            bool
 	SkipRegistration bool
+	ExposeExporter   bool
 }
 
 // Cache represent cache settings.
@@ -512,6 +513,8 @@ func Application(cfg *Config) (*kingpin.Application, *string) {
 		Envar("PMM_AGENT_SETUP_CUSTOM_LABELS").StringVar(&cfg.Setup.CustomLabels)
 	setupCmd.Flag("agent-password", "Custom password for /metrics endpoint [PMM_AGENT_SETUP_NODE_PASSWORD]").
 		Envar("PMM_AGENT_SETUP_NODE_PASSWORD").StringVar(&cfg.Setup.AgentPassword)
+	setupCmd.Flag("expose-exporter", "Expose the address of the agent's node-exporter publicly on 0.0.0.0").
+		Envar("PMM_AGENT_EXPOSE_EXPORTER").BoolVar(&cfg.Setup.ExposeExporter)
 
 	return app, configFileF
 }

agent/serviceinfobroker/service_info_broker.go

@@ -219,12 +219,32 @@ func (sib *ServiceInfoBroker) getPostgreSQLInfo(ctx context.Context, dsn string,
 	db := sql.OpenDB(c)
 	defer db.Close() //nolint:errcheck
 
+	var databaseList []string
+	databaseListQuery := "SELECT /* agent='serviceinfobroker' */ datname FROM pg_database WHERE datallowconn = true AND datistemplate = false AND has_database_privilege(current_user, datname, 'connect')" //nolint:lll
+	rows, err := db.QueryContext(ctx, databaseListQuery)
+	if err != nil {
+		res.Error = err.Error()
+		return &res
+	}
+	defer rows.Close() //nolint:errcheck
+	for rows.Next() {
+		var databaseName string
+		err := rows.Scan(&databaseName)
+		if err != nil {
+			res.Error = err.Error()
+			return &res
+		}
+
+		databaseList = append(databaseList, databaseName)
+	}
+	res.DatabaseList = databaseList
+
 	var version string
 	if err = db.QueryRowContext(ctx, "SHOW /* agent='serviceinfobroker' */ SERVER_VERSION").Scan(&version); err != nil {
 		res.Error = err.Error()
 	}
-
 	res.Version = version
+
 	return &res
 }