From 9d919dfd2f5e7f55d1264ab6524cbc05ffe0d5f3 Mon Sep 17 00:00:00 2001 From: Talha Bin Rizwan Date: Wed, 27 Sep 2023 21:51:02 +0500 Subject: [PATCH 1/8] PMM-7 Fix Clickhouse repository. (#2506) --- .../playbook/tasks/roles/clickhouse/tasks/main.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml b/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml index 2fb89b954b..5f94d66fdd 100644 --- a/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml @@ -13,7 +13,10 @@ register: supervisord_socket - name: Stop and remove clickhouse before update | EL9 - when: supervisord_socket.stat.exists and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - supervisord_socket.stat.exists + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' command: /usr/local/bin/supervisorctl {{ item }} clickhouse become: true loop: @@ -61,7 +64,7 @@ name: clickhouse file: clickhouse description: "Clickhouse repo" - baseurl: "https://repo.clickhouse.com/rpm/stable/x86_64/" + baseurl: "https://packages.clickhouse.com/rpm/stable/" enabled: no gpgcheck: 1 gpgkey: "https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG" @@ -74,7 +77,7 @@ name: clickhouse file: clickhouse description: "Clickhouse repo" - baseurl: "https://repo.clickhouse.com/rpm/stable/x86_64/" + baseurl: "https://packages.clickhouse.com/rpm/stable/" enabled: no gpgcheck: 0 From 29fbba7984b26fb68ecfe65e44c1c6b761c14e28 Mon Sep 17 00:00:00 2001 From: Alex Tymchuk Date: Thu, 28 Sep 2023 20:32:30 +0300 Subject: [PATCH 2/8] PMM-12547 Revert PMM-11670 Set the cluster label if empty (#2508) * Revert "PMM-11670 Set the cluster label if empty (#2459)" This reverts commit 3c58ed956b5ae6b19a621d7d4cd12087051abd62. * PMM-12547 use a different IP * PMM-12547 fix the debug command in actions --- api-tests/inventory/nodes_test.go | 5 ++- api-tests/inventory/services_test.go | 12 ------- api-tests/management/external_test.go | 2 -- api-tests/management/haproxy_test.go | 2 -- api-tests/management/mongodb_test.go | 7 ---- api-tests/management/mysql_test.go | 6 ---- api-tests/management/postgresql_test.go | 7 ---- api-tests/management/proxysql_test.go | 16 ++++----- managed/models/database.go | 3 -- managed/models/service_helpers_test.go | 33 +++++++------------ managed/models/service_model.go | 6 ---- managed/services/inventory/services_test.go | 10 ------ managed/services/qan/client_test.go | 4 --- .../victoriametrics/victoriametrics_test.go | 12 ------- 14 files changed, 21 insertions(+), 104 deletions(-) diff --git a/api-tests/inventory/nodes_test.go b/api-tests/inventory/nodes_test.go index fe7e3142ec..6cd3d5c466 100644 --- a/api-tests/inventory/nodes_test.go +++ b/api-tests/inventory/nodes_test.go @@ -353,7 +353,7 @@ func TestContainerNodeDeprecated(t *testing.T) { ContainerID: "docker-id", ContainerName: "docker-name", MachineID: "machine-id", - Address: "10.10.1.10", + Address: "10.10.1.11", }, Context: pmmapitests.Context, } @@ -377,7 +377,7 @@ func TestContainerNodeDeprecated(t *testing.T) { ContainerID: "docker-id", ContainerName: "docker-name", MachineID: "machine-id", - Address: "10.10.1.10", + Address: "10.10.1.11", }, }, } @@ -688,7 +688,6 @@ func TestRemoveNode(t *testing.T) { Address: "localhost", Port: 3306, ServiceName: serviceName, - Cluster: serviceName, }, }, }, listAgentsOK.Payload) diff --git a/api-tests/inventory/services_test.go b/api-tests/inventory/services_test.go index d019fb0f46..ae0ae00521 100644 --- a/api-tests/inventory/services_test.go +++ b/api-tests/inventory/services_test.go @@ -352,7 +352,6 @@ func TestMySQLService(t *testing.T) { Address: "localhost", Port: 3306, ServiceName: serviceName, - Cluster: serviceName, }, }, }, res) @@ -373,7 +372,6 @@ func TestMySQLService(t *testing.T) { Address: "localhost", Port: 3306, ServiceName: serviceName, - Cluster: serviceName, }, }, }, serviceRes) @@ -555,7 +553,6 @@ func TestMongoDBService(t *testing.T) { ServiceID: serviceID, NodeID: genericNodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "localhost", Port: 27017, }, @@ -576,7 +573,6 @@ func TestMongoDBService(t *testing.T) { ServiceID: serviceID, NodeID: genericNodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "localhost", Port: 27017, }, @@ -733,7 +729,6 @@ func TestMongoDBService(t *testing.T) { ServiceID: serviceID, NodeID: genericNodeID, ServiceName: serviceName, - Cluster: serviceName, Socket: "/tmp/mongodb-27017.sock", }, }, @@ -775,7 +770,6 @@ func TestPostgreSQLService(t *testing.T) { DatabaseName: defaultPostgresDBName, Port: 5432, ServiceName: serviceName, - Cluster: serviceName, }, }, }, res) @@ -797,7 +791,6 @@ func TestPostgreSQLService(t *testing.T) { DatabaseName: defaultPostgresDBName, Port: 5432, ServiceName: serviceName, - Cluster: serviceName, }, }, }, serviceRes) @@ -980,7 +973,6 @@ func TestProxySQLService(t *testing.T) { Address: "localhost", Port: 5432, ServiceName: serviceName, - Cluster: serviceName, }, }, }, res) @@ -1001,7 +993,6 @@ func TestProxySQLService(t *testing.T) { Address: "localhost", Port: 5432, ServiceName: serviceName, - Cluster: serviceName, }, }, }, serviceRes) @@ -1193,7 +1184,6 @@ func TestExternalService(t *testing.T) { ServiceID: serviceID, NodeID: genericNodeID, ServiceName: serviceName, - Cluster: serviceName, Group: "redis", }, }, @@ -1213,7 +1203,6 @@ func TestExternalService(t *testing.T) { ServiceID: serviceID, NodeID: genericNodeID, ServiceName: serviceName, - Cluster: serviceName, Group: "redis", }, }, @@ -1345,7 +1334,6 @@ func TestExternalService(t *testing.T) { ServiceID: serviceID, NodeID: genericNodeID, ServiceName: serviceName, - Cluster: serviceName, Group: "external", }, }, diff --git a/api-tests/management/external_test.go b/api-tests/management/external_test.go index c044ee654b..289466ff0f 100644 --- a/api-tests/management/external_test.go +++ b/api-tests/management/external_test.go @@ -74,7 +74,6 @@ func TestAddExternal(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Group: "external", }, }, *serviceOK.Payload) @@ -226,7 +225,6 @@ func TestAddExternal(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Group: "external", }, }, *serviceOK.Payload) diff --git a/api-tests/management/haproxy_test.go b/api-tests/management/haproxy_test.go index ceaa2bc147..7b25a0103e 100644 --- a/api-tests/management/haproxy_test.go +++ b/api-tests/management/haproxy_test.go @@ -76,7 +76,6 @@ func TestAddHAProxy(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, }, }, *serviceOK.Payload) @@ -226,7 +225,6 @@ func TestAddHAProxy(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, }, }, *serviceOK.Payload) diff --git a/api-tests/management/mongodb_test.go b/api-tests/management/mongodb_test.go index 75c6623fb9..17472529af 100644 --- a/api-tests/management/mongodb_test.go +++ b/api-tests/management/mongodb_test.go @@ -79,7 +79,6 @@ func TestAddMongoDB(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 27017, }, @@ -155,7 +154,6 @@ func TestAddMongoDB(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 27017, }, @@ -391,7 +389,6 @@ func TestAddMongoDB(t *testing.T) { ServiceID: serviceID, NodeID: newNodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 27017, }, @@ -610,7 +607,6 @@ func TestAddMongoDB(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Socket: "/tmp/mongodb-27017.sock", }, }, *serviceOK.Payload) @@ -682,7 +678,6 @@ func TestAddMongoDB(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 27017, }, @@ -755,7 +750,6 @@ func TestAddMongoDB(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 27017, }, @@ -827,7 +821,6 @@ func TestAddMongoDB(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 27017, }, diff --git a/api-tests/management/mysql_test.go b/api-tests/management/mysql_test.go index 5f7490183c..a80f8dc3e3 100644 --- a/api-tests/management/mysql_test.go +++ b/api-tests/management/mysql_test.go @@ -80,7 +80,6 @@ func TestAddMySQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 3306, }, @@ -160,7 +159,6 @@ func TestAddMySQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 3306, }, @@ -415,7 +413,6 @@ func TestAddMySQL(t *testing.T) { ServiceID: serviceID, NodeID: newNodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 27017, }, @@ -669,7 +666,6 @@ func TestAddMySQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 3306, }, @@ -745,7 +741,6 @@ func TestAddMySQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 3306, }, @@ -820,7 +815,6 @@ func TestAddMySQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 3306, }, diff --git a/api-tests/management/postgresql_test.go b/api-tests/management/postgresql_test.go index 9e575730f2..4b7876cd94 100644 --- a/api-tests/management/postgresql_test.go +++ b/api-tests/management/postgresql_test.go @@ -82,7 +82,6 @@ func TestAddPostgreSQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, DatabaseName: defaultPostgresDBName, Address: "10.10.10.10", Port: 5432, @@ -162,7 +161,6 @@ func TestAddPostgreSQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, DatabaseName: defaultPostgresDBName, Address: "10.10.10.10", Port: 5432, @@ -263,7 +261,6 @@ func TestAddPostgreSQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, DatabaseName: defaultPostgresDBName, Address: "10.10.10.10", Port: 5432, @@ -413,7 +410,6 @@ func TestAddPostgreSQL(t *testing.T) { ServiceID: serviceID, NodeID: newNodeID, ServiceName: serviceName, - Cluster: serviceName, DatabaseName: defaultPostgresDBName, Address: "10.10.10.10", Port: 27017, @@ -640,7 +636,6 @@ func TestAddPostgreSQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, DatabaseName: defaultPostgresDBName, Address: "10.10.10.10", Port: 5432, @@ -716,7 +711,6 @@ func TestAddPostgreSQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, DatabaseName: defaultPostgresDBName, Address: "10.10.10.10", Port: 5432, @@ -791,7 +785,6 @@ func TestAddPostgreSQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, DatabaseName: defaultPostgresDBName, Address: "10.10.10.10", Port: 5432, diff --git a/api-tests/management/proxysql_test.go b/api-tests/management/proxysql_test.go index 907987abe9..e7bc9cd594 100644 --- a/api-tests/management/proxysql_test.go +++ b/api-tests/management/proxysql_test.go @@ -80,7 +80,6 @@ func TestAddProxySQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 3306, }, @@ -156,7 +155,6 @@ func TestAddProxySQL(t *testing.T) { ServiceID: serviceID, NodeID: nodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 3306, }, @@ -262,12 +260,13 @@ func TestAddProxySQL(t *testing.T) { params := &proxysql.AddProxySQLParams{ Context: pmmapitests.Context, Body: proxysql.AddProxySQLBody{ - NodeID: nodeID, - PMMAgentID: pmmAgentID, - ServiceName: serviceName, - Address: "10.10.10.10", - Port: 3306, - Username: "username", + NodeID: nodeID, + PMMAgentID: pmmAgentID, + ServiceName: serviceName, + Address: "10.10.10.10", + Port: 3306, + Username: "username", + SkipConnectionCheck: true, }, } @@ -387,7 +386,6 @@ func TestAddProxySQL(t *testing.T) { ServiceID: serviceID, NodeID: newNodeID, ServiceName: serviceName, - Cluster: serviceName, Address: "10.10.10.10", Port: 27017, }, diff --git a/managed/models/database.go b/managed/models/database.go index 189037f922..0ce9add0ca 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -915,9 +915,6 @@ var databaseSchema = [][]string{ `ALTER TABLE agents ALTER COLUMN comments_parsing_disabled DROP DEFAULT`, }, - 85: { - `UPDATE services SET cluster = service_name WHERE cluster = ''`, - }, } // ^^^ Avoid default values in schema definition. ^^^ diff --git a/managed/models/service_helpers_test.go b/managed/models/service_helpers_test.go index 47d4dc317c..c3076809b2 100644 --- a/managed/models/service_helpers_test.go +++ b/managed/models/service_helpers_test.go @@ -158,11 +158,10 @@ func TestServiceHelpers(t *testing.T) { services, err = models.FindServices(q, models.ServiceFilters{NodeID: "N1"}) assert.NoError(t, err) assert.Equal(t, 3, len(services)) - assert.Equal(t, []*models.Service{{ + assert.Equal(t, services, []*models.Service{{ ServiceID: "S1", ServiceType: models.MongoDBServiceType, ServiceName: "Service without Agents", - Cluster: "Service without Agents", NodeID: "N1", Address: pointer.ToString("127.0.0.1"), Port: pointer.ToUint16OrNil(27017), @@ -172,7 +171,6 @@ func TestServiceHelpers(t *testing.T) { ServiceID: "S2", ServiceType: models.MySQLServiceType, ServiceName: "Service with Agents", - Cluster: "Service with Agents", NodeID: "N1", Address: pointer.ToString("127.0.0.1"), Port: pointer.ToUint16OrNil(3306), @@ -182,38 +180,35 @@ func TestServiceHelpers(t *testing.T) { ServiceID: "S5", ServiceType: models.ProxySQLServiceType, ServiceName: "Fifth service", - Cluster: "Fifth service", NodeID: "N1", Address: pointer.ToString("127.0.0.1"), Port: pointer.ToUint16OrNil(6032), CreatedAt: now, UpdatedAt: now, - }}, services) + }}) services, err = models.FindServices(q, models.ServiceFilters{NodeID: "N1", ServiceType: pointerToServiceType(models.MySQLServiceType)}) assert.NoError(t, err) assert.Equal(t, 1, len(services)) - assert.Equal(t, []*models.Service{{ + assert.Equal(t, services, []*models.Service{{ ServiceID: "S2", ServiceType: models.MySQLServiceType, ServiceName: "Service with Agents", - Cluster: "Service with Agents", NodeID: "N1", Address: pointer.ToString("127.0.0.1"), Port: pointer.ToUint16OrNil(3306), CreatedAt: now, UpdatedAt: now, - }}, services) + }}) services, err = models.FindServices(q, models.ServiceFilters{NodeID: "N2", ServiceType: pointerToServiceType(models.ExternalServiceType)}) assert.NoError(t, err) assert.Equal(t, 2, len(services)) - assert.Equal(t, []*models.Service{ + assert.Equal(t, services, []*models.Service{ { ServiceID: "S4", ServiceType: models.ExternalServiceType, ServiceName: "Fourth service", - Cluster: "Fourth service", ExternalGroup: "external", NodeID: "N2", CreatedAt: now, @@ -223,7 +218,6 @@ func TestServiceHelpers(t *testing.T) { ServiceID: "S7", ServiceType: models.ExternalServiceType, ServiceName: "Seventh service", - Cluster: "Seventh service", NodeID: "N2", Address: pointer.ToString("127.0.0.1"), Port: pointer.ToUint16OrNil(6379), @@ -231,50 +225,47 @@ func TestServiceHelpers(t *testing.T) { CreatedAt: now, UpdatedAt: now, }, - }, services) + }) services, err = models.FindServices(q, models.ServiceFilters{NodeID: "N2", ServiceType: pointerToServiceType(models.ProxySQLServiceType)}) assert.NoError(t, err) assert.Equal(t, 1, len(services)) - assert.Equal(t, []*models.Service{{ + assert.Equal(t, services, []*models.Service{{ ServiceID: "S6", ServiceType: models.ProxySQLServiceType, ServiceName: "Sixth service", - Cluster: "Sixth service", Socket: pointer.ToStringOrNil("/tmp/proxysql_admin.sock"), NodeID: "N2", CreatedAt: now, UpdatedAt: now, - }}, services) + }}) services, err = models.FindServices(q, models.ServiceFilters{ExternalGroup: "redis"}) assert.NoError(t, err) assert.Equal(t, 1, len(services)) - assert.Equal(t, []*models.Service{{ + assert.Equal(t, services, []*models.Service{{ ServiceID: "S7", ServiceType: models.ExternalServiceType, ServiceName: "Seventh service", - Cluster: "Seventh service", NodeID: "N2", Address: pointer.ToString("127.0.0.1"), Port: pointer.ToUint16OrNil(6379), ExternalGroup: "redis", CreatedAt: now, UpdatedAt: now, - }}, services) + }}) services, err = models.FindServices(q, models.ServiceFilters{NodeID: "N2", ServiceType: pointerToServiceType(models.HAProxyServiceType)}) assert.NoError(t, err) assert.Equal(t, 1, len(services)) - assert.Equal(t, []*models.Service{{ + assert.Equal(t, services, []*models.Service{{ ServiceID: "S8", ServiceType: models.HAProxyServiceType, ServiceName: "Eighth service", - Cluster: "Eighth service", NodeID: "N2", CreatedAt: now, UpdatedAt: now, - }}, services) + }}) }) t.Run("FindActiveServiceTypes", func(t *testing.T) { diff --git a/managed/models/service_model.go b/managed/models/service_model.go index 04c7484156..3842f9a808 100644 --- a/managed/models/service_model.go +++ b/managed/models/service_model.go @@ -74,9 +74,6 @@ func (s *Service) BeforeInsert() error { if len(s.CustomLabels) == 0 { s.CustomLabels = nil } - if s.Cluster == "" { - s.Cluster = s.ServiceName - } return nil } @@ -86,9 +83,6 @@ func (s *Service) BeforeUpdate() error { if len(s.CustomLabels) == 0 { s.CustomLabels = nil } - if s.Cluster == "" { - s.Cluster = s.ServiceName - } return nil } diff --git a/managed/services/inventory/services_test.go b/managed/services/inventory/services_test.go index 40f0d75d0b..3d091d7091 100644 --- a/managed/services/inventory/services_test.go +++ b/managed/services/inventory/services_test.go @@ -104,7 +104,6 @@ func TestServices(t *testing.T) { expectedService := &inventorypb.MySQLService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-mysql", - Cluster: "test-mysql", NodeId: models.PMMServerNodeID, Address: "127.0.0.1", Port: 3306, @@ -260,7 +259,6 @@ func TestServices(t *testing.T) { expectedService := &inventorypb.MySQLService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-mysql-socket", - Cluster: "test-mysql-socket", NodeId: models.PMMServerNodeID, Socket: "/var/run/mysqld/mysqld.sock", } @@ -335,7 +333,6 @@ func TestServices(t *testing.T) { expectedMongoDBService := &inventorypb.MongoDBService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-mongo", - Cluster: "test-mongo", NodeId: models.PMMServerNodeID, Address: "127.0.0.1", Port: 27017, @@ -377,7 +374,6 @@ func TestServices(t *testing.T) { expectedPostgreSQLService := &inventorypb.PostgreSQLService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-postgres", - Cluster: "test-postgres", DatabaseName: "postgres", NodeId: models.PMMServerNodeID, Address: "127.0.0.1", @@ -418,7 +414,6 @@ func TestServices(t *testing.T) { expectedPostgreSQLService := &inventorypb.PostgreSQLService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-postgres", - Cluster: "test-postgres", DatabaseName: "postgres", NodeId: models.PMMServerNodeID, Socket: "/var/run/postgresql", @@ -497,7 +492,6 @@ func TestServices(t *testing.T) { expectedProxySQLService := &inventorypb.ProxySQLService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-proxysql", - Cluster: "test-proxysql", NodeId: models.PMMServerNodeID, Address: "127.0.0.1", Port: 6033, @@ -537,7 +531,6 @@ func TestServices(t *testing.T) { expectedService := &inventorypb.ProxySQLService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-proxysql-socket", - Cluster: "test-proxysql-socket", NodeId: models.PMMServerNodeID, Socket: "/tmp/proxysql.sock", } @@ -609,7 +602,6 @@ func TestServices(t *testing.T) { expectedHAProxyService := &inventorypb.HAProxyService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-haproxy-service", - Cluster: "test-haproxy-service", NodeId: models.PMMServerNodeID, } assert.Equal(t, expectedHAProxyService, actualHAProxyService) @@ -647,7 +639,6 @@ func TestServices(t *testing.T) { expectedExternalService := &inventorypb.ExternalService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-external-service", - Cluster: "test-external-service", NodeId: models.PMMServerNodeID, Group: "external", } @@ -739,7 +730,6 @@ func TestServices(t *testing.T) { expectedService := &inventorypb.MongoDBService{ ServiceId: "/service_id/00000000-0000-4000-8000-000000000005", ServiceName: "test-mongodb-socket", - Cluster: "test-mongodb-socket", NodeId: models.PMMServerNodeID, Socket: "/tmp/mongodb-27017.sock", } diff --git a/managed/services/qan/client_test.go b/managed/services/qan/client_test.go index dce12fce95..700840abd2 100644 --- a/managed/services/qan/client_test.go +++ b/managed/services/qan/client_test.go @@ -185,7 +185,6 @@ func TestClient(t *testing.T) { MQueryTimeSum: 1234, ServiceId: "/service_id/014647c3-b2f5-44eb-94f4-d943260a968c", ServiceName: "test-mysql", - Cluster: "test-mysql", ServiceType: "mysql", NodeId: "/node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d", NodeName: "test-generic-node", @@ -255,7 +254,6 @@ func TestClient(t *testing.T) { NumQueries: 1, ServiceId: "/service_id/1fce2502-ecc7-46d4-968b-18d7907f2543", ServiceName: "test-mongodb", - Cluster: "test-mongodb", ServiceType: "mongodb", NodeId: "/node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d", NodeName: "test-generic-node", @@ -354,7 +352,6 @@ func TestClient(t *testing.T) { MQueryTimeCnt: 1, MQueryTimeSum: 55, ServiceName: "test-postgresql", - Cluster: "test-postgresql", ServiceType: "postgresql", ServiceId: "/service_id/9cffbdd4-3cd2-47f8-a5f9-a749c3d5fee1", NodeId: "/node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d", @@ -495,7 +492,6 @@ func TestClientPerformance(t *testing.T) { expectedBuckets[i] = &qanpb.MetricsBucket{ Queryid: fmt.Sprintf("bucket %d", i), ServiceName: "test-mysql", - Cluster: "test-mysql", NodeId: "pmm-server", NodeName: "pmm-server", NodeType: "generic", diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index d6a7db56cb..413afcf5f2 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -337,7 +337,6 @@ scrape_configs: _service_label: bam agent_id: /agent_id/cfec996c-4fe6-41d9-83cb-e1a3b1fe10a8 agent_type: mongodb_exporter - cluster: test-mongodb-noversion instance: /agent_id/cfec996c-4fe6-41d9-83cb-e1a3b1fe10a8 node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node @@ -368,7 +367,6 @@ scrape_configs: _service_label: bam agent_id: /agent_id/ecd8995a-d479-4b4d-bfb7-865bac4ac2fb agent_type: mongodb_exporter - cluster: test-mongodb instance: /agent_id/ecd8995a-d479-4b4d-bfb7-865bac4ac2fb node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node @@ -399,7 +397,6 @@ scrape_configs: _service_label: bam agent_id: /agent_id/ecd8995a-d479-4b4d-bfb7-865bac4ac2fb agent_type: mongodb_exporter - cluster: test-mongodb instance: /agent_id/ecd8995a-d479-4b4d-bfb7-865bac4ac2fb node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node @@ -432,7 +429,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd agent_type: mysqld_exporter - cluster: test-mysql instance: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node @@ -470,7 +466,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd agent_type: mysqld_exporter - cluster: test-mysql instance: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node @@ -514,7 +509,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd agent_type: mysqld_exporter - cluster: test-mysql instance: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node @@ -547,7 +541,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/f9ab9f7b-5e53-4952-a2e7-ff25fb90fe6a agent_type: mysqld_exporter - cluster: test-remote-mysql instance: /agent_id/f9ab9f7b-5e53-4952-a2e7-ff25fb90fe6a node_id: /node_id/4e2e07dc-40a1-18ca-aea9-d943260a9653 node_name: test-remote-node @@ -585,7 +578,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/f9ab9f7b-5e53-4952-a2e7-ff25fb90fe6a agent_type: mysqld_exporter - cluster: test-remote-mysql instance: /agent_id/f9ab9f7b-5e53-4952-a2e7-ff25fb90fe6a node_id: /node_id/4e2e07dc-40a1-18ca-aea9-d943260a9653 node_name: test-remote-node @@ -629,7 +621,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/f9ab9f7b-5e53-4952-a2e7-ff25fb90fe6a agent_type: mysqld_exporter - cluster: test-remote-mysql instance: /agent_id/f9ab9f7b-5e53-4952-a2e7-ff25fb90fe6a node_id: /node_id/4e2e07dc-40a1-18ca-aea9-d943260a9653 node_name: test-remote-node @@ -662,7 +653,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/29e14468-d479-4b4d-bfb7-4ac2fb865bac agent_type: postgres_exporter - cluster: test-postgresql instance: /agent_id/29e14468-d479-4b4d-bfb7-4ac2fb865bac node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node @@ -692,7 +682,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/29e14468-d479-4b4d-bfb7-4ac2fb865bac agent_type: postgres_exporter - cluster: test-postgresql instance: /agent_id/29e14468-d479-4b4d-bfb7-4ac2fb865bac node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node @@ -722,7 +711,6 @@ scrape_configs: _service_label: bar agent_id: /agent_id/29e14468-d479-4b4d-bfb7-4ac2fb865bac agent_type: postgres_exporter - cluster: test-postgresql instance: /agent_id/29e14468-d479-4b4d-bfb7-4ac2fb865bac node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d node_name: test-generic-node From 071946e2d84d285bc939c733ee6420fa1b857a72 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 11 Oct 2023 00:38:08 +0300 Subject: [PATCH 3/8] PMM-12562 Show error 503 instead of 500. --- update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf b/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf index dfdbfe8f11..48592e012c 100644 --- a/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf +++ b/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf @@ -62,6 +62,7 @@ error_page 503 @maintenance; location @maintenance { + auth_request off; root /usr/share/pmm-server/maintenance; rewrite ^(.*)$ /maintenance.html break; } From 79357f39d97384dc108d7be558578cb43c1363d5 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Tue, 26 Sep 2023 16:05:44 +0200 Subject: [PATCH 4/8] PMM-7 Fix datarace (#2501) * PMM-7 Fix datarace * PMM-7 Fix datarace --- managed/services/telemetry/telemetry.go | 6 +++--- managed/services/telemetry/telemetry_test.go | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/managed/services/telemetry/telemetry.go b/managed/services/telemetry/telemetry.go index 81f2450215..867f31f211 100644 --- a/managed/services/telemetry/telemetry.go +++ b/managed/services/telemetry/telemetry.go @@ -188,8 +188,8 @@ func (s *Service) processSendCh(ctx context.Context) { reportsBuf = []*pmmv1.ServerMetric{} reportsBufSync.Unlock() - go func() { - err := s.send(sendCtx, &reporter.ReportRequest{ + go func(ctx context.Context) { + err := s.send(ctx, &reporter.ReportRequest{ Metrics: reportsToSend, }) if err != nil { @@ -201,7 +201,7 @@ func (s *Service) processSendCh(ctx context.Context) { } s.l.Debug("Telemetry info sent.") - }() + }(sendCtx) } case <-ctx.Done(): if cancel != nil { diff --git a/managed/services/telemetry/telemetry_test.go b/managed/services/telemetry/telemetry_test.go index 842b6e9233..726e80cbaa 100644 --- a/managed/services/telemetry/telemetry_test.go +++ b/managed/services/telemetry/telemetry_test.go @@ -19,7 +19,6 @@ import ( "context" "io/fs" "os" - "reflect" "testing" "time" @@ -264,7 +263,7 @@ func initMockTelemetrySender(t *testing.T, expectedReport *reporter.ReportReques var mockTelemetrySender mockSender mockTelemetrySender.Test(t) mockTelemetrySender.On("SendTelemetry", - mock.AnythingOfType(reflect.TypeOf(context.TODO()).Name()), + mock.Anything, mock.MatchedBy(func(report *reporter.ReportRequest) bool { return matchExpectedReport(report, expectedReport) }), From 578bab30bd131b1e33c9f6408302cc7129590838 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 17 Oct 2023 00:14:21 +0300 Subject: [PATCH 5/8] PMM-12592 PMM-12587 fix SQLite to PostgreSQL (#2549) * PMM-7 More debug data. * PMM-12584 fix hex values in PostgreSQL DB. * PMM-12588 Outdated grafana-db-migrator used (#2547) * PMM-12584 fix provisioning problem. * PMM-12584 fix drop database issue. * PMM-12584 fix drop database issue for EL7. * PMM-12584 fix drop database issue for EL7. * PMM-12584 fix drop database issue for EL7. * Update update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml Co-authored-by: Alex Demidoff --------- Co-authored-by: Vadim Yalovets Co-authored-by: Alex Demidoff --- .../rpm/server/SPECS/grafana-db-migrator.spec | 9 ++-- build/scripts/build-server-rpm | 3 ++ .../tasks/roles/initialization/tasks/main.yml | 22 ++++++--- .../roles/sqlite-to-postgres/tasks/main.yml | 46 +++++++++++++++++++ 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/build/packages/rpm/server/SPECS/grafana-db-migrator.spec b/build/packages/rpm/server/SPECS/grafana-db-migrator.spec index 70de6bd075..c3f074740c 100644 --- a/build/packages/rpm/server/SPECS/grafana-db-migrator.spec +++ b/build/packages/rpm/server/SPECS/grafana-db-migrator.spec @@ -1,10 +1,10 @@ %undefine _missing_build_ids_terminate_build -%global release_hash 63e4bfdb7dd7162e238531c794f4ddef83173158 +%global release_hash 27b37c412e257bd8a5bcdbcaf52befa3fe555a9d Name: grafana-db-migrator -Version: 1.0.7 -Release: 1%{?dist} +Version: 1.0.8 +Release: 2%{?dist} Summary: A tool for Grafana database migration License: MIT URL: https://github.com/percona/grafana-db-migrator @@ -30,6 +30,9 @@ install -m 755 dist/grafana-db-migrator %{buildroot}%{_sbindir}/ %{_sbindir}/grafana-db-migrator %changelog +* Fri Oct 13 2023 Nurlan Moldomurov - 1.0.8-2 +- Fix issue with hexed values and folder fixes. + * Mon Feb 13 2023 Nikita Beletskii <2nikita.b@gmail.com> - 1.0.7-1 - Fix issue with convert_from() diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm index 0ceccebb23..58f619ffda 100755 --- a/build/scripts/build-server-rpm +++ b/build/scripts/build-server-rpm @@ -148,6 +148,9 @@ build() { s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \ || : fi + + cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir} + fi } diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index 2476b13d68..c3cc50d5f0 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -34,6 +34,11 @@ debug: msg: "Current version: {{ pmm_current_version }} Image Version: {{ pmm_image_version }}" +# We use current_version_file['failed'] because we don't want to run this on creating container +# and we use pmm_current_version is version(pmm_image_version, '>=') to run it only if upgrade is required +- name: Determine type of upgrade + set_fact: + docker_upgrade: not ui_upgrade and current_version_file['failed'] == false and not pmm_current_version is version(pmm_image_version, '>=') # We use current_version_file['failed'] because we don't want to run this on creating container # and we use pmm_current_version is version(pmm_image_version, '>=') to run it only if upgrade is required @@ -42,7 +47,7 @@ src: maintenance.html dest: /usr/share/pmm-server/maintenance/ mode: 0644 - when: not ui_upgrade and current_version_file['failed'] == false and not pmm_current_version is version(pmm_image_version, '>=') + when: docker_upgrade # PMM-10858 - In certain environments, including AWS EC2, some of the # EPEL repository mirrors do not respond within the time limit defined @@ -60,11 +65,6 @@ become: true changed_when: True -- name: Check if we need an update or not - include_role: - name: dashboards_upgrade - when: not pmm_current_version is version(pmm_image_version, '>=') - - name: Create backup directory file: path: /srv/backup @@ -159,7 +159,15 @@ tags: - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'. when: is_database_sqlite.stat.exists + when: docker_upgrade +- name: Check if we need an update or not + include_role: + name: dashboards_upgrade + when: not pmm_current_version is version(pmm_image_version, '>=') + +- name: Finalization + block: - name: Wait for PMM to be ready ansible.builtin.uri: url: "http://127.0.0.1:7772/v1/readyz" @@ -173,6 +181,6 @@ state: absent path: /usr/share/pmm-server/maintenance/maintenance.html # We use current_version_file['failed'] because we don't want to run this on creating container - when: not ui_upgrade and current_version_file['failed'] == false and not pmm_current_version is version(pmm_image_version, '>=') + when: docker_upgrade diff --git a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml b/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml index 490ce26c7f..5c12040c39 100644 --- a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml @@ -22,6 +22,39 @@ name: 'grafana' state: stopped +- name: Drop grafana database from postgres EL7 + postgresql_query: + db: postgres + query: DROP DATABASE grafana WITH (FORCE); + when: + - ansible_distribution == "CentOS" + - ansible_distribution_major_version == "7" + - not ansible_check_mode + +- name: Drop grafana database from postgres EL9 + postgresql_db: + name: grafana + state: absent + force: yes + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + +- name: Create grafana database in postgres + postgresql_db: + name: grafana + state: present + +- name: Create grafana user in postgres + postgresql_user: + db: grafana + name: grafana + password: grafana + priv: 'ALL' + expires: infinity + state: present + when: not ansible_check_mode + - name: Create backup for SQLite Grafana database copy: src: /srv/grafana/grafana.db @@ -105,6 +138,12 @@ register: migrator_output changed_when: "'All done' in migrator_output.stdout" +- name: copy the output of grafana-db-migrator to file + copy: + content: "{{ migrator_output.stdout }}" + dest: "/srv/logs/grafana-db-migrator.log" + when: not ansible_check_mode + - name: Enable provisioning after change database ini_file: dest: /etc/grafana/grafana.ini @@ -126,6 +165,13 @@ register: migrator_output changed_when: "'All done' in migrator_output.stdout" +- name: Copy the output of grafana-db-migrator to file + ansible.builtin.blockinfile: + dest: /srv/logs/grafana-db-migrator.log + block: "{{ migrator_output.stdout }}" + backup: yes + when: not ansible_check_mode + - name: Remove SQLite Grafana database file: path: /srv/grafana/grafana.db From 95643477e07061897bffa10adebf34198429626c Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 17 Oct 2023 12:36:37 +0300 Subject: [PATCH 6/8] PMM-12576 change admin ID to 1. (#2551) * PMM-12576 change admin ID to 1. * PMM-12576 Debug messages. --- .../tasks/roles/initialization/tasks/main.yml | 17 ++++++++++++++--- update/ansible/playbook/tasks/update.yml | 6 ++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index c3cc50d5f0..021b246b2b 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -38,7 +38,11 @@ # and we use pmm_current_version is version(pmm_image_version, '>=') to run it only if upgrade is required - name: Determine type of upgrade set_fact: - docker_upgrade: not ui_upgrade and current_version_file['failed'] == false and not pmm_current_version is version(pmm_image_version, '>=') + docker_upgrade: "{{ not ui_upgrade and current_version_file['failed'] != true and not pmm_current_version is version(pmm_image_version, '>=') }}" + +- name: Print Docker upgrade fact + debug: + msg: "Docker upgrade: {{ docker_upgrade }}" # We use current_version_file['failed'] because we don't want to run this on creating container # and we use pmm_current_version is version(pmm_image_version, '>=') to run it only if upgrade is required @@ -104,14 +108,14 @@ - name: Create empty configuration file for VictoriaMetrics file: path=/etc/victoriametrics-promscrape.yml state=touch owner=pmm group=pmm -- name: Run SQLite -> Postgres only for docker upgrade +- name: Run operations for docker-way upgrade block: - name: Check that the SQLite grafana database exists stat: path: /srv/grafana/grafana.db register: is_database_sqlite - - name: Temporary change database to SQLite + - name: Run SQLite -> Postgres only block: - name: Remove database options (SQLite is default) ini_file: @@ -159,6 +163,13 @@ tags: - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'. when: is_database_sqlite.stat.exists + + - name: Change default admin id + postgresql_query: + db: grafana + query: UPDATE "user" SET id='1' WHERE login='admin'; + when: not ansible_check_mode + when: docker_upgrade - name: Check if we need an update or not diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 61ae3a43a2..a4ebee9290 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -409,6 +409,12 @@ - ALTER TABLE api_key ALTER COLUMN name TYPE text; when: not ansible_check_mode + - name: Change default admin id + postgresql_query: + db: grafana + query: UPDATE "user" SET id='1' WHERE login='admin'; + when: not ansible_check_mode + # we need to put this step as one of the last steps, because it removes pmm.ini and /etc/alertmanager.yml - name: Remove old or redundant packages yum: From 85ca980088e59ba2bdc19cd7714d9178295dcf1a Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 18 Oct 2023 14:07:05 +0300 Subject: [PATCH 7/8] PMM-12587 Fix drop DB for EL7. (#2560) --- .../playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml b/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml index 5c12040c39..76a51febf6 100644 --- a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml @@ -23,9 +23,7 @@ state: stopped - name: Drop grafana database from postgres EL7 - postgresql_query: - db: postgres - query: DROP DATABASE grafana WITH (FORCE); + command: dropdb -f grafana -U postgres when: - ansible_distribution == "CentOS" - ansible_distribution_major_version == "7" From 12ce642fc58d1f442e834ad486e61d386d2fb77b Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 20 Oct 2023 15:40:13 +0300 Subject: [PATCH 8/8] PMM-7 ignore errors on not really necessary step. (#2564) --- .../playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml b/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml index 76a51febf6..8a905dcf3e 100644 --- a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml @@ -115,6 +115,7 @@ register: psql_result until: psql_result.rowcount == 1 when: not ansible_check_mode + ignore_errors: yes - name: Wait for grafana database initialization pause: