Skip to content

Commit

Permalink
PMM-11180 upgrade matrix (#3246)
Browse files Browse the repository at this point in the history
* PMM-7 Update pg_query_go

* PMM-7 Update test case

* PMM-7 Update PostgreSQL versions

* PMM-7 Update digests

* PMM-7 Update digests

* PMM-7 Update digests

* PMM-7 go mod tidy

* PMM-11180 update matrix and fix pgstatstatements_test.go.

* PMM-11180 fix mongodb tests.

* PMM-11180 fix mongodb tests.

* PMM-11180 fix mongodb tests.

* PMM-11180 fix mongodb tests.

* PMM-11180 fix init scripts for mongodb.

* PMM-11180 update mongodb version.

* PMM-11180 add debug info on fail.

* PMM-11180 Improvements.

* PMM-11180 update docker-compose.yml.

* PMM-11180 update docker-compose.yml.

* PMM-11180 provide more debug info.

* PMM-11180 revert host related changes.

* PMM-11180 revert host related changes.

* PMM-11180 fix test.

* PMM-11180 fix tests.

* PMM-11180 fix tests.

* PMM-11180 fix tests.

* PMM-11180 fix tests.

* PMM-11180 fix tests.

* PMM-11180 fix tests.

* PMM-11180 fix tests.

* PMM-11180 fix MySQL explain json output for MariaDB 11.

* PMM-11180 fix MySQL explain json test for MariaDB 11.

* PMM-11180 fix perfschema tests.

* PMM-11180 fix perfschema tests for MariaDB.

* PMM-11180 fix linter and mongodb test.

* PMM-11180 fix test.

* PMM-11180 fixing tests.

* PMM-11180 fixing tests.

* PMM-11180 fixing tests.

* PMM-11180 fixing linter.
  • Loading branch information
BupycHuk authored Oct 18, 2024
1 parent 7108418 commit 6a67936
Show file tree
Hide file tree
Showing 19 changed files with 188 additions and 162 deletions.
25 changes: 15 additions & 10 deletions .github/workflows/agent.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,18 @@ jobs:
strategy:
matrix:
images:
- { mysql: 'mysql:5.6', mongo: 'mongo:4.4', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:15', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:16', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:17rc1', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mysql:5.6', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mysql:5.7', mongo: 'mongo:5.0', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mysql:8.0', mongo: 'mongo:6.0', postgres: 'postgres:15', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mysql:lts', mongo: 'mongo:7.0', postgres: 'postgres:16', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mysql:latest', mongo: 'mongo:latest', postgres: 'postgres:latest', pmm_server: 'perconalab/pmm-server:3-dev-latest' }

# Percona + latest PMM Server release
- { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:17-dev', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:14', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:5.0', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:6.0', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'percona:8', mongo: 'percona/percona-server-mongodb:7.0', postgres: 'perconalab/percona-distribution-postgresql:17', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'percona:latest', mongo: 'percona/percona-server-mongodb:latest', postgres: 'perconalab/percona-distribution-postgresql:latest', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }

# MariaDB (only 3 latest GA versions)
# + older supported MongoDB versions
Expand All @@ -46,9 +49,9 @@ jobs:
# - MYSQL_IMAGE=mariadb:10.0
# - MYSQL_IMAGE=mariadb:10.1

- { mysql: 'mariadb:10.2', mongo: 'mongo:4.4', postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mariadb:10.4', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mariadb:11.2', mongo: 'mongo:4.4', postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mariadb:11.4', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:13', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
- { mysql: 'mariadb:11.5', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' }

continue-on-error: true

Expand Down Expand Up @@ -120,3 +123,5 @@ jobs:
echo "--- GO Environment ---"
go env | sort
git status
docker compose logs
docker compose ps -a
52 changes: 4 additions & 48 deletions agent/agents/mongodb/internal/profiler/profiler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@ package profiler

import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"strings"
"testing"
Expand All @@ -32,7 +30,6 @@ import (

"github.com/percona/pmm/agent/agents/mongodb/internal/profiler/aggregator"
"github.com/percona/pmm/agent/agents/mongodb/internal/report"
"github.com/percona/pmm/agent/runner/actions"
"github.com/percona/pmm/agent/utils/templates"
"github.com/percona/pmm/agent/utils/tests"
"github.com/percona/pmm/agent/utils/truncate"
Expand Down Expand Up @@ -69,11 +66,11 @@ func TestProfiler(t *testing.T) {
tempDir := t.TempDir()
sslDSN, err := templates.RenderDSN(sslDSNTemplate, files, tempDir)
require.NoError(t, err)
for _, url := range []string{
"mongodb://root:[email protected]:27017/admin",
sslDSN,
for name, url := range map[string]string{
"normal": tests.GetTestMongoDBDSN(t),
"ssl": sslDSN,
} {
t.Run(url, func(t *testing.T) {
t.Run(name, func(t *testing.T) {
testProfiler(t, url)
})
}
Expand Down Expand Up @@ -210,47 +207,6 @@ func testProfiler(t *testing.T, url string) {
require.NotNil(t, findBucket)
assert.Equal(t, "FIND people name_00\ufffd", findBucket.Common.Fingerprint)
assert.Equal(t, docsCount, findBucket.Mongodb.MDocsReturnedSum)

// PMM-4192 This seems to be out of place because it is an Explain test but there was a problem with
// the new MongoDB driver and bson.D and we were capturing invalid queries in the profiler.
// This test is here to ensure the query example the profiler captures is valid to be used in Explain.
t.Run("TestMongoDBExplain", func(t *testing.T) {
id := "abcd1234"

params := &agentv1.StartActionRequest_MongoDBExplainParams{
Dsn: tests.GetTestMongoDBDSN(t),
Query: findBucket.Common.Example,
}

ex, err := actions.NewMongoDBExplainAction(id, 5*time.Second, params, os.TempDir())
require.NoError(t, err)

ctx, cancel := context.WithTimeout(context.Background(), ex.Timeout())
defer cancel()
res, err := ex.Run(ctx)
assert.Nil(t, err)

want := map[string]interface{}{
"indexFilterSet": false,
"namespace": "test_00.people",
"parsedQuery": map[string]interface{}{
"name_00\ufffd": map[string]interface{}{
"$eq": "value_00\ufffd",
},
},
"plannerVersion": map[string]interface{}{"$numberInt": "1"},
"rejectedPlans": []interface{}{},
}

explainM := make(map[string]interface{})
err = json.Unmarshal(res, &explainM)
assert.Nil(t, err)
queryPlanner, ok := explainM["queryPlanner"].(map[string]interface{})
want["winningPlan"] = queryPlanner["winningPlan"]
assert.Equal(t, ok, true)
assert.NotEmpty(t, queryPlanner)
assert.Equal(t, want, queryPlanner)
})
}

func cleanUpDBs(t *testing.T, sess *mongo.Client) {
Expand Down
8 changes: 6 additions & 2 deletions agent/agents/mysql/perfschema/history.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,12 @@ func newHistoryCache(typ historyMap, retain time.Duration, sizeLimit uint, l *lo
return &historyCache{c}, err
}

func getHistory(q *reform.Querier) (historyMap, error) {
rows, err := q.SelectRows(eventsStatementsHistoryView, "WHERE DIGEST IS NOT NULL AND SQL_TEXT IS NOT NULL")
func getHistory(q *reform.Querier, long *bool) (historyMap, error) {
view := eventsStatementsHistoryView
if long != nil && *long {
view = eventsStatementsHistoryLongView
}
rows, err := q.SelectRows(view, "WHERE DIGEST IS NOT NULL AND SQL_TEXT IS NOT NULL")
if err != nil {
return nil, errors.Wrap(err, "failed to query events_statements_history")
}
Expand Down
19 changes: 19 additions & 0 deletions agent/agents/mysql/perfschema/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

package perfschema

import "gopkg.in/reform.v1/parse"

//go:generate ../../../../bin/reform

// eventsStatementsSummaryByDigest represents a row in performance_schema.events_statements_summary_by_digest table.
Expand Down Expand Up @@ -121,3 +123,20 @@ type setupInstruments struct {
Enabled string `reform:"ENABLED"`
Timed *string `reform:"TIMED"` // nullable in 8.0
}

// eventsStatementsHistoryView represents events_statements_history view or table in SQL database.
var eventsStatementsHistoryLongView = &eventsStatementsHistoryViewType{
s: parse.StructInfo{
Type: "eventsStatementsHistory",
SQLSchema: "performance_schema",
SQLName: "events_statements_history_long",
Fields: []parse.FieldInfo{
{Name: "SQLText", Type: "*string", Column: "SQL_TEXT"},
{Name: "Digest", Type: "*string", Column: "DIGEST"},
{Name: "DigestText", Type: "*string", Column: "DIGEST_TEXT"},
{Name: "CurrentSchema", Type: "*string", Column: "CURRENT_SCHEMA"},
},
PKFieldIndex: -1,
},
z: (&eventsStatementsHistory{}).Values(),
}
11 changes: 10 additions & 1 deletion agent/agents/mysql/perfschema/perfschema.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"github.com/percona/pmm/agent/queryparser"
"github.com/percona/pmm/agent/tlshelpers"
"github.com/percona/pmm/agent/utils/truncate"
"github.com/percona/pmm/agent/utils/version"
agentv1 "github.com/percona/pmm/api/agent/v1"
inventoryv1 "github.com/percona/pmm/api/inventory/v1"
"github.com/percona/pmm/utils/sqlmetrics"
Expand Down Expand Up @@ -67,6 +68,7 @@ type PerfSchema struct {
changes chan agents.Change
historyCache *historyCache
summaryCache *summaryCache
useLong *bool
}

// Params represent Agent parameters.
Expand Down Expand Up @@ -274,7 +276,14 @@ func (m *PerfSchema) runHistoryCacheRefresher(ctx context.Context) {
}

func (m *PerfSchema) refreshHistoryCache() error {
current, err := getHistory(m.q)
if m.useLong == nil {
sqlVersion, vendor, err := version.GetMySQLVersion(context.Background(), m.q)
if err != nil {
return errors.Wrap(err, "cannot get MySQL version")
}
m.useLong = pointer.ToBool(vendor == version.MariaDBVendor && sqlVersion.Float() >= 11)
}
current, err := getHistory(m.q, m.useLong)
if err != nil {
return err
}
Expand Down
55 changes: 37 additions & 18 deletions agent/agents/mysql/perfschema/perfschema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,15 @@ func setup(t *testing.T, sp *setupParams) *PerfSchema {

// filter removes buckets for queries that are not expected by tests.
func filter(mb []*agentv1.MetricsBucket) []*agentv1.MetricsBucket {
filterList := map[string]struct{}{
"ANALYZE TABLE `city`": {}, // OpenTestMySQL
"SHOW GLOBAL VARIABLES WHERE `Variable_name` = ?": {}, // MySQLVersion
"SHOW VARIABLES LIKE ?": {}, // MariaDBVersion
"SELECT `id` FROM `city` LIMIT ?": {}, // waitForFixtures
"SELECT ID FROM `city` LIMIT ?": {}, // waitForFixtures for MariaDB
"SELECT COUNT ( * ) FROM `city`": {}, // actions tests
"CREATE TABLE IF NOT EXISTS `t1` ( `col1` CHARACTER (?) ) CHARACTER SET `utf8mb4` COLLATE `utf8mb4_general_ci`": {}, // tests for invalid characters
}
res := make([]*agentv1.MetricsBucket, 0, len(mb))
for _, b := range mb {
switch {
Expand All @@ -215,23 +224,14 @@ func filter(mb []*agentv1.MetricsBucket) []*agentv1.MetricsBucket {
continue
case strings.Contains(b.Common.Example, "/* pmm-agent-tests:waitForFixtures */"):
continue
}

switch {
case b.Common.Fingerprint == "ANALYZE TABLE `city`": // OpenTestMySQL
continue
case b.Common.Fingerprint == "SHOW GLOBAL VARIABLES WHERE `Variable_name` = ?": // MySQLVersion
continue
case b.Common.Fingerprint == "SELECT `id` FROM `city` LIMIT ?": // waitForFixtures
case strings.Contains(b.Common.Fingerprint, "events_statements_history"):
continue
case b.Common.Fingerprint == "SELECT ID FROM `city` LIMIT ?": // waitForFixtures for MariaDB
continue
case b.Common.Fingerprint == "SELECT COUNT ( * ) FROM `city`": // actions tests
case strings.HasPrefix(b.Common.Fingerprint, "SELECT @@`slow_query_log"): // slowlog
continue
case b.Common.Fingerprint == "CREATE TABLE IF NOT EXISTS `t1` ( `col1` CHARACTER (?) ) CHARACTER SET `utf8mb4` COLLATE `utf8mb4_general_ci`": // tests for invalid characters
case strings.HasPrefix(b.Common.Fingerprint, "TRUNCATE"): // OpenTestMySQL
continue

case strings.HasPrefix(b.Common.Fingerprint, "SELECT @@`slow_query_log"): // slowlog
}
if _, ok := filterList[b.Common.Fingerprint]; ok {
continue
}

Expand All @@ -246,7 +246,7 @@ func TestPerfSchema(t *testing.T) {
db := reform.NewDB(sqlDB, mysql.Dialect, reform.NewPrintfLogger(t.Logf))

updateQuery := fmt.Sprintf("UPDATE /* %s */ ", queryTag)
_, err := db.Exec(updateQuery + "performance_schema.setup_consumers SET ENABLED='YES' WHERE NAME='events_statements_history'")
_, err := db.Exec(updateQuery + "performance_schema.setup_consumers SET ENABLED='YES'")
require.NoError(t, err, "failed to enable events_statements_history consumer")

structs, err := db.SelectAllFrom(setupConsumersView, "ORDER BY NAME")
Expand All @@ -259,6 +259,7 @@ func TestPerfSchema(t *testing.T) {
var rowsExamined float32
ctx := context.Background()
mySQLVersion, mySQLVendor, _ := version.GetMySQLVersion(ctx, db.WithTag("pmm-agent-tests:MySQLVersion"))
t.Logf("MySQL version: %s, vendor: %s", mySQLVersion, mySQLVendor)
var digests map[string]string // digest_text/fingerprint to digest/query_id
switch fmt.Sprintf("%s-%s", mySQLVersion, mySQLVendor) {
case "5.6-oracle":
Expand All @@ -283,7 +284,7 @@ func TestPerfSchema(t *testing.T) {
"SELECT * FROM `city`": "9c799bdb2460f79b3423b77cd10403da",
}

case "8.0-oracle", "8.0-percona":
case "8.0-oracle", "8.0-percona", "8.4-oracle", "9.0-oracle", "9.1-oracle":
digests = map[string]string{
"SELECT `sleep` (?)": "0b1b1c39d4ee2dda7df2a532d0a23406d86bd34e2cd7f22e3f7e9dedadff9b69",
"SELECT * FROM `city`": "950bdc225cf73c9096ba499351ed4376f4526abad3d8ceabc168b6b28cfc9eab",
Expand All @@ -308,6 +309,24 @@ func TestPerfSchema(t *testing.T) {
"SELECT * FROM `city`": "a65e76b1643273fa3206b11c4f4d8739",
}

case "11.2-mariadb":
digests = map[string]string{
"SELECT `sleep` (?)": "ffbde6c4dfda8dff9a4fefd7e8ed648f",
"SELECT * FROM `city`": "d0f2ac0577a44d383c5c0480a420caeb",
}

case "11.4-mariadb":
digests = map[string]string{
"SELECT `sleep` (?)": "860792b8f3d058489b287e30ccf3beae",
"SELECT * FROM `city`": "457a868ea48e4571327914f2831d62f5",
}

case "11.5-mariadb":
digests = map[string]string{
"SELECT `sleep` (?)": "860792b8f3d058489b287e30ccf3beae",
"SELECT * FROM `city`": "457a868ea48e4571327914f2831d62f5",
}

default:
t.Log("Unhandled version, assuming dummy digests.")
digests = map[string]string{
Expand Down Expand Up @@ -433,9 +452,9 @@ func TestPerfSchema(t *testing.T) {

require.NoError(t, m.refreshHistoryCache())
var example string
switch mySQLVersion.String() {
switch {
// Perf schema truncates queries with non-utf8 characters.
case "8.0":
case (mySQLVendor == version.PerconaVendor || mySQLVendor == version.OracleVendor) && mySQLVersion.Float() >= 8.0:
example = "SELECT /* t1 controller='test' */ * FROM t1 where col1='Bu"
default:
example = "SELECT /* t1 controller='test' */ * FROM t1 where col1=..."
Expand Down
26 changes: 3 additions & 23 deletions agent/agents/postgres/pgstatstatements/pgstatstatements_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,18 +120,6 @@ func TestPGStatStatementsQAN(t *testing.T) {
engineVersion := tests.PostgreSQLVersion(t, sqlDB)
var digests map[string]string // digest_text/fingerprint to digest/query_id
switch engineVersion {
case "9.4":
truncatedMSharedBlksHitSum = float32(1007)
digests = map[string]string{
selectAllCities: "3239586867",
selectAllCitiesLong: "2745128652",
}
case "9.5", "9.6":
truncatedMSharedBlksHitSum = float32(1007)
digests = map[string]string{
selectAllCities: "3994135135",
selectAllCitiesLong: "2677760328",
}
case "10":
truncatedMSharedBlksHitSum = float32(1007)
digests = map[string]string{
Expand Down Expand Up @@ -360,7 +348,7 @@ func TestPGStatStatementsQAN(t *testing.T) {
MSharedBlkReadTimeSum: actual.Postgresql.MSharedBlkReadTimeSum,
MLocalBlkReadTimeCnt: actual.Postgresql.MLocalBlkReadTimeCnt,
MLocalBlkReadTimeSum: actual.Postgresql.MLocalBlkReadTimeSum,
MSharedBlksHitCnt: 1,
MSharedBlksHitCnt: actual.Postgresql.MSharedBlksHitCnt,
MSharedBlksHitSum: actual.Postgresql.MSharedBlksHitSum,
MRowsCnt: 1,
MRowsSum: 499,
Expand Down Expand Up @@ -408,16 +396,8 @@ func TestPGStatStatementsQAN(t *testing.T) {
t.Logf("Actual:\n%s", tests.FormatBuckets(buckets))
require.Len(t, buckets, 1)

var fingerprint string
tables := []string{tableName}
fingerprint := fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName)

switch engineVersion {
case "9.4", "9.5", "9.6":
fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES (?, ?, ?, ?)`, tableName)
tables = []string{}
default:
fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName)
}
actual := buckets[0]
assert.NotZero(t, actual.Postgresql.MSharedBlkReadTimeSum+actual.Postgresql.MSharedBlkWriteTimeSum)
assert.Equal(t, float32(n), actual.Postgresql.MSharedBlkReadTimeCnt+actual.Postgresql.MSharedBlkWriteTimeCnt)
Expand All @@ -426,7 +406,7 @@ func TestPGStatStatementsQAN(t *testing.T) {
Queryid: actual.Common.Queryid,
Fingerprint: fingerprint,
Database: "pmm-agent",
Tables: tables,
Tables: []string{tableName},
Comments: map[string]string{"controller": "test"},
Username: "pmm-agent",
AgentId: "agent_id",
Expand Down
2 changes: 1 addition & 1 deletion agent/connectionchecker/connection_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ func TestConnectionChecker(t *testing.T) {
assert.Empty(t, resp.Error)
} else {
require.NotEmpty(t, resp.Error)
assert.Regexp(t, `^`+tt.expectedErr+`$`, resp.Error)
assert.Regexp(t, `(?i)^`+tt.expectedErr+`$`, resp.Error)
}
})
}
Expand Down
Loading

0 comments on commit 6a67936

Please sign in to comment.