From 625511926c990971dc1d5c28f19fa38c866df6da Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Mon, 15 Jul 2019 18:06:55 +0300 Subject: [PATCH 01/11] PMM-1901 Remove generator and add one endpoint for all metrics. --- basic/basic.go | 21 ++++- basic/generate/main.go | 179 ---------------------------------------- basic/generate/utils.go | 34 -------- basic/metrics.go | 7 +- main.go | 35 +++++++- 5 files changed, 53 insertions(+), 223 deletions(-) delete mode 100644 basic/generate/main.go delete mode 100644 basic/generate/utils.go diff --git a/basic/basic.go b/basic/basic.go index 4ef0dcd4..b71124e4 100644 --- a/basic/basic.go +++ b/basic/basic.go @@ -11,8 +11,6 @@ import ( "github.com/percona/rds_exporter/sessions" ) -//go:generate go run generate/main.go generate/utils.go - var ( scrapeTimeDesc = prometheus.NewDesc( "rds_exporter_scrape_duration_seconds", @@ -44,6 +42,25 @@ func New(config *config.Config, sessions *sessions.Sessions) *Exporter { } } +func (e *Exporter) Exclude(metrics ...string) { + var filtered []Metric + for _, v := range e.metrics { + if !contains(v.Name, metrics) { + filtered = append(filtered, v) + } + } + e.metrics = filtered +} + +func contains(metricName string, metrics []string) bool { + for _, v := range metrics { + if v == metricName { + return true + } + } + return false +} + func (e *Exporter) Collect(ch chan<- prometheus.Metric) { now := time.Now() e.collect(ch) diff --git a/basic/generate/main.go b/basic/generate/main.go deleted file mode 100644 index 9a1510a7..00000000 --- a/basic/generate/main.go +++ /dev/null @@ -1,179 +0,0 @@ -// The following directive is necessary to make the package coherent: -// This program generates metrics.go. It can be invoked by running -// go generate -package main - -import ( - "log" - "os" - "sort" - "text/template" -) - -type Metric string - -func (m Metric) FqName() string { - switch m { - case "FreeStorageSpace": - return "node_filesystem_free" - case "FreeableMemory": - return "node_memory_Cached" - case "CPUUtilization": - return "node_cpu_average" - case "EngineUptime": - return "node_boot_time" - } - - return safeName("AWS/RDS_" + toSnakeCase(string(m)) + "_average") -} - -func (m Metric) Labels() []string { - return []string{ - "instance", - "region", - } -} - -func (m Metric) ConstLabels() map[string]string { - switch m { - case "CPUUtilization": - return map[string]string{ - "cpu": "All", - "mode": "total", - } - } - - return nil -} - -func (m Metric) Name() string { - return string(m) -} - -func (m Metric) Help() string { - if v, ok := doc[m.Name()]; ok { - return v - } - - return m.Name() -} - -var ( - metrics = []Metric{ - "ActiveTransactions", - "AuroraBinlogReplicaLag", - "AuroraReplicaLag", - "AuroraReplicaLagMaximum", - "AuroraReplicaLagMinimum", - "BinLogDiskUsage", - "BlockedTransactions", - "BufferCacheHitRatio", - "BurstBalance", - "CommitLatency", - "CommitThroughput", - "CPUCreditBalance", - "CPUCreditUsage", - "CPUUtilization", - "DatabaseConnections", - "DDLLatency", - "DDLThroughput", - "Deadlocks", - "DeleteLatency", - "DeleteThroughput", - "DiskQueueDepth", - "DMLLatency", - "DMLThroughput", - "EngineUptime", - "FreeableMemory", - "FreeLocalStorage", - "FreeStorageSpace", - "InsertLatency", - "InsertThroughput", - "LoginFailures", - "NetworkReceiveThroughput", - "NetworkThroughput", - "NetworkTransmitThroughput", - "Queries", - "ReadIOPS", - "ReadLatency", - "ReadThroughput", - "ResultSetCacheHitRatio", - "SelectLatency", - "SelectThroughput", - "SwapUsage", - "UpdateLatency", - "UpdateThroughput", - "VolumeBytesUsed", - "VolumeReadIOPs", - "VolumeWriteIOPs", - "WriteIOPS", - "WriteLatency", - "WriteThroughput", - } - - doc = map[string]string{ - "BinLogDiskUsage": "The amount of disk space occupied by binary logs on the master. Applies to MySQL read replicas. Units: Bytes", - "BurstBalance": "The percent of General Purpose SSD (gp2) burst-bucket I/O credits available. Units: Percent", - "CPUUtilization": "The percentage of CPU utilization. Units: Percent", - "CPUCreditUsage": "[T2 instances] The number of CPU credits consumed by the instance. One CPU credit equals one vCPU running at 100% utilization for one minute or an equivalent combination of vCPUs, utilization, and time (for example, one vCPU running at 50% utilization for two minutes or two vCPUs running at 25% utilization for two minutes). CPU credit metrics are available only at a 5 minute frequency. If you specify a period greater than five minutes, use the Sum statistic instead of the Average statistic. Units: Count", - "CPUCreditBalance": "[T2 instances] The number of CPU credits available for the instance to burst beyond its base CPU utilization. Credits are stored in the credit balance after they are earned and removed from the credit balance after they expire. Credits expire 24 hours after they are earned. CPU credit metrics are available only at a 5 minute frequency. Units: Count", - "DatabaseConnections": "The number of database connections in use. Units: Count", - "DiskQueueDepth": "The number of outstanding IOs (read/write requests) waiting to access the disk. Units: Count", - "FreeableMemory": "The amount of available random access memory. Units: Bytes", - "FreeStorageSpace": "The amount of available storage space. Units: Bytes", - "MaximumUsedTransactionIDs": "The maximum transaction ID that has been used. Applies to PostgreSQL. Units: Count", - "NetworkReceiveThroughput": "The incoming (Receive) network traffic on the DB instance, including both customer database traffic and Amazon RDS traffic used for monitoring and replication. Units: Bytes/second", - "NetworkTransmitThroughput": "The outgoing (Transmit) network traffic on the DB instance, including both customer database traffic and Amazon RDS traffic used for monitoring and replication. Units: Bytes/second", - "OldestReplicationSlotLag": "The lagging size of the replica lagging the most in terms of WAL data received. Applies to PostgreSQL. Units: Megabytes", - "ReadIOPS": "The average number of disk I/O operations per second. Units: Count/Second", - "ReadLatency": "The average amount of time taken per disk I/O operation. Units: Seconds", - "ReadThroughput": "The average number of bytes read from disk per second. Units: Bytes/Second", - "ReplicaLag": "The amount of time a Read Replica DB instance lags behind the source DB instance. Applies to MySQL, MariaDB, and PostgreSQL Read Replicas. Units: Seconds", - "ReplicationSlotDiskUsage": "The disk space used by replication slot files. Applies to PostgreSQL. Units: Megabytes", - "SwapUsage": "The amount of swap space used on the DB instance. Units: Bytes", - "TransactionLogsDiskUsage": "The disk space used by transaction logs. Applies to PostgreSQL. Units: Megabytes", - "TransactionLogsGeneration": "The size of transaction logs generated per second. Applies to PostgreSQL. Units: Megabytes/second", - "WriteIOPS": "The average number of disk I/O operations per second. Units: Count/Second", - "WriteLatency": "The average amount of time taken per disk I/O operation. Units: Seconds", - "WriteThroughput": "The average number of bytes written to disk per second. Units: Bytes/Second", - } -) - -func main() { - f, err := os.Create("metrics.go") - if err != nil { - log.Fatal(err) - } - defer f.Close() - - sort.SliceStable(metrics, func(i, j int) bool { - return metrics[i] < metrics[j] - }) - packageTemplate.Execute(f, struct { - Metrics []Metric - }{ - Metrics: metrics, - }) -} - -var packageTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. -package basic - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -var Metrics = []Metric{ -{{- range .Metrics }} - { - Name: "{{.}}", - Desc: prometheus.NewDesc( - "{{.FqName}}", - "{{.Help}}", - {{printf "%#v" .Labels}}, - {{printf "%#v" .ConstLabels}}, - ), - }, -{{- end }} -} -`)) diff --git a/basic/generate/utils.go b/basic/generate/utils.go deleted file mode 100644 index ec75bc5f..00000000 --- a/basic/generate/utils.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "regexp" - "strings" - "unicode" -) - -var ( - sanitizeNameRegex, _ = regexp.Compile("[^a-zA-Z0-9:_]") - mergeUScoreRegex, _ = regexp.Compile("__+") -) - -func safeName(dirty string) string { - return mergeUScoreRegex.ReplaceAllString( - sanitizeNameRegex.ReplaceAllString( - strings.ToLower(dirty), "_"), - "_") -} - -func toSnakeCase(in string) string { - runes := []rune(in) - length := len(runes) - - var out []rune - for i := 0; i < length; i++ { - if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) { - out = append(out, '_') - } - out = append(out, unicode.ToLower(runes[i])) - } - - return string(out) -} diff --git a/basic/metrics.go b/basic/metrics.go index d460f0e6..4247460e 100644 --- a/basic/metrics.go +++ b/basic/metrics.go @@ -1,4 +1,3 @@ -// Code generated by go generate; DO NOT EDIT. package basic import ( @@ -108,7 +107,7 @@ var Metrics = []Metric{ { Name: "CPUUtilization", Desc: prometheus.NewDesc( - "node_cpu_average", + "node_cpu_average", // TODO: Conflict... "The percentage of CPU utilization. Units: Percent", []string{"instance", "region"}, map[string]string{"cpu": "All", "mode": "total"}, @@ -234,7 +233,7 @@ var Metrics = []Metric{ { Name: "FreeStorageSpace", Desc: prometheus.NewDesc( - "node_filesystem_free", + "node_filesystem_free", // TODO: Conflict... "The amount of available storage space. Units: Bytes", []string{"instance", "region"}, map[string]string(nil), @@ -243,7 +242,7 @@ var Metrics = []Metric{ { Name: "FreeableMemory", Desc: prometheus.NewDesc( - "node_memory_Cached", + "node_memory_Cached", // TODO: Conflict... "The amount of available random access memory. Units: Bytes", []string{"instance", "region"}, map[string]string(nil), diff --git a/main.go b/main.go index fa06f9fa..a7a27c3f 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "net/http" + "os" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -44,25 +45,51 @@ func main() { // basic metrics + client metrics + exporter own metrics (ProcessCollector and GoCollector) { - prometheus.MustRegister(basic.New(cfg, sess)) - prometheus.MustRegister(client) - http.Handle(*basicMetricsPathF, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + // Separate instance of basic collector for backward compatibility. See: https://jira.percona.com/browse/PMM-1901. + basicCollector := basic.New(cfg, sess) + registry := prometheus.NewRegistry() + registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) + registry.MustRegister(prometheus.NewGoCollector()) + registry.MustRegister(basicCollector) + registry.MustRegister(client) + http.Handle(*basicMetricsPathF, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ ErrorLog: log.NewErrorLogger(), ErrorHandling: promhttp.ContinueOnError, })) } + // This collector should be only one for both cases. + // It creates goroutines which sends API requests to Amazon in background. + enhancedCollector := enhanced.NewCollector(sess) + // enhanced metrics { registry := prometheus.NewRegistry() - registry.MustRegister(enhanced.NewCollector(sess)) + registry.MustRegister(enhancedCollector) http.Handle(*enhancedMetricsPathF, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ ErrorLog: log.NewErrorLogger(), ErrorHandling: promhttp.ContinueOnError, })) } + // all metrics + { + // Create separate instance of basic collector and remove metrics which cross with enhanced collector. + // Made for backward compatibility. See: https://jira.percona.com/browse/PMM-1901. + basicCollector := basic.New(cfg, sess) + basicCollector.Exclude("CPUUtilization", "FreeStorageSpace", "FreeableMemory") + + prometheus.MustRegister(client) + prometheus.MustRegister(basicCollector) + prometheus.MustRegister(enhancedCollector) + http.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + ErrorLog: log.NewErrorLogger(), + ErrorHandling: promhttp.ContinueOnError, + })) + } + log.Infof("Basic metrics : http://%s%s", *listenAddressF, *basicMetricsPathF) log.Infof("Enhanced metrics: http://%s%s", *listenAddressF, *enhancedMetricsPathF) + log.Infof("All metrics: http://%s%s", *listenAddressF, "/metrics") log.Fatal(http.ListenAndServe(*listenAddressF, nil)) } From ef6e472f97f9a733ea6bef483fdbaf968d29e790 Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Tue, 16 Jul 2019 19:56:58 +0300 Subject: [PATCH 02/11] PMM-1901 Add collectors filtering. --- README.md | 30 +++++++++++++- basic/basic.go | 2 +- main.go | 108 +++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 121 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index d62f3033..b9cd7a1c 100644 --- a/README.md +++ b/README.md @@ -52,20 +52,46 @@ scrape_configs: - job_name: rds-basic scrape_interval: 60s scrape_timeout: 55s - metrics_path: /basic honor_labels: true static_configs: - targets: - 127.0.0.1:9042 + params: + collect[]: + - basic - job_name: rds-enhanced scrape_interval: 10s scrape_timeout: 9s - metrics_path: /enhanced honor_labels: true static_configs: - targets: - 127.0.0.1:9042 + params: + collect[]: + - enhanced ``` `honor_labels: true` is important because exporter returns metrics with `instance` label set. + +## Collectors + +### Enabled by default + +Name | Description +---------|------------- +basic | Basic metrics from https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/MonitoringOverview.html#monitoring-cloudwatch. +enhanced | Enhanced metrics from https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_Monitoring.OS.html. + +### Filtering enabled collectors + +The `rds_exporter` will expose all metrics from enabled collectors by default. + +For advanced use the `rds_exporter` can be passed an optional list of collectors to filter metrics. The `collect[]` parameter may be used multiple times. In Prometheus configuration you can use this syntax under the [scrape config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#). + +``` + params: + collect[]: + - basic + - enhanced +``` diff --git a/basic/basic.go b/basic/basic.go index b71124e4..da23333c 100644 --- a/basic/basic.go +++ b/basic/basic.go @@ -42,7 +42,7 @@ func New(config *config.Config, sessions *sessions.Sessions) *Exporter { } } -func (e *Exporter) Exclude(metrics ...string) { +func (e *Exporter) ExcludeMetrics(metrics ...string) { var filtered []Metric for _, v := range e.metrics { if !contains(v.Name, metrics) { diff --git a/main.go b/main.go index a7a27c3f..9fcb3cd8 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "net/http" "os" @@ -44,12 +45,14 @@ func main() { } // basic metrics + client metrics + exporter own metrics (ProcessCollector and GoCollector) + // NOTE: This handler was retained for backward compatibility. See: https://jira.percona.com/browse/PMM-1901. { - // Separate instance of basic collector for backward compatibility. See: https://jira.percona.com/browse/PMM-1901. basicCollector := basic.New(cfg, sess) + registry := prometheus.NewRegistry() - registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) - registry.MustRegister(prometheus.NewGoCollector()) + registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) // from prometheus.DefaultGatherer + registry.MustRegister(prometheus.NewGoCollector()) // from prometheus.DefaultGatherer + registry.MustRegister(basicCollector) registry.MustRegister(client) http.Handle(*basicMetricsPathF, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ @@ -58,11 +61,11 @@ func main() { })) } - // This collector should be only one for both cases. - // It creates goroutines which sends API requests to Amazon in background. + // This collector should be only one for both cases, because it creates goroutines which sends API requests to Amazon in background. enhancedCollector := enhanced.NewCollector(sess) // enhanced metrics + // NOTE: This handler was retained for backward compatibility. See: https://jira.percona.com/browse/PMM-1901. { registry := prometheus.NewRegistry() registry.MustRegister(enhancedCollector) @@ -72,24 +75,97 @@ func main() { })) } - // all metrics + // all metrics (with filtering) { // Create separate instance of basic collector and remove metrics which cross with enhanced collector. - // Made for backward compatibility. See: https://jira.percona.com/browse/PMM-1901. basicCollector := basic.New(cfg, sess) - basicCollector.Exclude("CPUUtilization", "FreeStorageSpace", "FreeableMemory") + basicCollector.ExcludeMetrics("CPUUtilization", "FreeStorageSpace", "FreeableMemory") prometheus.MustRegister(client) - prometheus.MustRegister(basicCollector) - prometheus.MustRegister(enhancedCollector) - http.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ - ErrorLog: log.NewErrorLogger(), - ErrorHandling: promhttp.ContinueOnError, + http.Handle("/metrics", newHandler(map[string]prometheus.Collector{ + "basic": basicCollector, + "enhanced": enhancedCollector, })) } - log.Infof("Basic metrics : http://%s%s", *listenAddressF, *basicMetricsPathF) - log.Infof("Enhanced metrics: http://%s%s", *listenAddressF, *enhancedMetricsPathF) - log.Infof("All metrics: http://%s%s", *listenAddressF, "/metrics") + log.Infof("Metrics: http://%s%s", *listenAddressF, "/metrics") log.Fatal(http.ListenAndServe(*listenAddressF, nil)) } + +// handler wraps an unfiltered http.Handler but uses a filtered handler, +// created on the fly, if filtering is requested. Create instances with +// newHandler. It used for collectors filtering. +type handler struct { + unfilteredHandler http.Handler + collectors map[string]prometheus.Collector +} + +func newHandler(collectors map[string]prometheus.Collector) *handler { + h := &handler{collectors: collectors} + if innerHandler, err := h.innerHandler(); err != nil { + log.Fatalf("Couldn't create metrics handler: %s", err) + } else { + h.unfilteredHandler = innerHandler + } + return h +} + +// ServeHTTP implements http.Handler. +func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + filters := r.URL.Query()["collect[]"] + log.Debugln("collect query:", filters) + + if len(filters) == 0 { + // No filters, use the prepared unfiltered handler. + h.unfilteredHandler.ServeHTTP(w, r) + return + } + + filteredHandler, err := h.innerHandler(filters...) + if err != nil { + log.Warnln("Couldn't create filtered metrics handler:", err) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(fmt.Sprintf("Couldn't create filtered metrics handler: %s", err))) + return + } + filteredHandler.ServeHTTP(w, r) +} + +func (h *handler) innerHandler(filters ...string) (http.Handler, error) { + // clear all previously registered collector (to prevent repeatable registration). + for k, v := range h.collectors { + if ok := prometheus.Unregister(v); ok { + log.Infof("Collector '%s' was unregistered", k) + } + } + + // register all collectors by default. + if len(filters) == 0 { + for name, c := range h.collectors { + if err := prometheus.Register(c); err != nil { + return nil, err + } + log.Infof("Collector '%s' was registered", name) + } + } + + // register only filtered collectors. + for _, name := range filters { + if c, ok := h.collectors[name]; ok { + if err := prometheus.Register(c); err != nil { + return nil, err + } + log.Infof("Collector '%s' was registered", name) + } + } + + handler := promhttp.HandlerFor( + prometheus.DefaultGatherer, + promhttp.HandlerOpts{ + ErrorLog: log.NewErrorLogger(), + ErrorHandling: promhttp.ContinueOnError, + }, + ) + + return handler, nil +} From 1abd36457144cc8c151038159a72ad659d6c88a3 Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Tue, 16 Jul 2019 19:59:49 +0300 Subject: [PATCH 03/11] PMM-1901 Remove TODOs. --- basic/metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/basic/metrics.go b/basic/metrics.go index 4247460e..7360ed73 100644 --- a/basic/metrics.go +++ b/basic/metrics.go @@ -107,7 +107,7 @@ var Metrics = []Metric{ { Name: "CPUUtilization", Desc: prometheus.NewDesc( - "node_cpu_average", // TODO: Conflict... + "node_cpu_average", "The percentage of CPU utilization. Units: Percent", []string{"instance", "region"}, map[string]string{"cpu": "All", "mode": "total"}, @@ -233,7 +233,7 @@ var Metrics = []Metric{ { Name: "FreeStorageSpace", Desc: prometheus.NewDesc( - "node_filesystem_free", // TODO: Conflict... + "node_filesystem_free", "The amount of available storage space. Units: Bytes", []string{"instance", "region"}, map[string]string(nil), @@ -242,7 +242,7 @@ var Metrics = []Metric{ { Name: "FreeableMemory", Desc: prometheus.NewDesc( - "node_memory_Cached", // TODO: Conflict... + "node_memory_Cached", "The amount of available random access memory. Units: Bytes", []string{"instance", "region"}, map[string]string(nil), From 50e61e4d678f78f763174f315c62ff5e694aa448 Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Tue, 16 Jul 2019 20:04:00 +0300 Subject: [PATCH 04/11] PMM-1901 Fix comment. --- basic/basic.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/basic/basic.go b/basic/basic.go index da23333c..78e7b483 100644 --- a/basic/basic.go +++ b/basic/basic.go @@ -42,6 +42,8 @@ func New(config *config.Config, sessions *sessions.Sessions) *Exporter { } } +// ExcludeMetrics exclude some metrics from collector. +// This need for resolve metric names conflicts when registering basic and enhanced collectors. func (e *Exporter) ExcludeMetrics(metrics ...string) { var filtered []Metric for _, v := range e.metrics { From c17fe8d57d5981c7944452835d39f8588558b387 Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Fri, 19 Jul 2019 18:59:31 +0300 Subject: [PATCH 05/11] PMM-1901 Fix PR suggestions related to overlapped metrics. --- basic/basic.go | 33 ++++++++------------------ basic/basic_test.go | 42 +++++++++++++++++++++++++++++---- basic/metrics.go | 57 ++++++++++++++++++++++++--------------------- factory/factory.go | 46 ++++++++++++++++++++++++++++++++++++ main.go | 42 ++++++++++++++------------------- 5 files changed, 142 insertions(+), 78 deletions(-) create mode 100644 factory/factory.go diff --git a/basic/basic.go b/basic/basic.go index 78e7b483..5a2b4531 100644 --- a/basic/basic.go +++ b/basic/basic.go @@ -33,36 +33,23 @@ type Exporter struct { } // New creates a new instance of a Exporter. -func New(config *config.Config, sessions *sessions.Sessions) *Exporter { +// enableOverlapping is using for backward compatibility. +// See: https://jira.percona.com/browse/PMM-1901. +func New(config *config.Config, sessions *sessions.Sessions, enableOverlapping bool) *Exporter { + var m []Metric + m = append(m, Metrics...) + if enableOverlapping { + m = append(m, metricsOverlappingWithEnhancedCollector...) + } + return &Exporter{ config: config, sessions: sessions, - metrics: Metrics, + metrics: m, l: log.With("component", "basic"), } } -// ExcludeMetrics exclude some metrics from collector. -// This need for resolve metric names conflicts when registering basic and enhanced collectors. -func (e *Exporter) ExcludeMetrics(metrics ...string) { - var filtered []Metric - for _, v := range e.metrics { - if !contains(v.Name, metrics) { - filtered = append(filtered, v) - } - } - e.metrics = filtered -} - -func contains(metricName string, metrics []string) bool { - for _, v := range metrics { - if v == metricName { - return true - } - } - return false -} - func (e *Exporter) Collect(ch chan<- prometheus.Metric) { now := time.Now() e.collect(ch) diff --git a/basic/basic_test.go b/basic/basic_test.go index 19c95597..025468d1 100644 --- a/basic/basic_test.go +++ b/basic/basic_test.go @@ -13,7 +13,7 @@ import ( "github.com/percona/rds_exporter/sessions" ) -func getExporter(t *testing.T) *Exporter { +func getExporter(t *testing.T, enableOverlapping bool) *Exporter { t.Helper() cfg, err := config.Load("../config.yml") @@ -21,11 +21,28 @@ func getExporter(t *testing.T) *Exporter { client := client.New() sess, err := sessions.New(cfg.Instances, client.HTTP(), false) require.NoError(t, err) - return New(cfg, sess) + return New(cfg, sess, enableOverlapping) } func TestCollector_Describe(t *testing.T) { - c := getExporter(t) + c := getExporter(t, false) + ch := make(chan *prometheus.Desc) + go func() { + c.Describe(ch) + close(ch) + }() + + const expected = 47 + descs := make([]*prometheus.Desc, 0, expected) + for d := range ch { + descs = append(descs, d) + } + + assert.Equal(t, expected, len(descs), "%+v", descs) +} + +func TestCollector_Describe_WithOverlappingMetrics(t *testing.T) { + c := getExporter(t, true) ch := make(chan *prometheus.Desc) go func() { c.Describe(ch) @@ -42,7 +59,24 @@ func TestCollector_Describe(t *testing.T) { } func TestCollector_Collect(t *testing.T) { - c := getExporter(t) + c := getExporter(t, false) + ch := make(chan prometheus.Metric) + go func() { + c.Collect(ch) + close(ch) + }() + + const expected = 91 + metrics := make([]helpers.Metric, 0, expected) + for m := range ch { + metrics = append(metrics, *helpers.ReadMetric(m)) + } + + assert.Equal(t, expected, len(metrics), "%+v", metrics) +} + +func TestCollector_Collect_WithOverlappingMetrics(t *testing.T) { + c := getExporter(t, true) ch := make(chan prometheus.Metric) go func() { c.Collect(ch) diff --git a/basic/metrics.go b/basic/metrics.go index 7360ed73..75ff82c1 100644 --- a/basic/metrics.go +++ b/basic/metrics.go @@ -104,15 +104,6 @@ var Metrics = []Metric{ map[string]string(nil), ), }, - { - Name: "CPUUtilization", - Desc: prometheus.NewDesc( - "node_cpu_average", - "The percentage of CPU utilization. Units: Percent", - []string{"instance", "region"}, - map[string]string{"cpu": "All", "mode": "total"}, - ), - }, { Name: "CommitLatency", Desc: prometheus.NewDesc( @@ -230,24 +221,6 @@ var Metrics = []Metric{ map[string]string(nil), ), }, - { - Name: "FreeStorageSpace", - Desc: prometheus.NewDesc( - "node_filesystem_free", - "The amount of available storage space. Units: Bytes", - []string{"instance", "region"}, - map[string]string(nil), - ), - }, - { - Name: "FreeableMemory", - Desc: prometheus.NewDesc( - "node_memory_Cached", - "The amount of available random access memory. Units: Bytes", - []string{"instance", "region"}, - map[string]string(nil), - ), - }, { Name: "InsertLatency", Desc: prometheus.NewDesc( @@ -447,3 +420,33 @@ var Metrics = []Metric{ ), }, } + +var metricsOverlappingWithEnhancedCollector = []Metric{ + { + Name: "CPUUtilization", + Desc: prometheus.NewDesc( + "node_cpu_average", + "The percentage of CPU utilization. Units: Percent", + []string{"instance", "region"}, + map[string]string{"cpu": "All", "mode": "total"}, + ), + }, + { + Name: "FreeStorageSpace", + Desc: prometheus.NewDesc( + "node_filesystem_free", + "The amount of available storage space. Units: Bytes", + []string{"instance", "region"}, + map[string]string(nil), + ), + }, + { + Name: "FreeableMemory", + Desc: prometheus.NewDesc( + "node_memory_Cached", + "The amount of available random access memory. Units: Bytes", + []string{"instance", "region"}, + map[string]string(nil), + ), + }, +} diff --git a/factory/factory.go b/factory/factory.go new file mode 100644 index 00000000..f6a5c406 --- /dev/null +++ b/factory/factory.go @@ -0,0 +1,46 @@ +package factory + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/percona/rds_exporter/basic" + "github.com/percona/rds_exporter/config" + "github.com/percona/rds_exporter/sessions" +) + +// Collectors uses for creating collectors on fly. +type Collectors struct { + config *config.Config + sessions *sessions.Sessions + predefined map[string]prometheus.Collector +} + +// New creates collectors factory. +func New(cfg *config.Config, sess *sessions.Sessions, predefined map[string]prometheus.Collector) *Collectors { + return &Collectors{ + config: cfg, + sessions: sess, + predefined: predefined, + } +} + +// Create creates collectors map based on filters list. +func (f *Collectors) Create(filters []string) map[string]prometheus.Collector { + c := make(map[string]prometheus.Collector) + + // When we have only 1 filter and this is basic one, we need it with all metrics. + if len(filters) == 1 && filters[0] == "basic" { + c["basic"] = basic.New(f.config, f.sessions, true) + return c + } + // When we have no filters, all collectors will be enabled, so create "basic" one without overlapping metrics. + if len(filters) == 0 { + c["basic"] = basic.New(f.config, f.sessions, false) + } + // Just adding all predefined collectors in map. + for name, collector := range f.predefined { + c[name] = collector + } + + return c +} diff --git a/main.go b/main.go index 9fcb3cd8..728f06ba 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "github.com/percona/rds_exporter/client" "github.com/percona/rds_exporter/config" "github.com/percona/rds_exporter/enhanced" + "github.com/percona/rds_exporter/factory" "github.com/percona/rds_exporter/sessions" ) @@ -47,7 +48,7 @@ func main() { // basic metrics + client metrics + exporter own metrics (ProcessCollector and GoCollector) // NOTE: This handler was retained for backward compatibility. See: https://jira.percona.com/browse/PMM-1901. { - basicCollector := basic.New(cfg, sess) + basicCollector := basic.New(cfg, sess, true) registry := prometheus.NewRegistry() registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) // from prometheus.DefaultGatherer @@ -77,15 +78,9 @@ func main() { // all metrics (with filtering) { - // Create separate instance of basic collector and remove metrics which cross with enhanced collector. - basicCollector := basic.New(cfg, sess) - basicCollector.ExcludeMetrics("CPUUtilization", "FreeStorageSpace", "FreeableMemory") - - prometheus.MustRegister(client) - http.Handle("/metrics", newHandler(map[string]prometheus.Collector{ - "basic": basicCollector, - "enhanced": enhancedCollector, - })) + f := factory.New(cfg, sess, map[string]prometheus.Collector{"enhanced": enhancedCollector, "client": client}) + handler := newHandler(f) + http.Handle("/metrics", handler) } log.Infof("Metrics: http://%s%s", *listenAddressF, "/metrics") @@ -97,11 +92,11 @@ func main() { // newHandler. It used for collectors filtering. type handler struct { unfilteredHandler http.Handler - collectors map[string]prometheus.Collector + factory *factory.Collectors } -func newHandler(collectors map[string]prometheus.Collector) *handler { - h := &handler{collectors: collectors} +func newHandler(factory *factory.Collectors) *handler { + h := &handler{factory: factory} if innerHandler, err := h.innerHandler(); err != nil { log.Fatalf("Couldn't create metrics handler: %s", err) } else { @@ -132,17 +127,16 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func (h *handler) innerHandler(filters ...string) (http.Handler, error) { - // clear all previously registered collector (to prevent repeatable registration). - for k, v := range h.collectors { - if ok := prometheus.Unregister(v); ok { - log.Infof("Collector '%s' was unregistered", k) - } - } + registry := prometheus.NewRegistry() + registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) // from prometheus.DefaultGatherer + registry.MustRegister(prometheus.NewGoCollector()) // from prometheus.DefaultGatherer + + collectors := h.factory.Create(filters) // register all collectors by default. if len(filters) == 0 { - for name, c := range h.collectors { - if err := prometheus.Register(c); err != nil { + for name, c := range collectors { + if err := registry.Register(c); err != nil { return nil, err } log.Infof("Collector '%s' was registered", name) @@ -151,8 +145,8 @@ func (h *handler) innerHandler(filters ...string) (http.Handler, error) { // register only filtered collectors. for _, name := range filters { - if c, ok := h.collectors[name]; ok { - if err := prometheus.Register(c); err != nil { + if c, ok := collectors[name]; ok { + if err := registry.Register(c); err != nil { return nil, err } log.Infof("Collector '%s' was registered", name) @@ -160,7 +154,7 @@ func (h *handler) innerHandler(filters ...string) (http.Handler, error) { } handler := promhttp.HandlerFor( - prometheus.DefaultGatherer, + registry, promhttp.HandlerOpts{ ErrorLog: log.NewErrorLogger(), ErrorHandling: promhttp.ContinueOnError, From 834396b1927c804f33cebb4ec1a3805a175eb7a6 Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Fri, 19 Jul 2019 19:10:21 +0300 Subject: [PATCH 06/11] PMM-1901 Fix linter errors. --- basic/basic.go | 2 +- basic/metrics.go | 2 +- main.go | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/basic/basic.go b/basic/basic.go index 5a2b4531..87523c18 100644 --- a/basic/basic.go +++ b/basic/basic.go @@ -39,7 +39,7 @@ func New(config *config.Config, sessions *sessions.Sessions, enableOverlapping b var m []Metric m = append(m, Metrics...) if enableOverlapping { - m = append(m, metricsOverlappingWithEnhancedCollector...) + m = append(m, MetricsOverlappingWithEnhancedCollector...) } return &Exporter{ diff --git a/basic/metrics.go b/basic/metrics.go index 75ff82c1..7796cffe 100644 --- a/basic/metrics.go +++ b/basic/metrics.go @@ -421,7 +421,7 @@ var Metrics = []Metric{ }, } -var metricsOverlappingWithEnhancedCollector = []Metric{ +var MetricsOverlappingWithEnhancedCollector = []Metric{ { Name: "CPUUtilization", Desc: prometheus.NewDesc( diff --git a/main.go b/main.go index 728f06ba..f9407676 100644 --- a/main.go +++ b/main.go @@ -120,7 +120,10 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err != nil { log.Warnln("Couldn't create filtered metrics handler:", err) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(fmt.Sprintf("Couldn't create filtered metrics handler: %s", err))) + _, err := w.Write([]byte(fmt.Sprintf("Couldn't create filtered metrics handler: %s", err))) + if err != nil { + log.Errorln(err) + } return } filteredHandler.ServeHTTP(w, r) From a907a9d13067159c3bcb1947ea18af7b10d734ef Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Fri, 19 Jul 2019 19:16:15 +0300 Subject: [PATCH 07/11] PMM-1901 Fix linter errors. --- basic/metrics.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/basic/metrics.go b/basic/metrics.go index 7796cffe..7615f9a6 100644 --- a/basic/metrics.go +++ b/basic/metrics.go @@ -421,6 +421,8 @@ var Metrics = []Metric{ }, } +// MetricsOverlappingWithEnhancedCollector metrics which overlapping with enhanced collector. +// See: https://jira.percona.com/browse/PMM-1901. var MetricsOverlappingWithEnhancedCollector = []Metric{ { Name: "CPUUtilization", From 39ed8c3d2e33e9ce96896433ab48bd2d214e9a9f Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Tue, 30 Jul 2019 13:33:58 +0300 Subject: [PATCH 08/11] PMM-1901 Factory refactoring. --- basic/basic.go | 14 ++++++++++++-- basic/basic_test.go | 12 ++++++------ factory/factory.go | 27 ++++++++++++++++++++++----- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/basic/basic.go b/basic/basic.go index 87523c18..f79a9957 100644 --- a/basic/basic.go +++ b/basic/basic.go @@ -20,6 +20,16 @@ var ( ) ) +// OverlappingMetrics flag. +type OverlappingMetrics bool + +const ( + // EnableOverlapping flag for enabling overlapping version. + EnableOverlapping OverlappingMetrics = true + // DisableOverlapping flag for disabling overlapping version. + DisableOverlapping OverlappingMetrics = false +) + type Metric struct { Name string Desc *prometheus.Desc @@ -35,10 +45,10 @@ type Exporter struct { // New creates a new instance of a Exporter. // enableOverlapping is using for backward compatibility. // See: https://jira.percona.com/browse/PMM-1901. -func New(config *config.Config, sessions *sessions.Sessions, enableOverlapping bool) *Exporter { +func New(config *config.Config, sessions *sessions.Sessions, enableMetrics OverlappingMetrics) *Exporter { var m []Metric m = append(m, Metrics...) - if enableOverlapping { + if enableMetrics { m = append(m, MetricsOverlappingWithEnhancedCollector...) } diff --git a/basic/basic_test.go b/basic/basic_test.go index 025468d1..e3596b74 100644 --- a/basic/basic_test.go +++ b/basic/basic_test.go @@ -13,7 +13,7 @@ import ( "github.com/percona/rds_exporter/sessions" ) -func getExporter(t *testing.T, enableOverlapping bool) *Exporter { +func getExporter(t *testing.T, enableMetrics OverlappingMetrics) *Exporter { t.Helper() cfg, err := config.Load("../config.yml") @@ -21,11 +21,11 @@ func getExporter(t *testing.T, enableOverlapping bool) *Exporter { client := client.New() sess, err := sessions.New(cfg.Instances, client.HTTP(), false) require.NoError(t, err) - return New(cfg, sess, enableOverlapping) + return New(cfg, sess, enableMetrics) } func TestCollector_Describe(t *testing.T) { - c := getExporter(t, false) + c := getExporter(t, DisableOverlapping) ch := make(chan *prometheus.Desc) go func() { c.Describe(ch) @@ -42,7 +42,7 @@ func TestCollector_Describe(t *testing.T) { } func TestCollector_Describe_WithOverlappingMetrics(t *testing.T) { - c := getExporter(t, true) + c := getExporter(t, EnableOverlapping) ch := make(chan *prometheus.Desc) go func() { c.Describe(ch) @@ -59,7 +59,7 @@ func TestCollector_Describe_WithOverlappingMetrics(t *testing.T) { } func TestCollector_Collect(t *testing.T) { - c := getExporter(t, false) + c := getExporter(t, DisableOverlapping) ch := make(chan prometheus.Metric) go func() { c.Collect(ch) @@ -76,7 +76,7 @@ func TestCollector_Collect(t *testing.T) { } func TestCollector_Collect_WithOverlappingMetrics(t *testing.T) { - c := getExporter(t, true) + c := getExporter(t, EnableOverlapping) ch := make(chan prometheus.Metric) go func() { c.Collect(ch) diff --git a/factory/factory.go b/factory/factory.go index f6a5c406..c7e1b994 100644 --- a/factory/factory.go +++ b/factory/factory.go @@ -28,14 +28,22 @@ func New(cfg *config.Config, sess *sessions.Sessions, predefined map[string]prom func (f *Collectors) Create(filters []string) map[string]prometheus.Collector { c := make(map[string]prometheus.Collector) + // When we have no filters, all collectors will be enabled, so create "basic" one without overlapping metrics. + if len(filters) == 0 { + c["basic"] = basic.New(f.config, f.sessions, basic.DisableOverlapping) + } // When we have only 1 filter and this is basic one, we need it with all metrics. - if len(filters) == 1 && filters[0] == "basic" { - c["basic"] = basic.New(f.config, f.sessions, true) + if len(filters) == 1 && filterIn(filters, "basic") { + c["basic"] = basic.New(f.config, f.sessions, basic.EnableOverlapping) return c } - // When we have no filters, all collectors will be enabled, so create "basic" one without overlapping metrics. - if len(filters) == 0 { - c["basic"] = basic.New(f.config, f.sessions, false) + // When we have more than 1 filters and have basic one... + if len(filters) > 1 && filterIn(filters, "basic") { + if filterIn(filters, "enhanced") { + c["basic"] = basic.New(f.config, f.sessions, basic.DisableOverlapping) + } else { + c["basic"] = basic.New(f.config, f.sessions, basic.EnableOverlapping) + } } // Just adding all predefined collectors in map. for name, collector := range f.predefined { @@ -44,3 +52,12 @@ func (f *Collectors) Create(filters []string) map[string]prometheus.Collector { return c } + +func filterIn(slice []string, filter string) bool { + for _, v := range slice { + if v == filter { + return true + } + } + return false +} From 901675906e71d20cac23868d6749358d2ee206ea Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Tue, 30 Jul 2019 13:35:37 +0300 Subject: [PATCH 09/11] PMM-1901 Comment fix. --- basic/basic.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/basic/basic.go b/basic/basic.go index f79a9957..007af36c 100644 --- a/basic/basic.go +++ b/basic/basic.go @@ -24,9 +24,9 @@ var ( type OverlappingMetrics bool const ( - // EnableOverlapping flag for enabling overlapping version. + // EnableOverlapping flag for enabling overlapping metrics. EnableOverlapping OverlappingMetrics = true - // DisableOverlapping flag for disabling overlapping version. + // DisableOverlapping flag for disabling overlapping metrics. DisableOverlapping OverlappingMetrics = false ) From 57f1f3604d93d906e1f84a7e4f76e207cd50a55a Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Wed, 31 Jul 2019 19:20:44 +0300 Subject: [PATCH 10/11] PMM-1901 PR comments fixes. --- basic/basic.go | 4 ++-- main.go | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/basic/basic.go b/basic/basic.go index 007af36c..4b8e9f9e 100644 --- a/basic/basic.go +++ b/basic/basic.go @@ -45,10 +45,10 @@ type Exporter struct { // New creates a new instance of a Exporter. // enableOverlapping is using for backward compatibility. // See: https://jira.percona.com/browse/PMM-1901. -func New(config *config.Config, sessions *sessions.Sessions, enableMetrics OverlappingMetrics) *Exporter { +func New(config *config.Config, sessions *sessions.Sessions, enableOverlapping OverlappingMetrics) *Exporter { var m []Metric m = append(m, Metrics...) - if enableMetrics { + if enableOverlapping { m = append(m, MetricsOverlappingWithEnhancedCollector...) } diff --git a/main.go b/main.go index f9407676..ea43e005 100644 --- a/main.go +++ b/main.go @@ -78,7 +78,14 @@ func main() { // all metrics (with filtering) { - f := factory.New(cfg, sess, map[string]prometheus.Collector{"enhanced": enhancedCollector, "client": client}) + psCollector := prometheus.NewProcessCollector(os.Getpid(), "") // from prometheus.DefaultGatherer + goCollector := prometheus.NewGoCollector() // from prometheus.DefaultGatherer + f := factory.New(cfg, sess, map[string]prometheus.Collector{ + "enhanced": enhancedCollector, + "client": client, + "standard.process": psCollector, + "standard.go": goCollector, + }) handler := newHandler(f) http.Handle("/metrics", handler) } @@ -97,11 +104,12 @@ type handler struct { func newHandler(factory *factory.Collectors) *handler { h := &handler{factory: factory} - if innerHandler, err := h.innerHandler(); err != nil { + innerHandler, err := h.innerHandler() + if err != nil { log.Fatalf("Couldn't create metrics handler: %s", err) - } else { - h.unfilteredHandler = innerHandler } + + h.unfilteredHandler = innerHandler return h } @@ -131,8 +139,6 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *handler) innerHandler(filters ...string) (http.Handler, error) { registry := prometheus.NewRegistry() - registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) // from prometheus.DefaultGatherer - registry.MustRegister(prometheus.NewGoCollector()) // from prometheus.DefaultGatherer collectors := h.factory.Create(filters) From 0f01209040015854a2fec15088e89aa1ab9a95d9 Mon Sep 17 00:00:00 2001 From: Anton Kucherov Date: Wed, 31 Jul 2019 19:24:47 +0300 Subject: [PATCH 11/11] PMM-1901 Change golangci-lint parameters. --- .golangci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index d9728cb4..f50f1e66 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -13,6 +13,9 @@ linters-settings: linters: enable-all: true + disable: + - scopelint # too many false positives + - gochecknoglobals # mostly useless issues: exclude-use-default: false