From 7e56086cafb85e4f68acc2187e4f3905f55f27f6 Mon Sep 17 00:00:00 2001 From: AndrewChubatiuk Date: Fri, 21 Jun 2024 13:57:32 +0300 Subject: [PATCH] vmpodscrape/vmservicescrape/vmnodescrape/vmstaticscrape: support scrape_protocols option --- api/v1beta1/vmnodescrape_types.go | 5 ++++- api/v1beta1/vmpodscrape_types.go | 5 ++++- api/v1beta1/vmservicescrape_types.go | 5 ++++- api/v1beta1/vmstaticscrape_types.go | 5 ++++- api/v1beta1/zz_generated.deepcopy.go | 20 +++++++++++++++++++ .../v1beta1/vmnodescrape_types.go | 5 ++++- .../v1beta1/vmpodscrape_types.go | 5 ++++- .../v1beta1/vmservicescrape_types.go | 5 ++++- .../v1beta1/vmstaticscrape_types.go | 5 ++++- ...tor.victoriametrics.com_vmnodescrapes.yaml | 7 ++++++- ...ator.victoriametrics.com_vmpodscrapes.yaml | 7 ++++++- ....victoriametrics.com_vmservicescrapes.yaml | 7 ++++++- ...r.victoriametrics.com_vmstaticscrapes.yaml | 7 ++++++- docs/CHANGELOG.md | 2 ++ .../controller/factory/vmagent/nodescrape.go | 4 +++- .../factory/vmagent/nodescrape_test.go | 3 +++ .../controller/factory/vmagent/podscrape.go | 3 +++ .../factory/vmagent/podscrape_test.go | 7 +++++++ .../factory/vmagent/servicescrape.go | 3 +++ .../factory/vmagent/servicescrape_test.go | 7 +++++++ .../factory/vmagent/staticscrape.go | 4 ++++ 21 files changed, 108 insertions(+), 13 deletions(-) diff --git a/api/v1beta1/vmnodescrape_types.go b/api/v1beta1/vmnodescrape_types.go index ba64cd7ab..f5594e0eb 100644 --- a/api/v1beta1/vmnodescrape_types.go +++ b/api/v1beta1/vmnodescrape_types.go @@ -38,9 +38,12 @@ type VMNodeScrapeSpec struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/v1beta1/vmpodscrape_types.go b/api/v1beta1/vmpodscrape_types.go index dfb20332b..5fb8806ee 100644 --- a/api/v1beta1/vmpodscrape_types.go +++ b/api/v1beta1/vmpodscrape_types.go @@ -97,7 +97,7 @@ type PodMetricsEndpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` // SampleLimit defines per-podEndpoint limit on number of scraped samples that will be accepted. @@ -107,6 +107,9 @@ type PodMetricsEndpoint struct { // a single target can expose during all the scrapes on the time window of 24h. // +optional SeriesLimit uint64 `json:"seriesLimit,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // HonorLabels chooses the metric's labels on collisions with target labels. // +optional HonorLabels bool `json:"honorLabels,omitempty"` diff --git a/api/v1beta1/vmservicescrape_types.go b/api/v1beta1/vmservicescrape_types.go index 8ac42535a..ff13035fb 100644 --- a/api/v1beta1/vmservicescrape_types.go +++ b/api/v1beta1/vmservicescrape_types.go @@ -140,7 +140,7 @@ type Endpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` // SampleLimit defines per-endpoint limit on number of scraped samples that will be accepted. @@ -150,6 +150,9 @@ type Endpoint struct { // a single target can expose during all the scrapes on the time window of 24h. // +optional SeriesLimit uint64 `json:"seriesLimit,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/v1beta1/vmstaticscrape_types.go b/api/v1beta1/vmstaticscrape_types.go index 1f46e64a1..5b651b035 100644 --- a/api/v1beta1/vmstaticscrape_types.go +++ b/api/v1beta1/vmstaticscrape_types.go @@ -60,9 +60,12 @@ type TargetEndpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0a79af1a6..a501793d8 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -844,6 +844,11 @@ func (in *Endpoint) DeepCopyInto(out *Endpoint) { *out = new(bool) **out = **in } + if in.ScrapeProtocols != nil { + in, out := &in.ScrapeProtocols, &out.ScrapeProtocols + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.OAuth2 != nil { in, out := &in.OAuth2, &out.OAuth2 *out = new(OAuth2) @@ -1638,6 +1643,11 @@ func (in *PodMetricsEndpoint) DeepCopyInto(out *PodMetricsEndpoint) { *out = new(bool) **out = **in } + if in.ScrapeProtocols != nil { + in, out := &in.ScrapeProtocols, &out.ScrapeProtocols + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.HonorTimestamps != nil { in, out := &in.HonorTimestamps, &out.HonorTimestamps *out = new(bool) @@ -2648,6 +2658,11 @@ func (in *TargetEndpoint) DeepCopyInto(out *TargetEndpoint) { *out = new(bool) **out = **in } + if in.ScrapeProtocols != nil { + in, out := &in.ScrapeProtocols, &out.ScrapeProtocols + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.OAuth2 != nil { in, out := &in.OAuth2, &out.OAuth2 *out = new(OAuth2) @@ -5165,6 +5180,11 @@ func (in *VMNodeScrapeSpec) DeepCopyInto(out *VMNodeScrapeSpec) { *out = new(bool) **out = **in } + if in.ScrapeProtocols != nil { + in, out := &in.ScrapeProtocols, &out.ScrapeProtocols + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.OAuth2 != nil { in, out := &in.OAuth2, &out.OAuth2 *out = new(OAuth2) diff --git a/api/victoriametrics/v1beta1/vmnodescrape_types.go b/api/victoriametrics/v1beta1/vmnodescrape_types.go index ba64cd7ab..f5594e0eb 100644 --- a/api/victoriametrics/v1beta1/vmnodescrape_types.go +++ b/api/victoriametrics/v1beta1/vmnodescrape_types.go @@ -38,9 +38,12 @@ type VMNodeScrapeSpec struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/victoriametrics/v1beta1/vmpodscrape_types.go b/api/victoriametrics/v1beta1/vmpodscrape_types.go index dfb20332b..9d37623e1 100644 --- a/api/victoriametrics/v1beta1/vmpodscrape_types.go +++ b/api/victoriametrics/v1beta1/vmpodscrape_types.go @@ -97,9 +97,12 @@ type PodMetricsEndpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // SampleLimit defines per-podEndpoint limit on number of scraped samples that will be accepted. // +optional SampleLimit uint64 `json:"sampleLimit,omitempty"` diff --git a/api/victoriametrics/v1beta1/vmservicescrape_types.go b/api/victoriametrics/v1beta1/vmservicescrape_types.go index 8ac42535a..c37228f4e 100644 --- a/api/victoriametrics/v1beta1/vmservicescrape_types.go +++ b/api/victoriametrics/v1beta1/vmservicescrape_types.go @@ -140,9 +140,12 @@ type Endpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // SampleLimit defines per-endpoint limit on number of scraped samples that will be accepted. // +optional SampleLimit uint64 `json:"sampleLimit,omitempty"` diff --git a/api/victoriametrics/v1beta1/vmstaticscrape_types.go b/api/victoriametrics/v1beta1/vmstaticscrape_types.go index 1f46e64a1..5b651b035 100644 --- a/api/victoriametrics/v1beta1/vmstaticscrape_types.go +++ b/api/victoriametrics/v1beta1/vmstaticscrape_types.go @@ -60,9 +60,12 @@ type TargetEndpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml index 06bd34a77..3f5ca741a 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml @@ -425,8 +425,13 @@ spec: ScrapeInterval is the same as Interval and has priority over it. one of scrape_interval or interval can be used type: string + scrape_protocols: + description: ScrapeProtocols defines Prometheus scrape protocol. + items: + type: string + type: array scrapeTimeout: - description: Timeout after which the scrape is ended + description: ScrapeTimeout after which the scrape is ended type: string selector: description: Selector to select kubernetes Nodes. diff --git a/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml index bd99808f3..74ac6ff2c 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml @@ -468,8 +468,13 @@ spec: ScrapeInterval is the same as Interval and has priority over it. one of scrape_interval or interval can be used type: string + scrape_protocols: + description: ScrapeProtocols defines Prometheus scrape protocol. + items: + type: string + type: array scrapeTimeout: - description: Timeout after which the scrape is ended + description: ScrapeTimeout after which the scrape is ended type: string seriesLimit: description: |- diff --git a/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml index 0128243e2..820959935 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml @@ -458,8 +458,13 @@ spec: ScrapeInterval is the same as Interval and has priority over it. one of scrape_interval or interval can be used type: string + scrape_protocols: + description: ScrapeProtocols defines Prometheus scrape protocol. + items: + type: string + type: array scrapeTimeout: - description: Timeout after which the scrape is ended + description: ScrapeTimeout after which the scrape is ended type: string seriesLimit: description: |- diff --git a/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml index 1dca800cf..a154bb5b0 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml @@ -442,8 +442,13 @@ spec: ScrapeInterval is the same as Interval and has priority over it. one of scrape_interval or interval can be used type: string + scrape_protocols: + description: ScrapeProtocols defines Prometheus scrape protocol. + items: + type: string + type: array scrapeTimeout: - description: Timeout after which the scrape is ended + description: ScrapeTimeout after which the scrape is ended type: string seriesLimit: description: |- diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d7015596e..aed0c34bd 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -27,6 +27,8 @@ aliases: - [vmalertmanagerconfig](./api.md#vmalertmanagerconfig): adds missing `handleReconcileErr` callback to the reconcile loop. It must properly handle errors and deregister objects. +- [vmservicescrape](./api.md#vmservicescrape), [vmpodscrape](./api.md#vmpodscrape), [vmnodescrape](./api.md#vmnodescrape) - added `scrape_protocols` parameter for scrape protocols configuration + ## [v0.45.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.45.0) - 10 Jun 2024 diff --git a/internal/controller/factory/vmagent/nodescrape.go b/internal/controller/factory/vmagent/nodescrape.go index a9b787b34..468410fdb 100644 --- a/internal/controller/factory/vmagent/nodescrape.go +++ b/internal/controller/factory/vmagent/nodescrape.go @@ -48,7 +48,9 @@ func generateNodeScrapeConfig( if scrapeInterval != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_interval", Value: scrapeInterval}) } - + if len(nodeSpec.ScrapeProtocols) != 0 { + cfg = append(cfg, yaml.MapItem{Key: "scrape_protocols", Value: nodeSpec.ScrapeProtocols}) + } if nodeSpec.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: nodeSpec.ScrapeTimeout}) } diff --git a/internal/controller/factory/vmagent/nodescrape_test.go b/internal/controller/factory/vmagent/nodescrape_test.go index a02348197..317ae3563 100644 --- a/internal/controller/factory/vmagent/nodescrape_test.go +++ b/internal/controller/factory/vmagent/nodescrape_test.go @@ -96,6 +96,7 @@ relabel_configs: FollowRedirects: pointer.Bool(true), ScrapeTimeout: "10s", ScrapeInterval: "5s", + ScrapeProtocols: []string{"PrometheusProto"}, Params: map[string][]string{"module": {"client"}}, JobLabel: "env", HonorTimestamps: pointer.Bool(true), @@ -134,6 +135,8 @@ honor_timestamps: true kubernetes_sd_configs: - role: node scrape_interval: 5s +scrape_protocols: +- PrometheusProto scrape_timeout: 10s metrics_path: /metrics proxy_url: https://some-url diff --git a/internal/controller/factory/vmagent/podscrape.go b/internal/controller/factory/vmagent/podscrape.go index fe20d5e43..c7d9735e1 100644 --- a/internal/controller/factory/vmagent/podscrape.go +++ b/internal/controller/factory/vmagent/podscrape.go @@ -54,6 +54,9 @@ func generatePodScrapeConfig( if scrapeInterval != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_interval", Value: scrapeInterval}) } + if len(ep.ScrapeProtocols) != 0 { + cfg = append(cfg, yaml.MapItem{Key: "scrape_protocols", Value: ep.ScrapeProtocols}) + } if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) } diff --git a/internal/controller/factory/vmagent/podscrape_test.go b/internal/controller/factory/vmagent/podscrape_test.go index 50b8704b4..a89f43e01 100644 --- a/internal/controller/factory/vmagent/podscrape_test.go +++ b/internal/controller/factory/vmagent/podscrape_test.go @@ -158,6 +158,10 @@ relabel_configs: }, }, ep: vmv1beta1.PodMetricsEndpoint{ + ScrapeProtocols: []string{ + "PrometheusText", + "PrometheusProto", + }, Path: "/metric", Port: "web", }, @@ -170,6 +174,9 @@ kubernetes_sd_configs: selectors: - role: pod label: label-1=value-1,label-2=value-2,label-3=value-3 +scrape_protocols: +- PrometheusText +- PrometheusProto metrics_path: /metric relabel_configs: - action: drop diff --git a/internal/controller/factory/vmagent/servicescrape.go b/internal/controller/factory/vmagent/servicescrape.go index 463bef43e..e82948195 100644 --- a/internal/controller/factory/vmagent/servicescrape.go +++ b/internal/controller/factory/vmagent/servicescrape.go @@ -59,6 +59,9 @@ func generateServiceScrapeConfig( if scrapeInterval != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_interval", Value: scrapeInterval}) } + if len(ep.ScrapeProtocols) != 0 { + cfg = append(cfg, yaml.MapItem{Key: "scrape_protocols", Value: ep.ScrapeProtocols}) + } if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) } diff --git a/internal/controller/factory/vmagent/servicescrape_test.go b/internal/controller/factory/vmagent/servicescrape_test.go index 184d8d475..a4ce28f14 100644 --- a/internal/controller/factory/vmagent/servicescrape_test.go +++ b/internal/controller/factory/vmagent/servicescrape_test.go @@ -185,6 +185,10 @@ relabel_configs: }, BearerTokenFile: "/var/run/tolen", ScrapeInterval: "60m", + ScrapeProtocols: []string{ + "PrometheusText", + "OpenMetricsText", + }, }, i: 0, apiserverConfig: nil, @@ -202,6 +206,9 @@ kubernetes_sd_configs: names: - default scrape_interval: 40m +scrape_protocols: +- PrometheusText +- OpenMetricsText tls_config: insecure_skip_verify: false ca_file: /etc/vmagent-tls/certs/default_tls-secret_ca diff --git a/internal/controller/factory/vmagent/staticscrape.go b/internal/controller/factory/vmagent/staticscrape.go index 8b3bc2e41..6f78add97 100644 --- a/internal/controller/factory/vmagent/staticscrape.go +++ b/internal/controller/factory/vmagent/staticscrape.go @@ -50,6 +50,10 @@ func generateStaticScrapeConfig( cfg = append(cfg, yaml.MapItem{Key: "scrape_interval", Value: scrapeInterval}) } + if len(ep.ScrapeProtocols) != 0 { + cfg = append(cfg, yaml.MapItem{Key: "scrape_protocols", Value: ep.ScrapeProtocols}) + } + if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) }