From f137aa60e5194aa97195e27990433ae18d231b32 Mon Sep 17 00:00:00 2001 From: AndrewChubatiuk Date: Thu, 13 Jun 2024 09:01:08 +0300 Subject: [PATCH] vmpodscrape/vmservicescrape/vmnodescrape/vmstaticscrape: support scrape_protocols option --- api/v1beta1/vmnodescrape_types.go | 5 ++++- api/v1beta1/vmpodscrape_types.go | 5 ++++- api/v1beta1/vmservicescrape_types.go | 5 ++++- api/v1beta1/vmstaticscrape_types.go | 5 ++++- api/v1beta1/zz_generated.deepcopy.go | 20 +++++++++++++++++++ .../v1beta1/vmnodescrape_types.go | 5 ++++- .../v1beta1/vmpodscrape_types.go | 5 ++++- .../v1beta1/vmservicescrape_types.go | 5 ++++- .../v1beta1/vmstaticscrape_types.go | 5 ++++- ...tor.victoriametrics.com_vmnodescrapes.yaml | 7 ++++++- ...ator.victoriametrics.com_vmpodscrapes.yaml | 7 ++++++- ....victoriametrics.com_vmservicescrapes.yaml | 7 ++++++- ...r.victoriametrics.com_vmstaticscrapes.yaml | 7 ++++++- controllers/factory/vmagent/nodescrape.go | 4 +++- .../factory/vmagent/nodescrape_test.go | 3 +++ controllers/factory/vmagent/podscrape.go | 3 +++ controllers/factory/vmagent/podscrape_test.go | 7 +++++++ controllers/factory/vmagent/servicescrape.go | 3 +++ .../factory/vmagent/servicescrape_test.go | 7 +++++++ controllers/factory/vmagent/staticscrape.go | 4 ++++ docs/CHANGELOG.md | 2 ++ 21 files changed, 108 insertions(+), 13 deletions(-) diff --git a/api/v1beta1/vmnodescrape_types.go b/api/v1beta1/vmnodescrape_types.go index ba64cd7ab..f5594e0eb 100644 --- a/api/v1beta1/vmnodescrape_types.go +++ b/api/v1beta1/vmnodescrape_types.go @@ -38,9 +38,12 @@ type VMNodeScrapeSpec struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/v1beta1/vmpodscrape_types.go b/api/v1beta1/vmpodscrape_types.go index dfb20332b..5fb8806ee 100644 --- a/api/v1beta1/vmpodscrape_types.go +++ b/api/v1beta1/vmpodscrape_types.go @@ -97,7 +97,7 @@ type PodMetricsEndpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` // SampleLimit defines per-podEndpoint limit on number of scraped samples that will be accepted. @@ -107,6 +107,9 @@ type PodMetricsEndpoint struct { // a single target can expose during all the scrapes on the time window of 24h. // +optional SeriesLimit uint64 `json:"seriesLimit,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // HonorLabels chooses the metric's labels on collisions with target labels. // +optional HonorLabels bool `json:"honorLabels,omitempty"` diff --git a/api/v1beta1/vmservicescrape_types.go b/api/v1beta1/vmservicescrape_types.go index 8ac42535a..ff13035fb 100644 --- a/api/v1beta1/vmservicescrape_types.go +++ b/api/v1beta1/vmservicescrape_types.go @@ -140,7 +140,7 @@ type Endpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` // SampleLimit defines per-endpoint limit on number of scraped samples that will be accepted. @@ -150,6 +150,9 @@ type Endpoint struct { // a single target can expose during all the scrapes on the time window of 24h. // +optional SeriesLimit uint64 `json:"seriesLimit,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/v1beta1/vmstaticscrape_types.go b/api/v1beta1/vmstaticscrape_types.go index 1f46e64a1..5b651b035 100644 --- a/api/v1beta1/vmstaticscrape_types.go +++ b/api/v1beta1/vmstaticscrape_types.go @@ -60,9 +60,12 @@ type TargetEndpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0a79af1a6..a501793d8 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -844,6 +844,11 @@ func (in *Endpoint) DeepCopyInto(out *Endpoint) { *out = new(bool) **out = **in } + if in.ScrapeProtocols != nil { + in, out := &in.ScrapeProtocols, &out.ScrapeProtocols + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.OAuth2 != nil { in, out := &in.OAuth2, &out.OAuth2 *out = new(OAuth2) @@ -1638,6 +1643,11 @@ func (in *PodMetricsEndpoint) DeepCopyInto(out *PodMetricsEndpoint) { *out = new(bool) **out = **in } + if in.ScrapeProtocols != nil { + in, out := &in.ScrapeProtocols, &out.ScrapeProtocols + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.HonorTimestamps != nil { in, out := &in.HonorTimestamps, &out.HonorTimestamps *out = new(bool) @@ -2648,6 +2658,11 @@ func (in *TargetEndpoint) DeepCopyInto(out *TargetEndpoint) { *out = new(bool) **out = **in } + if in.ScrapeProtocols != nil { + in, out := &in.ScrapeProtocols, &out.ScrapeProtocols + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.OAuth2 != nil { in, out := &in.OAuth2, &out.OAuth2 *out = new(OAuth2) @@ -5165,6 +5180,11 @@ func (in *VMNodeScrapeSpec) DeepCopyInto(out *VMNodeScrapeSpec) { *out = new(bool) **out = **in } + if in.ScrapeProtocols != nil { + in, out := &in.ScrapeProtocols, &out.ScrapeProtocols + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.OAuth2 != nil { in, out := &in.OAuth2, &out.OAuth2 *out = new(OAuth2) diff --git a/api/victoriametrics/v1beta1/vmnodescrape_types.go b/api/victoriametrics/v1beta1/vmnodescrape_types.go index ba64cd7ab..f5594e0eb 100644 --- a/api/victoriametrics/v1beta1/vmnodescrape_types.go +++ b/api/victoriametrics/v1beta1/vmnodescrape_types.go @@ -38,9 +38,12 @@ type VMNodeScrapeSpec struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/victoriametrics/v1beta1/vmpodscrape_types.go b/api/victoriametrics/v1beta1/vmpodscrape_types.go index dfb20332b..9d37623e1 100644 --- a/api/victoriametrics/v1beta1/vmpodscrape_types.go +++ b/api/victoriametrics/v1beta1/vmpodscrape_types.go @@ -97,9 +97,12 @@ type PodMetricsEndpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // SampleLimit defines per-podEndpoint limit on number of scraped samples that will be accepted. // +optional SampleLimit uint64 `json:"sampleLimit,omitempty"` diff --git a/api/victoriametrics/v1beta1/vmservicescrape_types.go b/api/victoriametrics/v1beta1/vmservicescrape_types.go index 8ac42535a..c37228f4e 100644 --- a/api/victoriametrics/v1beta1/vmservicescrape_types.go +++ b/api/victoriametrics/v1beta1/vmservicescrape_types.go @@ -140,9 +140,12 @@ type Endpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // SampleLimit defines per-endpoint limit on number of scraped samples that will be accepted. // +optional SampleLimit uint64 `json:"sampleLimit,omitempty"` diff --git a/api/victoriametrics/v1beta1/vmstaticscrape_types.go b/api/victoriametrics/v1beta1/vmstaticscrape_types.go index 1f46e64a1..5b651b035 100644 --- a/api/victoriametrics/v1beta1/vmstaticscrape_types.go +++ b/api/victoriametrics/v1beta1/vmstaticscrape_types.go @@ -60,9 +60,12 @@ type TargetEndpoint struct { // one of scrape_interval or interval can be used // +optional ScrapeInterval string `json:"scrape_interval,omitempty"` - // Timeout after which the scrape is ended + // ScrapeTimeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // ScrapeProtocols defines Prometheus scrape protocol. + // +optional + ScrapeProtocols []string `json:"scrape_protocols,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml index 6e0129b00..bd5643343 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml @@ -453,8 +453,13 @@ spec: ScrapeInterval is the same as Interval and has priority over it. one of scrape_interval or interval can be used type: string + scrape_protocols: + description: ScrapeProtocols defines Prometheus scrape protocol. + items: + type: string + type: array scrapeTimeout: - description: Timeout after which the scrape is ended + description: ScrapeTimeout after which the scrape is ended type: string selector: description: Selector to select kubernetes Nodes. diff --git a/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml index adf35aa64..c1ec6b8c5 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml @@ -506,8 +506,13 @@ spec: ScrapeInterval is the same as Interval and has priority over it. one of scrape_interval or interval can be used type: string + scrape_protocols: + description: ScrapeProtocols defines Prometheus scrape protocol. + items: + type: string + type: array scrapeTimeout: - description: Timeout after which the scrape is ended + description: ScrapeTimeout after which the scrape is ended type: string seriesLimit: description: |- diff --git a/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml index 391c9be66..596205bb2 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml @@ -495,8 +495,13 @@ spec: ScrapeInterval is the same as Interval and has priority over it. one of scrape_interval or interval can be used type: string + scrape_protocols: + description: ScrapeProtocols defines Prometheus scrape protocol. + items: + type: string + type: array scrapeTimeout: - description: Timeout after which the scrape is ended + description: ScrapeTimeout after which the scrape is ended type: string seriesLimit: description: |- diff --git a/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml index 442bada47..3cc81b9a2 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml @@ -476,8 +476,13 @@ spec: ScrapeInterval is the same as Interval and has priority over it. one of scrape_interval or interval can be used type: string + scrape_protocols: + description: ScrapeProtocols defines Prometheus scrape protocol. + items: + type: string + type: array scrapeTimeout: - description: Timeout after which the scrape is ended + description: ScrapeTimeout after which the scrape is ended type: string seriesLimit: description: |- diff --git a/controllers/factory/vmagent/nodescrape.go b/controllers/factory/vmagent/nodescrape.go index ba0eeca44..e54cc9421 100644 --- a/controllers/factory/vmagent/nodescrape.go +++ b/controllers/factory/vmagent/nodescrape.go @@ -48,7 +48,9 @@ func generateNodeScrapeConfig( if scrapeInterval != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_interval", Value: scrapeInterval}) } - + if len(nodeSpec.ScrapeProtocols) != 0 { + cfg = append(cfg, yaml.MapItem{Key: "scrape_protocols", Value: nodeSpec.ScrapeProtocols}) + } if nodeSpec.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: nodeSpec.ScrapeTimeout}) } diff --git a/controllers/factory/vmagent/nodescrape_test.go b/controllers/factory/vmagent/nodescrape_test.go index a8ab2a256..fbb9cd435 100644 --- a/controllers/factory/vmagent/nodescrape_test.go +++ b/controllers/factory/vmagent/nodescrape_test.go @@ -96,6 +96,7 @@ relabel_configs: FollowRedirects: pointer.Bool(true), ScrapeTimeout: "10s", ScrapeInterval: "5s", + ScrapeProtocols: []string{"PrometheusProto"}, Params: map[string][]string{"module": {"client"}}, JobLabel: "env", HonorTimestamps: pointer.Bool(true), @@ -134,6 +135,8 @@ honor_timestamps: true kubernetes_sd_configs: - role: node scrape_interval: 5s +scrape_protocols: +- PrometheusProto scrape_timeout: 10s metrics_path: /metrics proxy_url: https://some-url diff --git a/controllers/factory/vmagent/podscrape.go b/controllers/factory/vmagent/podscrape.go index 21878cbab..58d394235 100644 --- a/controllers/factory/vmagent/podscrape.go +++ b/controllers/factory/vmagent/podscrape.go @@ -54,6 +54,9 @@ func generatePodScrapeConfig( if scrapeInterval != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_interval", Value: scrapeInterval}) } + if len(ep.ScrapeProtocols) != 0 { + cfg = append(cfg, yaml.MapItem{Key: "scrape_protocols", Value: ep.ScrapeProtocols}) + } if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) } diff --git a/controllers/factory/vmagent/podscrape_test.go b/controllers/factory/vmagent/podscrape_test.go index 4191fae29..034eaf10e 100644 --- a/controllers/factory/vmagent/podscrape_test.go +++ b/controllers/factory/vmagent/podscrape_test.go @@ -158,6 +158,10 @@ relabel_configs: }, }, ep: victoriametricsv1beta1.PodMetricsEndpoint{ + ScrapeProtocols: []string{ + "PrometheusText", + "PrometheusProto", + }, Path: "/metric", Port: "web", }, @@ -170,6 +174,9 @@ kubernetes_sd_configs: selectors: - role: pod label: label-1=value-1,label-2=value-2,label-3=value-3 +scrape_protocols: +- PrometheusText +- PrometheusProto metrics_path: /metric relabel_configs: - action: drop diff --git a/controllers/factory/vmagent/servicescrape.go b/controllers/factory/vmagent/servicescrape.go index 20594b2a9..ae897f394 100644 --- a/controllers/factory/vmagent/servicescrape.go +++ b/controllers/factory/vmagent/servicescrape.go @@ -59,6 +59,9 @@ func generateServiceScrapeConfig( if scrapeInterval != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_interval", Value: scrapeInterval}) } + if len(ep.ScrapeProtocols) != 0 { + cfg = append(cfg, yaml.MapItem{Key: "scrape_protocols", Value: ep.ScrapeProtocols}) + } if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) } diff --git a/controllers/factory/vmagent/servicescrape_test.go b/controllers/factory/vmagent/servicescrape_test.go index e50101413..3fda677f2 100644 --- a/controllers/factory/vmagent/servicescrape_test.go +++ b/controllers/factory/vmagent/servicescrape_test.go @@ -185,6 +185,10 @@ relabel_configs: }, BearerTokenFile: "/var/run/tolen", ScrapeInterval: "60m", + ScrapeProtocols: []string{ + "PrometheusText", + "OpenMetricsText", + }, }, i: 0, apiserverConfig: nil, @@ -202,6 +206,9 @@ kubernetes_sd_configs: names: - default scrape_interval: 40m +scrape_protocols: +- PrometheusText +- OpenMetricsText tls_config: insecure_skip_verify: false ca_file: /etc/vmagent-tls/certs/default_tls-secret_ca diff --git a/controllers/factory/vmagent/staticscrape.go b/controllers/factory/vmagent/staticscrape.go index 23fc2b896..168ce5434 100644 --- a/controllers/factory/vmagent/staticscrape.go +++ b/controllers/factory/vmagent/staticscrape.go @@ -50,6 +50,10 @@ func generateStaticScrapeConfig( cfg = append(cfg, yaml.MapItem{Key: "scrape_interval", Value: scrapeInterval}) } + if len(ep.ScrapeProtocols) != 0 { + cfg = append(cfg, yaml.MapItem{Key: "scrape_protocols", Value: ep.ScrapeProtocols}) + } + if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b02a5a125..d197a4694 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,8 @@ aliases: ## Next release +- [vmservicescrape](./api.md#vmservicescrape), [vmpodscrape](./api.md#vmpodscrape), [vmnodescrape](./api.md#vmnodescrape) - added `scrape_protocols` parameter for scrape protocols configuration + ## [v0.45.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.45.0) - 10 Jun 2024