diff --git a/api/v1beta1/vmnodescrape_types.go b/api/v1beta1/vmnodescrape_types.go index ba64cd7a..252af85e 100644 --- a/api/v1beta1/vmnodescrape_types.go +++ b/api/v1beta1/vmnodescrape_types.go @@ -41,6 +41,9 @@ type VMNodeScrapeSpec struct { // Timeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // MaxScrapeSize defines a maximum size of scraped data for a job + // +optional + MaxScrapeSize string `json:"max_scrape_size,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/api/v1beta1/vmpodscrape_types.go b/api/v1beta1/vmpodscrape_types.go index dfb20332..2ff7c51e 100644 --- a/api/v1beta1/vmpodscrape_types.go +++ b/api/v1beta1/vmpodscrape_types.go @@ -100,6 +100,9 @@ type PodMetricsEndpoint struct { // Timeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // MaxScrapeSize defines a maximum size of scraped data for a job + // +optional + MaxScrapeSize string `json:"max_scrape_size,omitempty"` // SampleLimit defines per-podEndpoint limit on number of scraped samples that will be accepted. // +optional SampleLimit uint64 `json:"sampleLimit,omitempty"` diff --git a/api/v1beta1/vmscrapeconfig_types.go b/api/v1beta1/vmscrapeconfig_types.go index 4f8dd9ac..f2938c06 100644 --- a/api/v1beta1/vmscrapeconfig_types.go +++ b/api/v1beta1/vmscrapeconfig_types.go @@ -66,6 +66,9 @@ type VMScrapeConfigSpec struct { // ScrapeTimeout is the number of seconds to wait until a scrape request times out. // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // MaxScrapeSize defines a maximum size of scraped data for a job + // +optional + MaxScrapeSize string `json:"max_scrape_size,omitempty"` // HonorTimestamps controls whether to respect the timestamps present in scraped data. // +optional HonorTimestamps *bool `json:"honorTimestamps,omitempty"` diff --git a/api/v1beta1/vmservicescrape_types.go b/api/v1beta1/vmservicescrape_types.go index 8ac42535..635d0ac9 100644 --- a/api/v1beta1/vmservicescrape_types.go +++ b/api/v1beta1/vmservicescrape_types.go @@ -143,6 +143,9 @@ type Endpoint struct { // Timeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // MaxScrapeSize defines a maximum size of scraped data for a job + // +optional + MaxScrapeSize string `json:"max_scrape_size,omitempty"` // SampleLimit defines per-endpoint limit on number of scraped samples that will be accepted. // +optional SampleLimit uint64 `json:"sampleLimit,omitempty"` diff --git a/api/v1beta1/vmstaticscrape_types.go b/api/v1beta1/vmstaticscrape_types.go index 1f46e64a..5f1e6391 100644 --- a/api/v1beta1/vmstaticscrape_types.go +++ b/api/v1beta1/vmstaticscrape_types.go @@ -63,6 +63,9 @@ type TargetEndpoint struct { // Timeout after which the scrape is ended // +optional ScrapeTimeout string `json:"scrapeTimeout,omitempty"` + // MaxScrapeSize defines a maximum size of scraped data for a job + // +optional + MaxScrapeSize string `json:"max_scrape_size,omitempty"` // OAuth2 defines auth configuration // +optional OAuth2 *OAuth2 `json:"oauth2,omitempty"` diff --git a/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml index 06bd34a7..1dba9865 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmnodescrapes.yaml @@ -165,6 +165,9 @@ spec: jobLabel: description: The label to use to retrieve the job name from. type: string + max_scrape_size: + description: MaxScrapeSize defines a maximum size of scraped data for a job + type: string metricRelabelConfigs: description: MetricRelabelConfigs to apply to samples before ingestion. items: diff --git a/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml index bd99808f..a315be1a 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmpodscrapes.yaml @@ -208,6 +208,9 @@ spec: interval: description: Interval at which metrics should be scraped type: string + max_scrape_size: + description: MaxScrapeSize defines a maximum size of scraped data for a job + type: string metricRelabelConfigs: description: MetricRelabelConfigs to apply to samples before ingestion. items: diff --git a/config/crd/bases/operator.victoriametrics.com_vmscrapeconfigs.yaml b/config/crd/bases/operator.victoriametrics.com_vmscrapeconfigs.yaml index acf5ff86..48c6a68f 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmscrapeconfigs.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmscrapeconfigs.yaml @@ -2359,6 +2359,9 @@ spec: - role type: object type: array + max_scrape_size: + description: MaxScrapeSize defines a maximum size of scraped data for a job + type: string metricRelabelConfigs: description: MetricRelabelConfigs to apply to samples before ingestion. items: diff --git a/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml index 0128243e..430d3f2b 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmservicescrapes.yaml @@ -198,6 +198,9 @@ spec: interval: description: Interval at which metrics should be scraped type: string + max_scrape_size: + description: MaxScrapeSize defines a maximum size of scraped data for a job + type: string metricRelabelConfigs: description: MetricRelabelConfigs to apply to samples before ingestion. items: diff --git a/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml b/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml index 1dca800c..b6e9e10b 100644 --- a/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml +++ b/config/crd/bases/operator.victoriametrics.com_vmstaticscrapes.yaml @@ -182,6 +182,9 @@ spec: type: string description: Labels static labels for targets. type: object + max_scrape_size: + description: MaxScrapeSize defines a maximum size of scraped data for a job + type: string metricRelabelConfigs: description: MetricRelabelConfigs to apply to samples before ingestion. items: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d7015596..d7855a2d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -27,6 +27,8 @@ aliases: - [vmalertmanagerconfig](./api.md#vmalertmanagerconfig): adds missing `handleReconcileErr` callback to the reconcile loop. It must properly handle errors and deregister objects. +- [vmservicescrape](./api.md#vmservicescrape), [vmpodscrape](./api.md#vmpodscrape), [vmnodescrape](./api.md#vmnodescrape) - added `max_scrape_size` parameter for scrape protocols configuration + ## [v0.45.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.45.0) - 10 Jun 2024 diff --git a/internal/controller/factory/vmagent/nodescrape.go b/internal/controller/factory/vmagent/nodescrape.go index a9b787b3..130d23bb 100644 --- a/internal/controller/factory/vmagent/nodescrape.go +++ b/internal/controller/factory/vmagent/nodescrape.go @@ -52,6 +52,9 @@ func generateNodeScrapeConfig( if nodeSpec.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: nodeSpec.ScrapeTimeout}) } + if nodeSpec.MaxScrapeSize != "" { + cfg = append(cfg, yaml.MapItem{Key: "max_scrape_size", Value: nodeSpec.MaxScrapeSize}) + } if nodeSpec.Path != "" { cfg = append(cfg, yaml.MapItem{Key: "metrics_path", Value: nodeSpec.Path}) } diff --git a/internal/controller/factory/vmagent/nodescrape_test.go b/internal/controller/factory/vmagent/nodescrape_test.go index a0234819..d31acd8a 100644 --- a/internal/controller/factory/vmagent/nodescrape_test.go +++ b/internal/controller/factory/vmagent/nodescrape_test.go @@ -96,6 +96,7 @@ relabel_configs: FollowRedirects: pointer.Bool(true), ScrapeTimeout: "10s", ScrapeInterval: "5s", + MaxScrapeSize: "10MB", Params: map[string][]string{"module": {"client"}}, JobLabel: "env", HonorTimestamps: pointer.Bool(true), @@ -135,6 +136,7 @@ kubernetes_sd_configs: - role: node scrape_interval: 5s scrape_timeout: 10s +max_scrape_size: 10MB metrics_path: /metrics proxy_url: https://some-url sample_limit: 50 diff --git a/internal/controller/factory/vmagent/podscrape.go b/internal/controller/factory/vmagent/podscrape.go index fe20d5e4..725b2b6b 100644 --- a/internal/controller/factory/vmagent/podscrape.go +++ b/internal/controller/factory/vmagent/podscrape.go @@ -57,6 +57,9 @@ func generatePodScrapeConfig( if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) } + if ep.MaxScrapeSize != "" { + cfg = append(cfg, yaml.MapItem{Key: "max_scrape_size", Value: ep.MaxScrapeSize}) + } if ep.Path != "" { cfg = append(cfg, yaml.MapItem{Key: "metrics_path", Value: ep.Path}) } diff --git a/internal/controller/factory/vmagent/podscrape_test.go b/internal/controller/factory/vmagent/podscrape_test.go index 50b8704b..e7562035 100644 --- a/internal/controller/factory/vmagent/podscrape_test.go +++ b/internal/controller/factory/vmagent/podscrape_test.go @@ -39,8 +39,9 @@ func Test_generatePodScrapeConfig(t *testing.T) { }, }, ep: vmv1beta1.PodMetricsEndpoint{ - Path: "/metric", - Port: "web", + MaxScrapeSize: "30B", + Path: "/metric", + Port: "web", AttachMetadata: vmv1beta1.AttachMetadata{ Node: pointer.Bool(true), }, @@ -56,6 +57,7 @@ kubernetes_sd_configs: namespaces: names: - default +max_scrape_size: 30B metrics_path: /metric relabel_configs: - action: drop diff --git a/internal/controller/factory/vmagent/servicescrape.go b/internal/controller/factory/vmagent/servicescrape.go index 463bef43..0805d32e 100644 --- a/internal/controller/factory/vmagent/servicescrape.go +++ b/internal/controller/factory/vmagent/servicescrape.go @@ -62,6 +62,9 @@ func generateServiceScrapeConfig( if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) } + if ep.MaxScrapeSize != "" { + cfg = append(cfg, yaml.MapItem{Key: "max_scrape_size", Value: ep.MaxScrapeSize}) + } if ep.Path != "" { cfg = append(cfg, yaml.MapItem{Key: "metrics_path", Value: ep.Path}) } diff --git a/internal/controller/factory/vmagent/servicescrape_test.go b/internal/controller/factory/vmagent/servicescrape_test.go index 184d8d47..25373143 100644 --- a/internal/controller/factory/vmagent/servicescrape_test.go +++ b/internal/controller/factory/vmagent/servicescrape_test.go @@ -185,6 +185,7 @@ relabel_configs: }, BearerTokenFile: "/var/run/tolen", ScrapeInterval: "60m", + MaxScrapeSize: "1MB", }, i: 0, apiserverConfig: nil, @@ -202,6 +203,7 @@ kubernetes_sd_configs: names: - default scrape_interval: 40m +max_scrape_size: 1MB tls_config: insecure_skip_verify: false ca_file: /etc/vmagent-tls/certs/default_tls-secret_ca diff --git a/internal/controller/factory/vmagent/staticscrape.go b/internal/controller/factory/vmagent/staticscrape.go index 8b3bc2e4..60530be8 100644 --- a/internal/controller/factory/vmagent/staticscrape.go +++ b/internal/controller/factory/vmagent/staticscrape.go @@ -53,6 +53,9 @@ func generateStaticScrapeConfig( if ep.ScrapeTimeout != "" { cfg = append(cfg, yaml.MapItem{Key: "scrape_timeout", Value: ep.ScrapeTimeout}) } + if ep.MaxScrapeSize != "" { + cfg = append(cfg, yaml.MapItem{Key: "max_scrape_size", Value: ep.MaxScrapeSize}) + } if ep.Path != "" { cfg = append(cfg, yaml.MapItem{Key: "metrics_path", Value: ep.Path}) } diff --git a/internal/controller/factory/vmagent/staticscrape_test.go b/internal/controller/factory/vmagent/staticscrape_test.go index a38bef57..72621393 100644 --- a/internal/controller/factory/vmagent/staticscrape_test.go +++ b/internal/controller/factory/vmagent/staticscrape_test.go @@ -156,6 +156,7 @@ metric_relabel_configs: [] InsecureSkipVerify: true, }, ScrapeTimeout: "55s", + MaxScrapeSize: "2KB", Interval: "10s", ScrapeInterval: "50s", FollowRedirects: pointer.Bool(true), @@ -245,6 +246,7 @@ static_configs: group: prod scrape_interval: 50s scrape_timeout: 55s +max_scrape_size: 2KB metrics_path: /metrics-1 proxy_url: https://some-proxy follow_redirects: true