From 9144becbb4fee82735b8d0e23dcd20c8fafa57c9 Mon Sep 17 00:00:00 2001 From: f41gh7 Date: Thu, 15 Aug 2024 00:46:55 +0200 Subject: [PATCH] api: adds status and lastSyncError to scrape objects * adds `status` and `lastSyncError` to `VMServiceScrape`, `VMPodScrape`, `VMNodeScrape`, `VMStaticScrape`, `VMProbeScrape` and `VMScrapeConfig`. It provides better obersvability for API users. * adds generic functions for scrape objects secrets fetching. It reduces boilerplat and simplifies code. * adds status track for scrape objects * unifies tlsAssetLoad and secretValue fetch into single function for vmagent config builder. It should improve readibilty. Signed-off-by: f41gh7 --- api/operator/v1beta1/vmextra_types.go | 10 + api/operator/v1beta1/vmnodescrape_types.go | 13 +- api/operator/v1beta1/vmpodscrape_types.go | 16 +- api/operator/v1beta1/vmprobe_types.go | 15 +- api/operator/v1beta1/vmscrapeconfig_types.go | 16 +- api/operator/v1beta1/vmservicescrape_types.go | 15 +- api/operator/v1beta1/vmstaticscrape_types.go | 16 +- api/operator/v1beta1/zz_generated.deepcopy.go | 105 +-- config/crd/overlay/crd.yaml | 132 ++- docs/CHANGELOG.md | 3 +- .../factory/k8stools/secret_content.go | 29 +- .../factory/vmagent/collect_scrapes.go | 185 ++-- .../factory/vmagent/collect_scrapes_test.go | 16 +- .../operator/factory/vmagent/vmagent.go | 159 +--- .../factory/vmagent/vmagent_scrapeconfig.go | 832 ++++++++---------- .../vmagent/vmagent_scrapeconfig_test.go | 40 - .../operator/factory/vmagent/vmagent_test.go | 167 +++- 17 files changed, 861 insertions(+), 908 deletions(-) diff --git a/api/operator/v1beta1/vmextra_types.go b/api/operator/v1beta1/vmextra_types.go index 87878c3c..788f50f7 100644 --- a/api/operator/v1beta1/vmextra_types.go +++ b/api/operator/v1beta1/vmextra_types.go @@ -984,3 +984,13 @@ type TLSClientConfig struct { // Certs defines cert, CA and key for TLS auth Certs `json:",inline"` } + +// ScrapeObjectStatus defines the observed state of ScrapeObjects +type ScrapeObjectStatus struct { + // Status defines update status of resource + Status UpdateStatus `json:"status,omitempty"` + // LastSyncError contains error message for unsuccessful config generation + LastSyncError string `json:"lastSyncError,omitempty"` + // CurrentSyncError holds an error occured during reconcile loop + CurrentSyncError string `json:"-"` +} diff --git a/api/operator/v1beta1/vmnodescrape_types.go b/api/operator/v1beta1/vmnodescrape_types.go index d36510a7..2b245871 100644 --- a/api/operator/v1beta1/vmnodescrape_types.go +++ b/api/operator/v1beta1/vmnodescrape_types.go @@ -29,22 +29,22 @@ type VMNodeScrapeSpec struct { Selector metav1.LabelSelector `json:"selector,omitempty"` } -// VMNodeScrapeStatus defines the observed state of VMNodeScrape -type VMNodeScrapeStatus struct{} - // VMNodeScrape defines discovery for targets placed on kubernetes nodes, // usually its node-exporters and other host services. // InternalIP is used as __address__ for scraping. // +kubebuilder:object:root=true // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status" +// +kubebuilder:printcolumn:name="Sync Error",type="string",JSONPath=".status.lastSyncError" // +genclient type VMNodeScrape struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` Spec VMNodeScrapeSpec `json:"spec,omitempty"` - Status VMNodeScrapeStatus `json:"status,omitempty"` + Status ScrapeObjectStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true @@ -66,6 +66,11 @@ func (cr VMNodeScrape) AsMapKey() string { return fmt.Sprintf("nodeScrape/%s/%s", cr.Namespace, cr.Name) } +// GetStatus returns scrape object status +func (cr *VMNodeScrape) GetStatus() *ScrapeObjectStatus { + return &cr.Status +} + func init() { SchemeBuilder.Register(&VMNodeScrape{}, &VMNodeScrapeList{}) } diff --git a/api/operator/v1beta1/vmpodscrape_types.go b/api/operator/v1beta1/vmpodscrape_types.go index 556dcbb7..c75c3e76 100644 --- a/api/operator/v1beta1/vmpodscrape_types.go +++ b/api/operator/v1beta1/vmpodscrape_types.go @@ -38,9 +38,6 @@ type VMPodScrapeSpec struct { AttachMetadata AttachMetadata `json:"attach_metadata,omitempty"` } -// VMPodScrapeStatus defines the observed state of VMPodScrape -type VMPodScrapeStatus struct{} - // VMPodScrape is scrape configuration for pods, // it generates vmagent's config for scraping pod targets // based on selectors. @@ -49,15 +46,17 @@ type VMPodScrapeStatus struct{} // +kubebuilder:subresource:status // +kubebuilder:resource:path=vmpodscrapes,scope=Namespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status" +// +kubebuilder:printcolumn:name="Sync Error",type="string",JSONPath=".status.lastSyncError" // +genclient type VMPodScrape struct { metav1.TypeMeta `json:",inline"` // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - Spec VMPodScrapeSpec `json:"spec,omitempty"` - // +optional - Status VMPodScrapeStatus `json:"status"` + Spec VMPodScrapeSpec `json:"spec,omitempty"` + Status ScrapeObjectStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -114,6 +113,11 @@ func (cr *VMPodScrape) AsMapKey(i int) string { return fmt.Sprintf("podScrape/%s/%s/%d", cr.Namespace, cr.Name, i) } +// GetStatus returns scrape object status +func (cr *VMPodScrape) GetStatus() *ScrapeObjectStatus { + return &cr.Status +} + func init() { SchemeBuilder.Register(&VMPodScrape{}, &VMPodScrapeList{}) } diff --git a/api/operator/v1beta1/vmprobe_types.go b/api/operator/v1beta1/vmprobe_types.go index 8c442a72..c79384af 100644 --- a/api/operator/v1beta1/vmprobe_types.go +++ b/api/operator/v1beta1/vmprobe_types.go @@ -90,14 +90,14 @@ type VMProberSpec struct { Path string `json:"path,omitempty"` } -// VMProbeStatus defines the observed state of VMProbe -type VMProbeStatus struct{} - // VMProbe defines a probe for targets, that will be executed with prober, // like blackbox exporter. // It helps to monitor reachability of target with various checks. // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status" +// +kubebuilder:printcolumn:name="Sync Error",type="string",JSONPath=".status.lastSyncError" // +genclient // +k8s:openapi-gen=true type VMProbe struct { @@ -106,8 +106,8 @@ type VMProbe struct { // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - Spec VMProbeSpec `json:"spec"` - Status VMProbeStatus `json:"status,omitempty"` + Spec VMProbeSpec `json:"spec"` + Status ScrapeObjectStatus `json:"status,omitempty"` } // VMProbeList contains a list of VMProbe @@ -127,6 +127,11 @@ func (cr VMProbe) AsMapKey() string { return fmt.Sprintf("probeScrape/%s/%s", cr.Namespace, cr.Name) } +// GetStatus returns scrape object status +func (cr *VMProbe) GetStatus() *ScrapeObjectStatus { + return &cr.Status +} + func init() { SchemeBuilder.Register(&VMProbe{}, &VMProbeList{}) } diff --git a/api/operator/v1beta1/vmscrapeconfig_types.go b/api/operator/v1beta1/vmscrapeconfig_types.go index 2e43f6a5..e5559065 100644 --- a/api/operator/v1beta1/vmscrapeconfig_types.go +++ b/api/operator/v1beta1/vmscrapeconfig_types.go @@ -28,14 +28,16 @@ import ( // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:path=vmscrapeconfigs,scope=Namespaced +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status" +// +kubebuilder:printcolumn:name="Sync Error",type="string",JSONPath=".status.lastSyncError" // +genclient type VMScrapeConfig struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec VMScrapeConfigSpec `json:"spec,omitempty"` - // +optional - Status VMScrapeConfigStatus `json:"status"` + Spec VMScrapeConfigSpec `json:"spec,omitempty"` + Status ScrapeObjectStatus `json:"status,omitempty"` } // VMScrapeConfigSpec defines the desired state of VMScrapeConfig @@ -488,9 +490,6 @@ type DigitalOceanSDConfig struct { Port *int `json:"port,omitempty"` } -// VMScrapeConfigStatus defines the observed state of VMScrapeConfig -type VMScrapeConfigStatus struct{} - // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // VMScrapeConfigList contains a list of VMScrapeConfig @@ -510,6 +509,11 @@ func (cr VMScrapeConfig) AsMapKey(prefix string, i int) string { return fmt.Sprintf("scrapeConfig/%s/%s/%s/%d", cr.Namespace, cr.Name, prefix, i) } +// GetStatus returns scrape object status +func (cr *VMScrapeConfig) GetStatus() *ScrapeObjectStatus { + return &cr.Status +} + func init() { SchemeBuilder.Register(&VMScrapeConfig{}, &VMScrapeConfigList{}) } diff --git a/api/operator/v1beta1/vmservicescrape_types.go b/api/operator/v1beta1/vmservicescrape_types.go index 3c8b5d45..4c090328 100644 --- a/api/operator/v1beta1/vmservicescrape_types.go +++ b/api/operator/v1beta1/vmservicescrape_types.go @@ -49,9 +49,6 @@ type VMServiceScrapeSpec struct { AttachMetadata AttachMetadata `json:"attach_metadata,omitempty"` } -// VMServiceScrapeStatus defines the observed state of VMServiceScrape -type VMServiceScrapeStatus struct{} - // VMServiceScrape is scrape configuration for endpoints associated with // kubernetes service, // it generates scrape configuration for vmagent based on selectors. @@ -61,13 +58,16 @@ type VMServiceScrapeStatus struct{} // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:path=vmservicescrapes,scope=Namespaced +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status" +// +kubebuilder:printcolumn:name="Sync Error",type="string",JSONPath=".status.lastSyncError" // +genclient type VMServiceScrape struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec VMServiceScrapeSpec `json:"spec"` - Status VMServiceScrapeStatus `json:"status,omitempty"` + Spec VMServiceScrapeSpec `json:"spec"` + Status ScrapeObjectStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -140,6 +140,11 @@ func (cr VMServiceScrape) AsMapKey(i int) string { return fmt.Sprintf("serviceScrape/%s/%s/%d", cr.Namespace, cr.Name, i) } +// GetStatus returns scrape object status +func (cr *VMServiceScrape) GetStatus() *ScrapeObjectStatus { + return &cr.Status +} + func init() { SchemeBuilder.Register(&VMServiceScrape{}, &VMServiceScrapeList{}) } diff --git a/api/operator/v1beta1/vmstaticscrape_types.go b/api/operator/v1beta1/vmstaticscrape_types.go index a2c1a1cc..0a63ca31 100644 --- a/api/operator/v1beta1/vmstaticscrape_types.go +++ b/api/operator/v1beta1/vmstaticscrape_types.go @@ -34,23 +34,22 @@ type TargetEndpoint struct { EndpointScrapeParams `json:",inline"` } -// VMStaticScrapeStatus defines the observed state of VMStaticScrape -type VMStaticScrapeStatus struct{} - // VMStaticScrape defines static targets configuration for scraping. // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status" +// +kubebuilder:printcolumn:name="Sync Error",type="string",JSONPath=".status.lastSyncError" // +genclient type VMStaticScrape struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec VMStaticScrapeSpec `json:"spec,omitempty"` - Status VMStaticScrapeStatus `json:"status,omitempty"` + Spec VMStaticScrapeSpec `json:"spec,omitempty"` + Status ScrapeObjectStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true - // VMStaticScrapeList contains a list of VMStaticScrape type VMStaticScrapeList struct { metav1.TypeMeta `json:",inline"` @@ -68,6 +67,11 @@ func (cr VMStaticScrape) AsMapKey(i int) string { return fmt.Sprintf("staticScrape/%s/%s/%d", cr.Namespace, cr.Name, i) } +// GetStatus returns scrape object status +func (cr *VMStaticScrape) GetStatus() *ScrapeObjectStatus { + return &cr.Status +} + func init() { SchemeBuilder.Register(&VMStaticScrape{}, &VMStaticScrapeList{}) } diff --git a/api/operator/v1beta1/zz_generated.deepcopy.go b/api/operator/v1beta1/zz_generated.deepcopy.go index 3b5bb7c4..defa8671 100644 --- a/api/operator/v1beta1/zz_generated.deepcopy.go +++ b/api/operator/v1beta1/zz_generated.deepcopy.go @@ -2157,6 +2157,21 @@ func (in *RuleGroup) DeepCopy() *RuleGroup { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScrapeObjectStatus) DeepCopyInto(out *ScrapeObjectStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScrapeObjectStatus. +func (in *ScrapeObjectStatus) DeepCopy() *ScrapeObjectStatus { + if in == nil { + return nil + } + out := new(ScrapeObjectStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretOrConfigMap) DeepCopyInto(out *SecretOrConfigMap) { *out = *in @@ -5463,21 +5478,6 @@ func (in *VMNodeScrapeSpec) DeepCopy() *VMNodeScrapeSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VMNodeScrapeStatus) DeepCopyInto(out *VMNodeScrapeStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMNodeScrapeStatus. -func (in *VMNodeScrapeStatus) DeepCopy() *VMNodeScrapeStatus { - if in == nil { - return nil - } - out := new(VMNodeScrapeStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VMPodScrape) DeepCopyInto(out *VMPodScrape) { *out = *in @@ -5567,21 +5567,6 @@ func (in *VMPodScrapeSpec) DeepCopy() *VMPodScrapeSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VMPodScrapeStatus) DeepCopyInto(out *VMPodScrapeStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMPodScrapeStatus. -func (in *VMPodScrapeStatus) DeepCopy() *VMPodScrapeStatus { - if in == nil { - return nil - } - out := new(VMPodScrapeStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VMProbe) DeepCopyInto(out *VMProbe) { *out = *in @@ -5671,21 +5656,6 @@ func (in *VMProbeSpec) DeepCopy() *VMProbeSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VMProbeStatus) DeepCopyInto(out *VMProbeStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMProbeStatus. -func (in *VMProbeStatus) DeepCopy() *VMProbeStatus { - if in == nil { - return nil - } - out := new(VMProbeStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VMProbeTargetStaticConfig) DeepCopyInto(out *VMProbeTargetStaticConfig) { *out = *in @@ -6053,21 +6023,6 @@ func (in *VMScrapeConfigSpec) DeepCopy() *VMScrapeConfigSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VMScrapeConfigStatus) DeepCopyInto(out *VMScrapeConfigStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMScrapeConfigStatus. -func (in *VMScrapeConfigStatus) DeepCopy() *VMScrapeConfigStatus { - if in == nil { - return nil - } - out := new(VMScrapeConfigStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VMScrapeParams) DeepCopyInto(out *VMScrapeParams) { *out = *in @@ -6394,21 +6349,6 @@ func (in *VMServiceScrapeSpec) DeepCopy() *VMServiceScrapeSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VMServiceScrapeStatus) DeepCopyInto(out *VMServiceScrapeStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMServiceScrapeStatus. -func (in *VMServiceScrapeStatus) DeepCopy() *VMServiceScrapeStatus { - if in == nil { - return nil - } - out := new(VMServiceScrapeStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VMSingle) DeepCopyInto(out *VMSingle) { *out = *in @@ -6761,21 +6701,6 @@ func (in *VMStaticScrapeSpec) DeepCopy() *VMStaticScrapeSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VMStaticScrapeStatus) DeepCopyInto(out *VMStaticScrapeStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMStaticScrapeStatus. -func (in *VMStaticScrapeStatus) DeepCopy() *VMStaticScrapeStatus { - if in == nil { - return nil - } - out := new(VMStaticScrapeStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VMStorage) DeepCopyInto(out *VMStorage) { *out = *in diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml index 52b17e82..616f8d86 100644 --- a/config/crd/overlay/crd.yaml +++ b/config/crd/overlay/crd.yaml @@ -18728,7 +18728,17 @@ spec: singular: vmnodescrape scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.lastSyncError + name: Sync Error + type: string + name: v1beta1 schema: openAPIV3Schema: description: |- @@ -19721,7 +19731,15 @@ spec: type: object type: object status: - description: VMNodeScrapeStatus defines the observed state of VMNodeScrape + description: ScrapeObjectStatus defines the observed state of ScrapeObjects + properties: + lastSyncError: + description: LastSyncError contains error message for unsuccessful + config generation + type: string + status: + description: Status defines update status of resource + type: string type: object type: object served: true @@ -19744,7 +19762,17 @@ spec: singular: vmpodscrape scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.lastSyncError + name: Sync Error + type: string + name: v1beta1 schema: openAPIV3Schema: description: |- @@ -20820,7 +20848,15 @@ spec: - podMetricsEndpoints type: object status: - description: VMPodScrapeStatus defines the observed state of VMPodScrape + description: ScrapeObjectStatus defines the observed state of ScrapeObjects + properties: + lastSyncError: + description: LastSyncError contains error message for unsuccessful + config generation + type: string + status: + description: Status defines update status of resource + type: string type: object type: object served: true @@ -20843,7 +20879,17 @@ spec: singular: vmprobe scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.lastSyncError + name: Sync Error + type: string + name: v1beta1 schema: openAPIV3Schema: description: |- @@ -21980,7 +22026,15 @@ spec: - vmProberSpec type: object status: - description: VMProbeStatus defines the observed state of VMProbe + description: ScrapeObjectStatus defines the observed state of ScrapeObjects + properties: + lastSyncError: + description: LastSyncError contains error message for unsuccessful + config generation + type: string + status: + description: Status defines update status of resource + type: string type: object required: - spec @@ -22237,7 +22291,17 @@ spec: singular: vmscrapeconfig scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.lastSyncError + name: Sync Error + type: string + name: v1beta1 schema: openAPIV3Schema: description: VMScrapeConfig specifies a set of targets and parameters describing @@ -26357,7 +26421,15 @@ spec: type: object type: object status: - description: VMScrapeConfigStatus defines the observed state of VMScrapeConfig + description: ScrapeObjectStatus defines the observed state of ScrapeObjects + properties: + lastSyncError: + description: LastSyncError contains error message for unsuccessful + config generation + type: string + status: + description: Status defines update status of resource + type: string type: object type: object served: true @@ -26380,7 +26452,17 @@ spec: singular: vmservicescrape scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.lastSyncError + name: Sync Error + type: string + name: v1beta1 schema: openAPIV3Schema: description: |- @@ -27469,7 +27551,15 @@ spec: - endpoints type: object status: - description: VMServiceScrapeStatus defines the observed state of VMServiceScrape + description: ScrapeObjectStatus defines the observed state of ScrapeObjects + properties: + lastSyncError: + description: LastSyncError contains error message for unsuccessful + config generation + type: string + status: + description: Status defines update status of resource + type: string type: object required: - spec @@ -29180,7 +29270,17 @@ spec: singular: vmstaticscrape scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.lastSyncError + name: Sync Error + type: string + name: v1beta1 schema: openAPIV3Schema: description: VMStaticScrape defines static targets configuration for scraping. @@ -30161,7 +30261,15 @@ spec: - targetEndpoints type: object status: - description: VMStaticScrapeStatus defines the observed state of VMStaticScrape + description: ScrapeObjectStatus defines the observed state of ScrapeObjects + properties: + lastSyncError: + description: LastSyncError contains error message for unsuccessful + config generation + type: string + status: + description: Status defines update status of resource + type: string type: object type: object served: true diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4c193a43..05e75144 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -20,8 +20,9 @@ aliases: - [config-reloader](./README.md): adds new flags `tlsCaFile`, `tlsCertFile`,`tlsKeyFile`,`tlsServerName`,`tlsInsecureSkipVerify`. It allows to configure `tls` for reload endpoint. Related [issue](https://github.com/VictoriaMetrics/operator/issues/1033). - [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/): adds `status.lastSyncError` field, adds server-side validation for `spec.targetRefs.crd.kind`. Adds small refactoring. - [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/): allows to skip `VMUser` from `VMAuth` config generation if it has misconfigured fields. Such as references to non-exist `CRD` objects or missing fields. It's highly recommended to enable `Validation` webhook for `VMUsers`, it should reduce surface of potential misconfiguration. See this [issue](https://github.com/VictoriaMetrics/operator/issues/1047) for details. +- [vmagent](./resources/vmagent.md): adds `status` and `lastSyncError` status fields to all scrape objects - `VMServiceScrape`, `VMPodScrape`, `VMNodeScrape`,`VMPodScrape`, `VMStaticScrape` and `VMScrapeConfig`. It allows to track config generation for `vmagent` from scrape objects. +- [operator](./README.md): refactors config builder for `VMAgent`. It fixes minor bug with incorrect skip of scrape object with incorrect references for secrets and configmaps. - [operator](./README.md): allows to secure `metrics-bind-address` webserver with `TLS` and `mTLS` protection via flags `tls.enable`,`tls.certDir`,`tls.certName`,`tls.key``,`mtls.enable`,`mtls.clietCA`. See this [issue](https://github.com/VictoriaMetrics/operator/issues/1033) for details. - - [operator](./README.md): properly release `PodDisruptionBudget` object finalizer. Previously it could be kept due to typo. See this [issue](https://github.com/VictoriaMetrics/operator/issues/1036) for details. - [operator](./README.md): refactors finalizers usage. Simplifies finalizer manipulation with helper functions - [operator](./README.md): adds `tls_config` and `authKey` settings to auto-created `VMServiceScrape` for CRD objects from `extraArgs`. See [this](https://github.com/VictoriaMetrics/operator/issues/1033) issue for details. diff --git a/internal/controller/operator/factory/k8stools/secret_content.go b/internal/controller/operator/factory/k8stools/secret_content.go index 18760b1f..210f0a08 100644 --- a/internal/controller/operator/factory/k8stools/secret_content.go +++ b/internal/controller/operator/factory/k8stools/secret_content.go @@ -10,6 +10,28 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// KeyNotFoundError represents an error if expected key +// was not found at secret or configmap data +type KeyNotFoundError struct { + key string + cacheKey string + context string +} + +// NewKeyNotFoundError returns NewKeyNotFoundError +func NewKeyNotFoundError(key, cacheKey, object string) *KeyNotFoundError { + return &KeyNotFoundError{ + key: key, + cacheKey: cacheKey, + context: object, + } +} + +// Error implements interface +func (ke *KeyNotFoundError) Error() string { + return fmt.Sprintf("expected key=%q was not found at=%q cache_key=%q", ke.key, ke.context, ke.cacheKey) +} + // OAuthCreds represents OAuth2 secret values within plain text type OAuthCreds struct { ClientSecret string @@ -77,7 +99,7 @@ func extractCredKey(secret *corev1.Secret, sel corev1.SecretKeySelector) (string if s, ok := secret.Data[sel.Key]; ok { return string(s), nil } - return "", fmt.Errorf("secret key %q in secret %q not found", sel.Key, sel.Name) + return "", &KeyNotFoundError{sel.Key, sel.Name, "secret"} } // GetCredFromSecret fetch content of secret by given key @@ -103,7 +125,7 @@ func GetCredFromSecret( } v, err := extractCredKey(s, *sel) if err != nil { - return "", fmt.Errorf("cannot find key: %q at secret: %q for object: %q", sel.Key, s.Name, cacheKey) + return "", err } return v, nil } @@ -132,7 +154,8 @@ func GetCredFromConfigMap( if a, ok := s.Data[sel.Key]; ok { return a, nil } - return "", fmt.Errorf("key not found at configmap, key: %s, configmap %s ", sel.Key, sel.Name) + return "", &KeyNotFoundError{sel.Key, cacheKey, "configmap"} + // fmt.Errorf("key not found at configmap, key: %s, configmap %s ", sel.Key, sel.Name) } func buildCacheKey(ns, keyName string) string { diff --git a/internal/controller/operator/factory/vmagent/collect_scrapes.go b/internal/controller/operator/factory/vmagent/collect_scrapes.go index 208b6fd4..a9ef09ee 100644 --- a/internal/controller/operator/factory/vmagent/collect_scrapes.go +++ b/internal/controller/operator/factory/vmagent/collect_scrapes.go @@ -2,6 +2,8 @@ package vmagent import ( "context" + "fmt" + "sort" "strings" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" @@ -11,9 +13,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func selectScrapeConfig(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) (map[string]*vmv1beta1.VMScrapeConfig, error) { - res := make(map[string]*vmv1beta1.VMScrapeConfig) - var scrapeConfigsCombined []vmv1beta1.VMScrapeConfig +func selectScrapeConfig(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMScrapeConfig, error) { + var scrapeConfigsCombined []*vmv1beta1.VMScrapeConfig + var namespacedNames []string if err := k8stools.VisitObjectsForSelectorsAtNs(ctx, rclient, cr.Spec.NodeScrapeNamespaceSelector, cr.Spec.ScrapeConfigSelector, cr.Namespace, cr.Spec.SelectAllByDefault, func(list *vmv1beta1.VMScrapeConfigList) { @@ -21,30 +23,24 @@ func selectScrapeConfig(ctx context.Context, cr *vmv1beta1.VMAgent, rclient clie if !item.DeletionTimestamp.IsZero() { continue } - scrapeConfigsCombined = append(scrapeConfigsCombined, item) + item := item + scrapeConfigsCombined = append(scrapeConfigsCombined, &item) + namespacedNames = append(namespacedNames, fmt.Sprintf("%s/%s", item.Namespace, item.Name)) } }); err != nil { return nil, err } - - for _, scrapeConf := range scrapeConfigsCombined { - sc := scrapeConf.DeepCopy() - res[scrapeConf.Namespace+"/"+scrapeConf.Name] = sc - } - scrapeConfigs := make([]string, 0) - for key := range res { - scrapeConfigs = append(scrapeConfigs, key) + sort.Sort(&namespacedNameSorter[*vmv1beta1.VMScrapeConfig]{target: scrapeConfigsCombined, sorter: namespacedNames}) + if len(namespacedNames) > 0 { + logger.WithContext(ctx).Info("selected scrapeConfigs", "scrapeConfigs", strings.Join(namespacedNames, ","), "namespace", cr.Namespace, "vmagent", cr.Name) } - logger.WithContext(ctx).Info("selected scrapeConfigs", "scrapeConfigs", strings.Join(scrapeConfigs, ","), "namespace", cr.Namespace, "vmagent", cr.Name) - - return res, nil + return scrapeConfigsCombined, nil } -func selectPodScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) (map[string]*vmv1beta1.VMPodScrape, error) { - res := make(map[string]*vmv1beta1.VMPodScrape) - - var podScrapesCombined []vmv1beta1.VMPodScrape +func selectPodScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMPodScrape, error) { + var podScrapesCombined []*vmv1beta1.VMPodScrape + var namespacedNames []string if err := k8stools.VisitObjectsForSelectorsAtNs(ctx, rclient, cr.Spec.PodScrapeNamespaceSelector, cr.Spec.PodScrapeSelector, cr.Namespace, cr.Spec.SelectAllByDefault, func(list *vmv1beta1.VMPodScrapeList) { @@ -52,65 +48,56 @@ func selectPodScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client if !item.DeletionTimestamp.IsZero() { continue } - podScrapesCombined = append(podScrapesCombined, item) + item := item + podScrapesCombined = append(podScrapesCombined, &item) + namespacedNames = append(namespacedNames, fmt.Sprintf("%s/%s", item.Namespace, item.Name)) } }); err != nil { return nil, err } - for _, podScrape := range podScrapesCombined { - pm := podScrape.DeepCopy() - res[podScrape.Namespace+"/"+podScrape.Name] = pm + sort.Sort(&namespacedNameSorter[*vmv1beta1.VMPodScrape]{target: podScrapesCombined, sorter: namespacedNames}) + if len(namespacedNames) > 0 { + logger.WithContext(ctx).Info("selected PodScrapes", "podscrapes", strings.Join(namespacedNames, ","), "namespace", cr.Namespace, "vmagent", cr.Name) } - podScrapes := make([]string, 0, len(res)) - for key := range res { - podScrapes = append(podScrapes, key) - } - - logger.WithContext(ctx).Info("selected PodScrapes", "podscrapes", strings.Join(podScrapes, ","), "namespace", cr.Namespace, "vmagent", cr.Name) - return res, nil + return podScrapesCombined, nil } -func selectVMProbes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) (map[string]*vmv1beta1.VMProbe, error) { - res := make(map[string]*vmv1beta1.VMProbe) - var probesCombined []vmv1beta1.VMProbe +func selectVMProbes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMProbe, error) { + var probesCombined []*vmv1beta1.VMProbe + var namespacedNames []string if err := k8stools.VisitObjectsForSelectorsAtNs(ctx, rclient, cr.Spec.ProbeNamespaceSelector, cr.Spec.ProbeSelector, cr.Namespace, cr.Spec.SelectAllByDefault, func(list *vmv1beta1.VMProbeList) { for _, item := range list.Items { if !item.DeletionTimestamp.IsZero() { continue } - probesCombined = append(probesCombined, item) + item := item + probesCombined = append(probesCombined, &item) + namespacedNames = append(namespacedNames, fmt.Sprintf("%s/%s", item.Namespace, item.Name)) } }); err != nil { return nil, err } - for _, probe := range probesCombined { - pm := probe.DeepCopy() - res[probe.Namespace+"/"+probe.Name] = pm - } - probesList := make([]string, 0) - for key := range res { - probesList = append(probesList, key) + sort.Sort(&namespacedNameSorter[*vmv1beta1.VMProbe]{target: probesCombined, sorter: namespacedNames}) + if len(namespacedNames) > 0 { + logger.WithContext(ctx).Info("selected VMProbes", "vmProbes", strings.Join(namespacedNames, ","), "namespace", cr.Namespace, "vmagent", cr.Name) } - logger.WithContext(ctx).Info("selected VMProbes", "vmProbes", strings.Join(probesList, ","), "namespace", cr.Namespace, "vmagent", cr.Name) - - return res, nil + return probesCombined, nil } -func selectVMNodeScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) (map[string]*vmv1beta1.VMNodeScrape, error) { +func selectVMNodeScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMNodeScrape, error) { l := logger.WithContext(ctx).WithValues("vmagent", cr.Name) if !config.IsClusterWideAccessAllowed() && cr.IsOwnsServiceAccount() { l.Info("cannot use VMNodeScrape at operator in single namespace mode with default permissions. Create ServiceAccount for VMAgent manually if needed. Skipping config generation for it") return nil, nil } - res := make(map[string]*vmv1beta1.VMNodeScrape) - - var nodesCombined []vmv1beta1.VMNodeScrape + var nodesCombined []*vmv1beta1.VMNodeScrape + var namespacedNames []string if err := k8stools.VisitObjectsForSelectorsAtNs(ctx, rclient, cr.Spec.NodeScrapeNamespaceSelector, cr.Spec.NodeScrapeSelector, cr.Namespace, cr.Spec.SelectAllByDefault, func(list *vmv1beta1.VMNodeScrapeList) { @@ -118,102 +105,112 @@ func selectVMNodeScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient cli if !item.DeletionTimestamp.IsZero() { continue } - nodesCombined = append(nodesCombined, item) + item := item + nodesCombined = append(nodesCombined, &item) + namespacedNames = append(namespacedNames, fmt.Sprintf("%s/%s", item.Namespace, item.Name)) + } }); err != nil { return nil, err } - for _, node := range nodesCombined { - pm := node.DeepCopy() - res[node.Namespace+"/"+node.Name] = pm - } - nodesList := make([]string, 0) - for key := range res { - nodesList = append(nodesList, key) + sort.Sort(&namespacedNameSorter[*vmv1beta1.VMNodeScrape]{target: nodesCombined, sorter: namespacedNames}) + if len(namespacedNames) > 0 { + l.Info("selected VMNodeScrapes", "VMNodeScrapes", strings.Join(namespacedNames, ",")) } - l.Info("selected VMNodeScrapes", "VMNodeScrapes", strings.Join(nodesList, ",")) - - return res, nil + return nodesCombined, nil } -func selectStaticScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) (map[string]*vmv1beta1.VMStaticScrape, error) { - res := make(map[string]*vmv1beta1.VMStaticScrape) - var staticScrapesCombined []vmv1beta1.VMStaticScrape - +func selectStaticScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMStaticScrape, error) { + var staticScrapesCombined []*vmv1beta1.VMStaticScrape + var namespacedNames []string if err := k8stools.VisitObjectsForSelectorsAtNs(ctx, rclient, cr.Spec.StaticScrapeNamespaceSelector, cr.Spec.StaticScrapeSelector, cr.Namespace, cr.Spec.SelectAllByDefault, func(list *vmv1beta1.VMStaticScrapeList) { for _, item := range list.Items { if !item.DeletionTimestamp.IsZero() { continue } - staticScrapesCombined = append(staticScrapesCombined, item) + item := item + staticScrapesCombined = append(staticScrapesCombined, &item) + namespacedNames = append(namespacedNames, fmt.Sprintf("%s/%s", item.Namespace, item.Name)) } }); err != nil { return nil, err } - - for _, staticScrape := range staticScrapesCombined { - pm := staticScrape.DeepCopy() - res[staticScrape.Namespace+"/"+staticScrape.Name] = pm - } - staticScrapes := make([]string, 0) - for key := range res { - staticScrapes = append(staticScrapes, key) + sort.Sort(&namespacedNameSorter[*vmv1beta1.VMStaticScrape]{target: staticScrapesCombined, sorter: namespacedNames}) + if len(namespacedNames) > 0 { + logger.WithContext(ctx).Info("selected StaticScrapes", "staticScrapes", strings.Join(namespacedNames, ","), "namespace", cr.Namespace, "vmagent", cr.Name) } - logger.WithContext(ctx).Info("selected StaticScrapes", "staticScrapes", strings.Join(staticScrapes, ","), "namespace", cr.Namespace, "vmagent", cr.Name) - - return res, nil + return staticScrapesCombined, nil } -func selectServiceScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) (map[string]*vmv1beta1.VMServiceScrape, error) { - res := make(map[string]*vmv1beta1.VMServiceScrape) - - var servScrapesCombined []vmv1beta1.VMServiceScrape - +func selectServiceScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMServiceScrape, error) { + var servScrapesCombined []*vmv1beta1.VMServiceScrape + var serviceScrapeNamespacedNames []string if err := k8stools.VisitObjectsForSelectorsAtNs(ctx, rclient, cr.Spec.ServiceScrapeNamespaceSelector, cr.Spec.ServiceScrapeSelector, cr.Namespace, cr.Spec.SelectAllByDefault, func(list *vmv1beta1.VMServiceScrapeList) { for _, item := range list.Items { if !item.DeletionTimestamp.IsZero() { continue } - servScrapesCombined = append(servScrapesCombined, item) + item := item + serviceScrapeNamespacedNames = append(serviceScrapeNamespacedNames, fmt.Sprintf("%s/%s", item.Namespace, item.Name)) + servScrapesCombined = append(servScrapesCombined, &item) } }); err != nil { return nil, err } - for _, servScrape := range servScrapesCombined { - m := servScrape.DeepCopy() - res[servScrape.Namespace+"/"+servScrape.Name] = m - } - // filter out all service scrapes that access // the file system. // TODO this restriction applies only ServiceScrape, make it global to any other objects if cr.Spec.ArbitraryFSAccessThroughSMs.Deny { - for namespaceAndName, sm := range res { + var cnt int + OUTER: + for idx, sm := range servScrapesCombined { for _, endpoint := range sm.Spec.Endpoints { if err := testForArbitraryFSAccess(endpoint.EndpointAuth); err != nil { - delete(res, namespaceAndName) logger.WithContext(ctx).Info("skipping vmservicescrape", "error", err.Error(), - "vmservicescrape", namespaceAndName, + "vmservicescrape", serviceScrapeNamespacedNames[idx], "namespace", cr.Namespace, "vmagent", cr.Name, ) + continue OUTER } } + servScrapesCombined[cnt] = sm + serviceScrapeNamespacedNames[cnt] = serviceScrapeNamespacedNames[idx] + cnt++ } + servScrapesCombined = servScrapesCombined[:cnt] + serviceScrapeNamespacedNames = serviceScrapeNamespacedNames[:cnt] } + sort.Sort(&namespacedNameSorter[*vmv1beta1.VMServiceScrape]{sorter: serviceScrapeNamespacedNames, target: servScrapesCombined}) - serviceScrapes := make([]string, 0, len(res)) - for k := range res { - serviceScrapes = append(serviceScrapes, k) + if len(serviceScrapeNamespacedNames) > 0 { + logger.WithContext(ctx).Info("selected ServiceScrapes", "servicescrapes", strings.Join(serviceScrapeNamespacedNames, ","), "namespace", cr.Namespace, "vmagent", cr.Name) } - logger.WithContext(ctx).Info("selected ServiceScrapes", "servicescrapes", strings.Join(serviceScrapes, ","), "namespace", cr.Namespace, "vmagent", cr.Name) - return res, nil + return servScrapesCombined, nil +} + +type namespacedNameSorter[T any] struct { + target []T + sorter []string +} + +func (nn *namespacedNameSorter[T]) Len() int { + return len(nn.sorter) +} + +func (nn *namespacedNameSorter[T]) Less(i, j int) bool { + return nn.sorter[i] < nn.sorter[j] +} + +func (nn *namespacedNameSorter[T]) Swap(i, j int) { + nn.target[i], nn.target[j] = nn.target[j], nn.target[i] + nn.sorter[i], nn.sorter[j] = nn.sorter[j], nn.sorter[i] } diff --git a/internal/controller/operator/factory/vmagent/collect_scrapes_test.go b/internal/controller/operator/factory/vmagent/collect_scrapes_test.go index 318ac2e9..9ded7eed 100644 --- a/internal/controller/operator/factory/vmagent/collect_scrapes_test.go +++ b/internal/controller/operator/factory/vmagent/collect_scrapes_test.go @@ -2,6 +2,7 @@ package vmagent import ( "context" + "fmt" "reflect" "sort" "testing" @@ -281,8 +282,8 @@ func TestSelectServiceMonitors(t *testing.T) { return } gotNames := []string{} - for monitorName := range got { - gotNames = append(gotNames, monitorName) + for _, monitorName := range got { + gotNames = append(gotNames, fmt.Sprintf("%s/%s", monitorName.Namespace, monitorName.Name)) } sort.Strings(gotNames) if !reflect.DeepEqual(gotNames, tt.want) { @@ -376,9 +377,10 @@ func TestSelectPodMonitors(t *testing.T) { t.Errorf("SelectPodScrapes() error = %v, wantErr %v", err, tt.wantErr) return } - gotNames := []string{} - for podName := range got { - gotNames = append(gotNames, podName) + var gotNames []string + + for _, k := range got { + gotNames = append(gotNames, fmt.Sprintf("%s/%s", k.Namespace, k.Name)) } sort.Strings(gotNames) if !reflect.DeepEqual(gotNames, tt.want) { @@ -433,8 +435,8 @@ func TestSelectVMProbes(t *testing.T) { return } var result []string - for k := range got { - result = append(result, k) + for _, k := range got { + result = append(result, fmt.Sprintf("%s/%s", k.Namespace, k.Name)) } sort.Strings(result) if !reflect.DeepEqual(result, tt.want) { diff --git a/internal/controller/operator/factory/vmagent/vmagent.go b/internal/controller/operator/factory/vmagent/vmagent.go index 4431c3c6..84aeaaed 100644 --- a/internal/controller/operator/factory/vmagent/vmagent.go +++ b/internal/controller/operator/factory/vmagent/vmagent.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" @@ -854,154 +853,18 @@ func createOrUpdateTLSAssets(ctx context.Context, cr *vmv1beta1.VMAgent, rclient return rclient.Update(ctx, tlsAssetsSecret) } -func loadTLSAssets( - ctx context.Context, - rclient client.Client, - cr *vmv1beta1.VMAgent, - scrapes map[string]*vmv1beta1.VMServiceScrape, - podScrapes map[string]*vmv1beta1.VMPodScrape, - probes map[string]*vmv1beta1.VMProbe, - nodes map[string]*vmv1beta1.VMNodeScrape, - statics map[string]*vmv1beta1.VMStaticScrape, -) (map[string]string, error) { - assets := map[string]string{} - nsSecretCache := make(map[string]*corev1.Secret) - nsConfigMapCache := make(map[string]*corev1.ConfigMap) - - for _, rw := range cr.Spec.RemoteWrite { - if rw.TLSConfig == nil { - continue - } - if err := addAssetsToCache(ctx, rclient, cr.Namespace, rw.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - return nil, fmt.Errorf("cannot add asset for remote write target: %s,err: %w", cr.Name, err) - } - } - if cr.Spec.APIServerConfig != nil && cr.Spec.APIServerConfig.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, cr.Namespace, cr.Spec.APIServerConfig.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - return nil, fmt.Errorf("cannot add asset for remote write target: %s,err: %w", cr.Name, err) - } - } - - var errG utils.ErrGroup - for key, pod := range podScrapes { - var epCnt int - for _, ep := range pod.Spec.PodMetricsEndpoints { - if ep.VMScrapeParams != nil && ep.VMScrapeParams.ProxyClientConfig != nil && ep.VMScrapeParams.ProxyClientConfig.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, pod.Namespace, ep.VMScrapeParams.ProxyClientConfig.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - errG.Add(fmt.Errorf("cannot add proxy tlsAsset for VMPodScrape: %w", err)) - continue - } - } - if ep.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, pod.Namespace, ep.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - errG.Add(fmt.Errorf("cannot add tlsAsset for VMPodScrape: %w", err)) - continue - } - } - pod.Spec.PodMetricsEndpoints[epCnt] = ep - epCnt++ - } - pod.Spec.PodMetricsEndpoints = pod.Spec.PodMetricsEndpoints[:epCnt] - if len(pod.Spec.PodMetricsEndpoints) == 0 { - delete(podScrapes, key) - } - } - for key, mon := range scrapes { - var epCnt int - for _, ep := range mon.Spec.Endpoints { - if ep.VMScrapeParams != nil && ep.VMScrapeParams.ProxyClientConfig != nil && ep.VMScrapeParams.ProxyClientConfig.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, mon.Namespace, ep.VMScrapeParams.ProxyClientConfig.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - errG.Add(err) - continue - } - } - if ep.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, mon.Namespace, ep.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - errG.Add(err) - continue - } - } - mon.Spec.Endpoints[epCnt] = ep - epCnt++ - } - mon.Spec.Endpoints = mon.Spec.Endpoints[:epCnt] - if len(mon.Spec.Endpoints) == 0 { - delete(scrapes, key) - } - } - for key, probe := range probes { - onErr := func(err error) { - errG.Add(err) - delete(probes, key) - } - if probe.Spec.VMScrapeParams != nil && probe.Spec.VMScrapeParams.ProxyClientConfig != nil && probe.Spec.VMScrapeParams.ProxyClientConfig.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, probe.Namespace, probe.Spec.VMScrapeParams.ProxyClientConfig.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - onErr(err) - continue - } - } - if probe.Spec.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, probe.Namespace, probe.Spec.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - onErr(err) - continue - } - } - } - for key, staticCfg := range statics { - var epCnt int - for _, ep := range staticCfg.Spec.TargetEndpoints { - if ep.VMScrapeParams != nil && ep.VMScrapeParams.ProxyClientConfig != nil && ep.VMScrapeParams.ProxyClientConfig.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, staticCfg.Namespace, ep.VMScrapeParams.ProxyClientConfig.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - errG.Add(err) - continue - } - } - if ep.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, staticCfg.Namespace, ep.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - errG.Add(err) - continue - } - } - staticCfg.Spec.TargetEndpoints[epCnt] = ep - epCnt++ - } - staticCfg.Spec.TargetEndpoints = staticCfg.Spec.TargetEndpoints[:epCnt] - if len(staticCfg.Spec.TargetEndpoints) == 0 { - delete(statics, key) - } - } - - for key, node := range nodes { - onErr := func(err error) { - errG.Add(err) - delete(nodes, key) - } - if node.Spec.VMScrapeParams != nil && node.Spec.VMScrapeParams.ProxyClientConfig != nil && node.Spec.VMScrapeParams.ProxyClientConfig.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, node.Namespace, node.Spec.VMScrapeParams.ProxyClientConfig.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - onErr(err) - continue - } - } - if node.Spec.TLSConfig != nil { - if err := addAssetsToCache(ctx, rclient, node.Namespace, node.Spec.TLSConfig, assets, nsSecretCache, nsConfigMapCache); err != nil { - onErr(err) - continue - } - } - - } - return assets, errG.Err() -} - func addAssetsToCache( ctx context.Context, rclient client.Client, objectNS string, tlsConfig *vmv1beta1.TLSConfig, - assets map[string]string, - nsSecretCache map[string]*corev1.Secret, - nsConfigMapCache map[string]*corev1.ConfigMap, + ssCache *scrapesSecretsCache, ) error { + if tlsConfig == nil { + return nil + } + assets, nsSecretCache, nsConfigMapCache := ssCache.tlsAssets, ssCache.nsSecretCache, ssCache.nsCMCache + prefix := objectNS + "/" secretSelectors := map[string]*corev1.SecretKeySelector{} configMapSelectors := map[string]*corev1.ConfigMapKeySelector{} @@ -1037,9 +900,8 @@ func addAssetsToCache( nsSecretCache, ) if err != nil { - return fmt.Errorf( - "failed to extract endpoint tls asset from secret %s and key %s in namespace %s", - selector.Name, selector.Key, objectNS, + return k8stools.NewKeyNotFoundError( + selector.Key, fmt.Sprintf("%s/%s/%s", objectNS, selector.Name, key), "tls_secret", ) } @@ -1056,9 +918,8 @@ func addAssetsToCache( nsConfigMapCache, ) if err != nil { - return fmt.Errorf( - "failed to extract endpoint tls asset from configmap %s and key %s in namespace %s", - selector.Name, selector.Key, objectNS, + return k8stools.NewKeyNotFoundError( + selector.Key, fmt.Sprintf("%s/%s/%s", objectNS, selector.Name, key), "tls_configmap", ) } diff --git a/internal/controller/operator/factory/vmagent/vmagent_scrapeconfig.go b/internal/controller/operator/factory/vmagent/vmagent_scrapeconfig.go index 5469b992..18061143 100644 --- a/internal/controller/operator/factory/vmagent/vmagent_scrapeconfig.go +++ b/internal/controller/operator/factory/vmagent/vmagent_scrapeconfig.go @@ -12,7 +12,6 @@ import ( "sort" "strings" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/metricsql" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/config" @@ -22,7 +21,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -46,6 +44,19 @@ type scrapesSecretsCache struct { baSecrets map[string]*k8stools.BasicAuthCredentials oauth2Secrets map[string]*k8stools.OAuthCreds authorizationSecrets map[string]string + nsSecretCache map[string]*corev1.Secret + nsCMCache map[string]*corev1.ConfigMap + tlsAssets map[string]string +} + +type scrapeObjects struct { + sss []*vmv1beta1.VMServiceScrape + pss []*vmv1beta1.VMPodScrape + stss []*vmv1beta1.VMStaticScrape + nss []*vmv1beta1.VMNodeScrape + prss []*vmv1beta1.VMProbe + scss []*vmv1beta1.VMScrapeConfig + badObjects []scrapeObjectWithStatus } // CreateOrUpdateConfigurationSecret builds scrape configuration for VMAgent @@ -60,7 +71,7 @@ func createOrUpdateConfigurationSecret(ctx context.Context, cr *vmv1beta1.VMAgen if cr.Spec.IngestOnlyMode { return nil, nil } - sScrapes, err := selectServiceScrapes(ctx, cr, rclient) + sss, err := selectServiceScrapes(ctx, cr, rclient) if err != nil { return nil, fmt.Errorf("selecting ServiceScrapes failed: %w", err) } @@ -89,26 +100,21 @@ func createOrUpdateConfigurationSecret(ctx context.Context, cr *vmv1beta1.VMAgen if err != nil { return nil, fmt.Errorf("selecting ScrapeConfigs failed: %w", err) } - - ssCache, err := loadScrapeSecrets(ctx, rclient, sScrapes, nodes, pScrapes, probes, statics, scrapeConfigs, cr.Spec.APIServerConfig, cr.Spec.RemoteWrite, cr.Namespace) - if err != nil { - var ge *utils.ErrGroup - if !stderrors.As(err, &ge) { - return nil, fmt.Errorf("cannot load scrape target secrets for api server or remote writes: %w", err) - } - vmagentSecretFetchErrsTotal.Inc() - logger.WithContext(ctx).Error(err, "found invalid secret references at objects, excluding it from configuration") + sos := &scrapeObjects{ + sss: sss, + pss: pScrapes, + prss: probes, + nss: nodes, + stss: statics, + scss: scrapeConfigs, } - assets, err := loadTLSAssets(ctx, rclient, cr, sScrapes, pScrapes, probes, nodes, statics) + + ssCache, err := loadScrapeSecrets(ctx, rclient, sos, cr.Namespace, cr.Spec.APIServerConfig, cr.Spec.RemoteWrite) if err != nil { - var ge *utils.ErrGroup - if !stderrors.As(err, &ge) { - return nil, fmt.Errorf("cannot load tls assets for api server or remote writes: %w", err) - } - vmagentSecretFetchErrsTotal.Inc() - logger.WithContext(ctx).Error(err, "cannot load tls assets for targets, excluding it from configuration") + return nil, fmt.Errorf("cannot load scrape target secrets for api server or remote writes: %w", err) } - if err := createOrUpdateTLSAssets(ctx, cr, rclient, assets); err != nil { + + if err := createOrUpdateTLSAssets(ctx, cr, rclient, ssCache.tlsAssets); err != nil { return nil, fmt.Errorf("cannot create tls assets secret for vmagent: %w", err) } @@ -121,12 +127,7 @@ func createOrUpdateConfigurationSecret(ctx context.Context, cr *vmv1beta1.VMAgen generatedConfig, err := generateConfig( ctx, cr, - sScrapes, - pScrapes, - probes, - nodes, - statics, - scrapeConfigs, + sos, ssCache, additionalScrapeConfigs, ) @@ -150,7 +151,10 @@ func createOrUpdateConfigurationSecret(ctx context.Context, cr *vmv1beta1.VMAgen if err := rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: s.Name}, curSecret); err != nil { if errors.IsNotFound(err) { logger.WithContext(ctx).Info("creating new configuration secret for vmagent") - return ssCache, rclient.Create(ctx, s) + if err := rclient.Create(ctx, s); err != nil { + return nil, fmt.Errorf("failed to create new configuration secret for vmagent: %w", err) + } + return ssCache, nil } return nil, fmt.Errorf("cannot get secret for vmagent: %q : %w", cr.Name, err) } @@ -161,539 +165,497 @@ func createOrUpdateConfigurationSecret(ctx context.Context, cr *vmv1beta1.VMAgen s.Annotations = labels.Merge(curSecret.Annotations, s.Annotations) vmv1beta1.AddFinalizer(s, curSecret) - return ssCache, rclient.Update(ctx, s) + if err := rclient.Update(ctx, s); err != nil { + return nil, fmt.Errorf("cannot update exist secret with config: %w", err) + } + + if err := updateStatusesForScrapeObjects(ctx, rclient, sos); err != nil { + return nil, err + } + + return ssCache, nil +} + +func updateStatusesForScrapeObjects(ctx context.Context, rclient client.Client, sos *scrapeObjects) error { + if len(sos.badObjects) > 0 { + var errorContexts []string + for _, bo := range sos.badObjects { + vmagentSecretFetchErrsTotal.Inc() + errorContexts = append(errorContexts, fmt.Sprintf("object=%s/%s/%s: sync error: %s", bo.GetObjectKind().GroupVersionKind().Kind, bo.GetNamespace(), bo.GetName(), bo.GetStatus().CurrentSyncError)) + } + logger.WithContext(ctx).Error(fmt.Errorf("found invalid secret references at objects"), "excluding it from configuration", "object_errors", strings.Join(errorContexts, ",")) + } + if err := updateStatusForEach(ctx, rclient, sos.badObjects, vmv1beta1.UpdateStatusFailed); err != nil { + return fmt.Errorf("cannot update statuses for bad scrape objects: %w", err) + } + if err := updateStatusForEach(ctx, rclient, sos.sss, vmv1beta1.UpdateStatusOperational); err != nil { + return fmt.Errorf("cannot update statuses for service scrape objects: %w", err) + } + if err := updateStatusForEach(ctx, rclient, sos.pss, vmv1beta1.UpdateStatusOperational); err != nil { + return fmt.Errorf("cannot update statuses for pod scrape objects: %w", err) + } + if err := updateStatusForEach(ctx, rclient, sos.nss, vmv1beta1.UpdateStatusOperational); err != nil { + return fmt.Errorf("cannot update statuses for node scrape objects: %w", err) + } + if err := updateStatusForEach(ctx, rclient, sos.prss, vmv1beta1.UpdateStatusOperational); err != nil { + return fmt.Errorf("cannot update statuses for probe scrape objects: %w", err) + } + if err := updateStatusForEach(ctx, rclient, sos.stss, vmv1beta1.UpdateStatusOperational); err != nil { + return fmt.Errorf("cannot update statuses for static scrape objects: %w", err) + } + if err := updateStatusForEach(ctx, rclient, sos.scss, vmv1beta1.UpdateStatusOperational); err != nil { + return fmt.Errorf("cannot update statuses for scrapeconfig scrape objects: %w", err) + } + + return nil +} + +func updateStatusForEach[T scrapeObjectWithStatus](ctx context.Context, rclient client.Client, iterable []T, desiredStatus vmv1beta1.UpdateStatus) error { + for _, so := range iterable { + cs := so.GetStatus() + if cs.Status != desiredStatus { + // patch update status + pt := client.RawPatch(types.MergePatchType, + []byte(fmt.Sprintf(`{"status": {"lastSyncError": %q , "status": %q} }`, cs.CurrentSyncError, desiredStatus))) + if err := rclient.Status().Patch(ctx, so, pt); err != nil { + return fmt.Errorf("failed to patch status of broken scrape object=%q: %w", so.GetName(), err) + } + } + } + return nil +} + +type scrapeObjectWithStatus interface { + client.Object + GetStatus() *vmv1beta1.ScrapeObjectStatus +} + +// returned objects with not found links have erased type +func forEachCollectSkipNotFound[T scrapeObjectWithStatus](src []T, apply func(s T) error) ([]T, []scrapeObjectWithStatus, error) { + var cnt int + var notNotFoundLinks []scrapeObjectWithStatus + for _, o := range src { + if err := apply(o); err != nil { + var ne *k8stools.KeyNotFoundError + switch { + case stderrors.As(err, &ne): + notNotFoundLinks = append(notNotFoundLinks, o) + case errors.IsNotFound(err): + notNotFoundLinks = append(notNotFoundLinks, o) + default: + return nil, nil, err + } + st := o.GetStatus() + st.CurrentSyncError = fmt.Sprintf("cannot find refrenced object: %s", err) + continue + } + src[cnt] = o + cnt++ + } + src = src[:cnt] + return src, notNotFoundLinks, nil +} + +func loadSecretsToCacheFrom(ctx context.Context, rclient client.Client, ep *vmv1beta1.EndpointAuth, cacheKey, namespace string, ss *scrapesSecretsCache) error { + if ep.BasicAuth != nil { + credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, ep.BasicAuth, namespace, ss.nsSecretCache) + if err != nil { + return fmt.Errorf("cannot load basicAuth secret for=%s: %w", cacheKey, err) + } + ss.baSecrets[cacheKey] = credentials + } + + if ep.OAuth2 != nil { + oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, ep.OAuth2, namespace, ss.nsSecretCache, ss.nsCMCache) + if err != nil { + return fmt.Errorf("cannot load oauth2 secret for=%s: %w", cacheKey, err) + } + ss.oauth2Secrets[cacheKey] = oauth2 + } + if ep.BearerTokenSecret != nil && ep.BearerTokenSecret.Name != "" { + token, err := k8stools.GetCredFromSecret(ctx, rclient, namespace, ep.BearerTokenSecret, buildCacheKey(namespace, ep.BearerTokenSecret.Name), ss.nsSecretCache) + if err != nil { + return fmt.Errorf("cannot load bearer secret for=%s: %w", cacheKey, err) + } + ss.bearerTokens[cacheKey] = token + } + if ep.Authorization != nil && ep.Authorization.Credentials != nil { + secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, namespace, ep.Authorization.Credentials, buildCacheKey(namespace, ep.Authorization.Credentials.Name), ss.nsSecretCache) + if err != nil { + return fmt.Errorf("cannot load authorization secret for=%s: %w", cacheKey, err) + } + ss.authorizationSecrets[cacheKey] = secretValue + } + + if err := addAssetsToCache(ctx, rclient, namespace, ep.TLSConfig, ss); err != nil { + return fmt.Errorf("cannot add tlsAsset for=%s %w", cacheKey, err) + } + + return nil } -// TODO: @f41gh7 -// refactor it, use vmv1beta1.HTTPAuth for objects as embed struct -// it should remove boilerplate code func loadScrapeSecrets( ctx context.Context, rclient client.Client, - mons map[string]*vmv1beta1.VMServiceScrape, - nodes map[string]*vmv1beta1.VMNodeScrape, - pods map[string]*vmv1beta1.VMPodScrape, - probes map[string]*vmv1beta1.VMProbe, - statics map[string]*vmv1beta1.VMStaticScrape, - scrapeConfigs map[string]*vmv1beta1.VMScrapeConfig, + sos *scrapeObjects, + vmagentCRNamespace string, apiserverConfig *vmv1beta1.APIServerConfig, remoteWriteSpecs []vmv1beta1.VMAgentRemoteWriteSpec, - namespace string, ) (*scrapesSecretsCache, error) { - oauth2Secret := make(map[string]*k8stools.OAuthCreds) - authorizationSecrets := make(map[string]string) - baSecrets := make(map[string]*k8stools.BasicAuthCredentials) - bearerSecrets := make(map[string]string) - nsSecretCache := make(map[string]*corev1.Secret) - nsCMCache := make(map[string]*corev1.ConfigMap) - var errG utils.ErrGroup - for key, mon := range mons { - var epCnt int + ssCache := &scrapesSecretsCache{ + baSecrets: map[string]*k8stools.BasicAuthCredentials{}, + oauth2Secrets: map[string]*k8stools.OAuthCreds{}, + bearerTokens: map[string]string{}, + authorizationSecrets: map[string]string{}, + nsSecretCache: map[string]*corev1.Secret{}, + nsCMCache: map[string]*corev1.ConfigMap{}, + tlsAssets: map[string]string{}, + } + var err error + var badObjects []scrapeObjectWithStatus + var tempBo []scrapeObjectWithStatus + sos.sss, tempBo, err = forEachCollectSkipNotFound(sos.sss, func(mon *vmv1beta1.VMServiceScrape) error { for i, ep := range mon.Spec.Endpoints { - if ep.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, ep.BasicAuth, mon.Namespace, nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMServiceScrape: %w", err)) - continue - } - baSecrets[mon.AsMapKey(i)] = credentials - } - - if ep.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, ep.OAuth2, mon.Namespace, nsSecretCache, nsCMCache) - if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMServiceScrape: %w", err)) - continue - } - oauth2Secret[mon.AsMapKey(i)] = oauth2 - } - if ep.BearerTokenSecret != nil && ep.BearerTokenSecret.Name != "" { - token, err := k8stools.GetCredFromSecret(ctx, rclient, mon.Namespace, ep.BearerTokenSecret, buildCacheKey(mon.Namespace, ep.BearerTokenSecret.Name), nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMServiceScrape: %w", err)) - continue - } - bearerSecrets[mon.AsMapKey(i)] = token - } - if ep.Authorization != nil && ep.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, mon.Namespace, ep.Authorization.Credentials, buildCacheKey(mon.Namespace, ep.Authorization.Credentials.Name), nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMServiceScrape: %w", err)) - continue - } - authorizationSecrets[mon.AsMapKey(i)] = secretValue + if err := loadSecretsToCacheFrom(ctx, rclient, &ep.EndpointAuth, mon.AsMapKey(i), mon.Namespace, ssCache); err != nil { + return err } if ep.VMScrapeParams != nil && ep.VMScrapeParams.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, ep.VMScrapeParams.ProxyClientConfig, mon.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, ep.VMScrapeParams.ProxyClientConfig, mon.Namespace, ssCache.nsSecretCache) if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMServiceScrape: %w", err)) - continue + return fmt.Errorf("cannot load secret for VMServiceScrape: %w", err) } if ba != nil { - baSecrets[mon.AsProxyKey(i)] = ba + ssCache.baSecrets[mon.AsProxyKey(i)] = ba } - bearerSecrets[mon.AsProxyKey(i)] = token + ssCache.bearerTokens[mon.AsProxyKey(i)] = token + if err := addAssetsToCache(ctx, rclient, mon.Namespace, ep.VMScrapeParams.ProxyClientConfig.TLSConfig, ssCache); err != nil { + return fmt.Errorf("cannot add proxy tlsAsset: %w", err) + } + } - mon.Spec.Endpoints[epCnt] = ep - epCnt++ - } - mon.Spec.Endpoints = mon.Spec.Endpoints[:epCnt] - if len(mon.Spec.Endpoints) == 0 { - delete(mons, key) } + return nil + }) + if err != nil { + return nil, err } + badObjects = append(badObjects, tempBo...) - for key, node := range nodes { - onErr := func(err error) { - delete(nodes, key) - errG.Add(fmt.Errorf("cannot load secret for VMNodeScrape: %w", err)) - } - if node.Spec.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, - rclient, - node.Spec.BasicAuth, - node.Namespace, - nsSecretCache) - if err != nil { - onErr(err) - continue - } - baSecrets[node.AsMapKey()] = credentials - - } - if node.Spec.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, node.Spec.OAuth2, node.Namespace, nsSecretCache, nsCMCache) - if err != nil { - onErr(err) - continue - } - oauth2Secret[node.AsMapKey()] = oauth2 - } - if node.Spec.BearerTokenSecret != nil && node.Spec.BearerTokenSecret.Name != "" { - token, err := k8stools.GetCredFromSecret(ctx, rclient, node.Namespace, node.Spec.BearerTokenSecret, buildCacheKey(node.Namespace, node.Spec.BearerTokenSecret.Name), nsSecretCache) - if err != nil { - onErr(err) - continue - } - bearerSecrets[node.AsMapKey()] = token + sos.nss, tempBo, err = forEachCollectSkipNotFound(sos.nss, func(node *vmv1beta1.VMNodeScrape) error { + if err := loadSecretsToCacheFrom(ctx, rclient, &node.Spec.EndpointAuth, node.AsMapKey(), node.Namespace, ssCache); err != nil { + return err } if node.Spec.VMScrapeParams != nil && node.Spec.VMScrapeParams.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, node.Spec.VMScrapeParams.ProxyClientConfig, node.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, node.Spec.VMScrapeParams.ProxyClientConfig, node.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(err) - continue + return err } if ba != nil { - baSecrets[node.AsProxyKey()] = ba + ssCache.baSecrets[node.AsProxyKey()] = ba + } + ssCache.bearerTokens[node.AsProxyKey()] = token + if err := addAssetsToCache(ctx, rclient, node.Namespace, node.Spec.VMScrapeParams.ProxyClientConfig.TLSConfig, ssCache); err != nil { + return fmt.Errorf("cannot add proxy tlsAsset: %w", err) } - bearerSecrets[node.AsProxyKey()] = token } + + return nil + }) + if err != nil { + return nil, err } - for key, pod := range pods { - var epCnt int + badObjects = append(badObjects, tempBo...) + + sos.pss, tempBo, err = forEachCollectSkipNotFound(sos.pss, func(pod *vmv1beta1.VMPodScrape) error { for i, ep := range pod.Spec.PodMetricsEndpoints { - if ep.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, ep.BasicAuth, pod.Namespace, nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMPodScrape: %w", err)) - continue - } - baSecrets[pod.AsMapKey(i)] = credentials - } - if ep.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, ep.OAuth2, pod.Namespace, nsSecretCache, nsCMCache) - if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMPodScrape: %w", err)) - continue - } - oauth2Secret[pod.AsMapKey(i)] = oauth2 - } - if ep.BearerTokenSecret != nil && ep.BearerTokenSecret.Name != "" { - token, err := k8stools.GetCredFromSecret(ctx, rclient, pod.Namespace, ep.BearerTokenSecret, buildCacheKey(pod.Namespace, ep.BearerTokenSecret.Name), nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMPodScrape: %w", err)) - continue - } - bearerSecrets[pod.AsMapKey(i)] = token + if err := loadSecretsToCacheFrom(ctx, rclient, &ep.EndpointAuth, pod.AsMapKey(i), pod.Namespace, ssCache); err != nil { + return err } if ep.VMScrapeParams != nil && ep.VMScrapeParams.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, ep.VMScrapeParams.ProxyClientConfig, pod.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, ep.VMScrapeParams.ProxyClientConfig, pod.Namespace, ssCache.nsSecretCache) if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMPodScrape: %w", err)) - continue + return fmt.Errorf("cannot load secret for VMPodScrape: %w", err) } if ba != nil { - baSecrets[pod.AsProxyKey(i)] = ba + ssCache.baSecrets[pod.AsProxyKey(i)] = ba } - bearerSecrets[pod.AsProxyKey(i)] = token - } - if ep.Authorization != nil && ep.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, pod.Namespace, ep.Authorization.Credentials, buildCacheKey(pod.Namespace, ep.Authorization.Credentials.Name), nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("cannot load secret for VMPodScrape: %w", err)) - continue + ssCache.bearerTokens[pod.AsProxyKey(i)] = token + if err := addAssetsToCache(ctx, rclient, pod.Namespace, ep.VMScrapeParams.ProxyClientConfig.TLSConfig, ssCache); err != nil { + return fmt.Errorf("cannot add proxy tlsAsset: %w", err) } - authorizationSecrets[pod.AsMapKey(i)] = secretValue } - pod.Spec.PodMetricsEndpoints[epCnt] = ep - epCnt++ - } - pod.Spec.PodMetricsEndpoints = pod.Spec.PodMetricsEndpoints[:epCnt] - if len(pod.Spec.PodMetricsEndpoints) == 0 { - delete(pods, key) } + + return nil + }) + if err != nil { + return nil, err } + badObjects = append(badObjects, tempBo...) - for key, probe := range probes { - onErr := func(err error) { - delete(nodes, key) - errG.Add(fmt.Errorf("cannot load secret for VMProbe: %w", err)) - } - if probe.Spec.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, probe.Spec.BasicAuth, probe.Namespace, nsSecretCache) - if err != nil { - onErr(fmt.Errorf("could not generate basicAuth for vmstaticScrape %s. %w", probe.Name, err)) - continue - } - baSecrets[probe.AsMapKey()] = credentials - } - if probe.Spec.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, probe.Spec.OAuth2, probe.Namespace, nsSecretCache, nsCMCache) - if err != nil { - onErr(err) - continue - } - oauth2Secret[probe.AsMapKey()] = oauth2 - } - if probe.Spec.BearerTokenSecret != nil && probe.Spec.BearerTokenSecret.Name != "" { - token, err := k8stools.GetCredFromSecret(ctx, rclient, probe.Namespace, probe.Spec.BearerTokenSecret, buildCacheKey(probe.Namespace, probe.Spec.BearerTokenSecret.Name), nsSecretCache) - if err != nil { - onErr(err) - continue - } - bearerSecrets[probe.AsMapKey()] = token + sos.prss, tempBo, err = forEachCollectSkipNotFound(sos.prss, func(probe *vmv1beta1.VMProbe) error { + if err := loadSecretsToCacheFrom(ctx, rclient, &probe.Spec.EndpointAuth, probe.AsMapKey(), probe.Namespace, ssCache); err != nil { + return err } if probe.Spec.VMScrapeParams != nil && probe.Spec.VMScrapeParams.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, probe.Spec.VMScrapeParams.ProxyClientConfig, probe.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, probe.Spec.VMScrapeParams.ProxyClientConfig, probe.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(err) - continue + return err } if ba != nil { - baSecrets[probe.AsProxyKey()] = ba + ssCache.baSecrets[probe.AsProxyKey()] = ba } - bearerSecrets[probe.AsProxyKey()] = token - } - if probe.Spec.Authorization != nil && probe.Spec.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, probe.Namespace, probe.Spec.Authorization.Credentials, buildCacheKey(probe.Namespace, probe.Spec.Authorization.Credentials.Name), nsSecretCache) - if err != nil { - onErr(err) - continue + ssCache.bearerTokens[probe.AsProxyKey()] = token + if err := addAssetsToCache(ctx, rclient, probe.Namespace, probe.Spec.VMScrapeParams.ProxyClientConfig.TLSConfig, ssCache); err != nil { + return fmt.Errorf("cannot add proxy tlsAsset: %w", err) } - authorizationSecrets[probe.AsMapKey()] = secretValue } + return nil + }) + if err != nil { + return nil, err } + badObjects = append(badObjects, tempBo...) - for key, staticCfg := range statics { - var epCnt int + sos.stss, tempBo, err = forEachCollectSkipNotFound(sos.stss, func(staticCfg *vmv1beta1.VMStaticScrape) error { for i, ep := range staticCfg.Spec.TargetEndpoints { - if ep.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, ep.BasicAuth, staticCfg.Namespace, nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("could not load secret for vmstaticScrape: %w", err)) - continue - } - baSecrets[staticCfg.AsMapKey(i)] = credentials - } - if ep.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, ep.OAuth2, staticCfg.Namespace, nsSecretCache, nsCMCache) - if err != nil { - errG.Add(fmt.Errorf("could not load secret for vmstaticScrape: %w", err)) - continue - } - oauth2Secret[staticCfg.AsMapKey(i)] = oauth2 - } - if ep.BearerTokenSecret != nil && ep.BearerTokenSecret.Name != "" { - token, err := k8stools.GetCredFromSecret(ctx, rclient, staticCfg.Namespace, ep.BearerTokenSecret, buildCacheKey(staticCfg.Namespace, ep.BearerTokenSecret.Name), nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("could not load secret for vmstaticScrape: %w", err)) - continue - } - bearerSecrets[staticCfg.AsMapKey(i)] = token + if err := loadSecretsToCacheFrom(ctx, rclient, &ep.EndpointAuth, staticCfg.AsMapKey(i), staticCfg.Namespace, ssCache); err != nil { + return err } + if ep.VMScrapeParams != nil && ep.VMScrapeParams.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, ep.VMScrapeParams.ProxyClientConfig, staticCfg.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, ep.VMScrapeParams.ProxyClientConfig, staticCfg.Namespace, ssCache.nsSecretCache) if err != nil { - errG.Add(fmt.Errorf("could not load secret for vmstaticScrape: %w", err)) - continue + return fmt.Errorf("could not load secret for vmstaticScrape: %w", err) } if ba != nil { - baSecrets[staticCfg.AsProxyKey(i)] = ba + ssCache.baSecrets[staticCfg.AsProxyKey(i)] = ba } - bearerSecrets[staticCfg.AsProxyKey(i)] = token - } - if ep.Authorization != nil && ep.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, staticCfg.Namespace, ep.Authorization.Credentials, buildCacheKey(staticCfg.Namespace, ep.Authorization.Credentials.Name), nsSecretCache) - if err != nil { - errG.Add(fmt.Errorf("could not load secret for vmstaticScrape: %w", err)) - continue + ssCache.bearerTokens[staticCfg.AsProxyKey(i)] = token + if err := addAssetsToCache(ctx, rclient, staticCfg.Namespace, ep.VMScrapeParams.ProxyClientConfig.TLSConfig, ssCache); err != nil { + return fmt.Errorf("cannot add proxy tlsAsset: %w", err) } - authorizationSecrets[staticCfg.AsMapKey(i)] = secretValue } - staticCfg.Spec.TargetEndpoints[epCnt] = ep - epCnt++ - } - staticCfg.Spec.TargetEndpoints = staticCfg.Spec.TargetEndpoints[:epCnt] - if len(staticCfg.Spec.TargetEndpoints) == 0 { - delete(statics, key) } + return nil + }) + if err != nil { + return nil, err } + badObjects = append(badObjects, tempBo...) - for key, scrapeConfig := range scrapeConfigs { - onErr := func(err error) { - delete(nodes, key) - errG.Add(fmt.Errorf("cannot load secret for VMScrapeConfig: %w", err)) - } - if scrapeConfig.Spec.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, scrapeConfig.Spec.BasicAuth, scrapeConfig.Namespace, nsSecretCache) - if err != nil { - onErr(fmt.Errorf("could not generate basicAuth for VMScrapeConfig %s. %w", scrapeConfig.Name, err)) - continue - } - baSecrets[scrapeConfig.AsMapKey("", 0)] = credentials - } - if scrapeConfig.Spec.Authorization != nil && scrapeConfig.Spec.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, scrapeConfig.Spec.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, scrapeConfig.Spec.Authorization.Credentials.Name), nsSecretCache) - if err != nil { - onErr(fmt.Errorf("could not generate authorization for VMScrapeConfig %s. %w", scrapeConfig.Name, err)) - continue - } - authorizationSecrets[scrapeConfig.AsMapKey("", 0)] = secretValue - } - if scrapeConfig.Spec.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, scrapeConfig.Spec.OAuth2, scrapeConfig.Namespace, nsSecretCache, nsCMCache) - if err != nil { - onErr(fmt.Errorf("could not generate oauth2 for VMScrapeConfig %s. %w", scrapeConfig.Name, err)) - continue - } - oauth2Secret[scrapeConfig.AsMapKey("", 0)] = oauth2 + sos.scss, tempBo, err = forEachCollectSkipNotFound(sos.scss, func(scrapeConfig *vmv1beta1.VMScrapeConfig) error { + if err := loadSecretsToCacheFrom(ctx, rclient, &scrapeConfig.Spec.EndpointAuth, scrapeConfig.AsMapKey("", 0), scrapeConfig.Namespace, ssCache); err != nil { + return err } if scrapeConfig.Spec.VMScrapeParams != nil && scrapeConfig.Spec.VMScrapeParams.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, scrapeConfig.Spec.VMScrapeParams.ProxyClientConfig, scrapeConfig.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, scrapeConfig.Spec.VMScrapeParams.ProxyClientConfig, scrapeConfig.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate proxy auth for VMScrapeConfig %s. %w", scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate proxy auth for VMScrapeConfig %s. %w", scrapeConfig.Name, err) } if ba != nil { - baSecrets[scrapeConfig.AsProxyKey("", 0)] = ba + ssCache.baSecrets[scrapeConfig.AsProxyKey("", 0)] = ba + } + ssCache.bearerTokens[scrapeConfig.AsProxyKey("", 0)] = token + if err := addAssetsToCache(ctx, rclient, scrapeConfig.Namespace, scrapeConfig.Spec.VMScrapeParams.ProxyClientConfig.TLSConfig, ssCache); err != nil { + return fmt.Errorf("cannot add proxy tlsAsset: %w", err) } - bearerSecrets[scrapeConfig.AsProxyKey("", 0)] = token } for i, hc := range scrapeConfig.Spec.HTTPSDConfigs { if hc.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, hc.BasicAuth, scrapeConfig.Namespace, nsSecretCache) + credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, hc.BasicAuth, scrapeConfig.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate basicAuth for httpSDConfig %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate basicAuth for httpSDConfig %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - baSecrets[scrapeConfig.AsMapKey("httpsd", i)] = credentials + ssCache.baSecrets[scrapeConfig.AsMapKey("httpsd", i)] = credentials } if hc.Authorization != nil && hc.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, hc.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, hc.Authorization.Credentials.Name), nsSecretCache) + secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, hc.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, hc.Authorization.Credentials.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate authorization for httpSDConfig %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate authorization for httpSDConfig %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - authorizationSecrets[scrapeConfig.AsMapKey("httpsd", i)] = secretValue + ssCache.authorizationSecrets[scrapeConfig.AsMapKey("httpsd", i)] = secretValue } if hc.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, hc.ProxyClientConfig, scrapeConfig.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, hc.ProxyClientConfig, scrapeConfig.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate proxy auth for httpSDConfig %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate proxy auth for httpSDConfig %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } if ba != nil { - baSecrets[scrapeConfig.AsProxyKey("httpsd", i)] = ba + ssCache.baSecrets[scrapeConfig.AsProxyKey("httpsd", i)] = ba } - bearerSecrets[scrapeConfig.AsProxyKey("httpsd", i)] = token + ssCache.bearerTokens[scrapeConfig.AsProxyKey("httpsd", i)] = token } } for i, kc := range scrapeConfig.Spec.KubernetesSDConfigs { if kc.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, kc.BasicAuth, scrapeConfig.Namespace, nsSecretCache) + credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, kc.BasicAuth, scrapeConfig.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate basicAuth for kubernetesSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate basicAuth for kubernetesSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - baSecrets[scrapeConfig.AsMapKey("kubesd", i)] = credentials + ssCache.baSecrets[scrapeConfig.AsMapKey("kubesd", i)] = credentials } if kc.Authorization != nil && kc.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, kc.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, kc.Authorization.Credentials.Name), nsSecretCache) + secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, kc.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, kc.Authorization.Credentials.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate authorization for kubernetesSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate authorization for kubernetesSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - authorizationSecrets[scrapeConfig.AsMapKey("kubesd", i)] = secretValue + ssCache.authorizationSecrets[scrapeConfig.AsMapKey("kubesd", i)] = secretValue } if kc.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, kc.OAuth2, scrapeConfig.Namespace, nsSecretCache, nsCMCache) + oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, kc.OAuth2, scrapeConfig.Namespace, ssCache.nsSecretCache, ssCache.nsCMCache) if err != nil { - onErr(fmt.Errorf("could not generate oauth2 for kubernetesSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate oauth2 for kubernetesSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - oauth2Secret[scrapeConfig.AsMapKey("kubesd", i)] = oauth2 + ssCache.oauth2Secrets[scrapeConfig.AsMapKey("kubesd", i)] = oauth2 } if kc.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, kc.ProxyClientConfig, scrapeConfig.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, kc.ProxyClientConfig, scrapeConfig.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate proxy auth for kubernetesSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate proxy auth for kubernetesSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } if ba != nil { - baSecrets[scrapeConfig.AsProxyKey("kubesd", i)] = ba + ssCache.baSecrets[scrapeConfig.AsProxyKey("kubesd", i)] = ba } - bearerSecrets[scrapeConfig.AsProxyKey("kubesd", i)] = token + ssCache.bearerTokens[scrapeConfig.AsProxyKey("kubesd", i)] = token } } for i, cc := range scrapeConfig.Spec.ConsulSDConfigs { if cc.TokenRef != nil { - token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, cc.TokenRef, buildCacheKey(scrapeConfig.Namespace, cc.TokenRef.Name), nsSecretCache) + token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, cc.TokenRef, buildCacheKey(scrapeConfig.Namespace, cc.TokenRef.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate token for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate token for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - bearerSecrets[scrapeConfig.AsMapKey("consulsd", i)] = token + ssCache.bearerTokens[scrapeConfig.AsMapKey("consulsd", i)] = token } if cc.BasicAuth != nil { - credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, cc.BasicAuth, scrapeConfig.Namespace, nsSecretCache) + credentials, err := loadBasicAuthSecretFromAPI(ctx, rclient, cc.BasicAuth, scrapeConfig.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate basicAuth for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate basicAuth for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - baSecrets[scrapeConfig.AsMapKey("consulsd", i)] = credentials + ssCache.baSecrets[scrapeConfig.AsMapKey("consulsd", i)] = credentials } if cc.Authorization != nil && cc.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, cc.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, cc.Authorization.Credentials.Name), nsSecretCache) + secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, cc.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, cc.Authorization.Credentials.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate authorization for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate authorization for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - authorizationSecrets[scrapeConfig.AsMapKey("consulsd", i)] = secretValue + ssCache.authorizationSecrets[scrapeConfig.AsMapKey("consulsd", i)] = secretValue } if cc.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, cc.OAuth2, scrapeConfig.Namespace, nsSecretCache, nsCMCache) + oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, cc.OAuth2, scrapeConfig.Namespace, ssCache.nsSecretCache, ssCache.nsCMCache) if err != nil { - onErr(fmt.Errorf("could not generate oauth2 for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate oauth2 for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - oauth2Secret[scrapeConfig.AsMapKey("consulsd", i)] = oauth2 + ssCache.oauth2Secrets[scrapeConfig.AsMapKey("consulsd", i)] = oauth2 } if cc.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, cc.ProxyClientConfig, scrapeConfig.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, cc.ProxyClientConfig, scrapeConfig.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate proxy auth for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate proxy auth for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } if ba != nil { - baSecrets[scrapeConfig.AsProxyKey("consulsd", i)] = ba + ssCache.baSecrets[scrapeConfig.AsProxyKey("consulsd", i)] = ba } - bearerSecrets[scrapeConfig.AsProxyKey("consulsd", i)] = token + ssCache.bearerTokens[scrapeConfig.AsProxyKey("consulsd", i)] = token } } for i, ec := range scrapeConfig.Spec.EC2SDConfigs { if ec.AccessKey != nil { - token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, ec.AccessKey, buildCacheKey(scrapeConfig.Namespace, ec.AccessKey.Name), nsSecretCache) + token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, ec.AccessKey, buildCacheKey(scrapeConfig.Namespace, ec.AccessKey.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate token for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate token for consulSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - authorizationSecrets[scrapeConfig.AsMapKey("ec2sdAccess", i)] = token + ssCache.authorizationSecrets[scrapeConfig.AsMapKey("ec2sdAccess", i)] = token } if ec.SecretKey != nil { - token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, ec.SecretKey, buildCacheKey(scrapeConfig.Namespace, ec.SecretKey.Name), nsSecretCache) + token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, ec.SecretKey, buildCacheKey(scrapeConfig.Namespace, ec.SecretKey.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate token for ec2SDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate token for ec2SDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - authorizationSecrets[scrapeConfig.AsMapKey("ec2sdSecret", i)] = token + ssCache.authorizationSecrets[scrapeConfig.AsMapKey("ec2sdSecret", i)] = token } } for i, ac := range scrapeConfig.Spec.AzureSDConfigs { if ac.ClientSecret != nil { - token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, ac.ClientSecret, buildCacheKey(scrapeConfig.Namespace, ac.ClientSecret.Name), nsSecretCache) + token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, ac.ClientSecret, buildCacheKey(scrapeConfig.Namespace, ac.ClientSecret.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate token for azureSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate token for azureSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - oauth2Secret[scrapeConfig.AsMapKey("azuresd", i)] = &k8stools.OAuthCreds{ClientSecret: token} + ssCache.oauth2Secrets[scrapeConfig.AsMapKey("azuresd", i)] = &k8stools.OAuthCreds{ClientSecret: token} } } for i, oc := range scrapeConfig.Spec.OpenStackSDConfigs { if oc.Password != nil { - token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, oc.Password, buildCacheKey(scrapeConfig.Namespace, oc.Password.Name), nsSecretCache) + token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, oc.Password, buildCacheKey(scrapeConfig.Namespace, oc.Password.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not read password for openStackSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not read password for openStackSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - authorizationSecrets[scrapeConfig.AsMapKey("openstacksd_password", i)] = token + ssCache.authorizationSecrets[scrapeConfig.AsMapKey("openstacksd_password", i)] = token } if oc.ApplicationCredentialSecret != nil { - token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, oc.ApplicationCredentialSecret, buildCacheKey(scrapeConfig.Namespace, oc.ApplicationCredentialSecret.Name), nsSecretCache) + token, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, oc.ApplicationCredentialSecret, buildCacheKey(scrapeConfig.Namespace, oc.ApplicationCredentialSecret.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not read applicationCredentialSecret for openStackSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not read applicationCredentialSecret for openStackSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - authorizationSecrets[scrapeConfig.AsMapKey("openstacksd_app", i)] = token + ssCache.authorizationSecrets[scrapeConfig.AsMapKey("openstacksd_app", i)] = token } } for i, dc := range scrapeConfig.Spec.DigitalOceanSDConfigs { if dc.Authorization != nil && dc.Authorization.Credentials != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, dc.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, dc.Authorization.Credentials.Name), nsSecretCache) + secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, scrapeConfig.Namespace, dc.Authorization.Credentials, buildCacheKey(scrapeConfig.Namespace, dc.Authorization.Credentials.Name), ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate authorization for digitalOceanSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate authorization for digitalOceanSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - authorizationSecrets[scrapeConfig.AsMapKey("digitaloceansd", i)] = secretValue + ssCache.authorizationSecrets[scrapeConfig.AsMapKey("digitaloceansd", i)] = secretValue } if dc.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, dc.OAuth2, scrapeConfig.Namespace, nsSecretCache, nsCMCache) + oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, dc.OAuth2, scrapeConfig.Namespace, ssCache.nsSecretCache, ssCache.nsCMCache) if err != nil { - onErr(fmt.Errorf("could not generate oauth2 for digitalOceanSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate oauth2 for digitalOceanSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } - oauth2Secret[scrapeConfig.AsMapKey("digitaloceansd", i)] = oauth2 + ssCache.oauth2Secrets[scrapeConfig.AsMapKey("digitaloceansd", i)] = oauth2 } if dc.ProxyClientConfig != nil { - ba, token, err := loadProxySecrets(ctx, rclient, dc.ProxyClientConfig, scrapeConfig.Namespace, nsSecretCache) + ba, token, err := loadProxySecrets(ctx, rclient, dc.ProxyClientConfig, scrapeConfig.Namespace, ssCache.nsSecretCache) if err != nil { - onErr(fmt.Errorf("could not generate proxy auth for digitalOceanSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err)) - continue + return fmt.Errorf("could not generate proxy auth for digitalOceanSDConfigs %d in VMScrapeConfig %s. %w", i, scrapeConfig.Name, err) } if ba != nil { - baSecrets[scrapeConfig.AsProxyKey("digitaloceansd", i)] = ba + ssCache.baSecrets[scrapeConfig.AsProxyKey("digitaloceansd", i)] = ba } - bearerSecrets[scrapeConfig.AsProxyKey("digitaloceansd", i)] = token + ssCache.bearerTokens[scrapeConfig.AsProxyKey("digitaloceansd", i)] = token } } + + return nil + }) + if err != nil { + return nil, err } + badObjects = append(badObjects, tempBo...) + // load apiserver basic auth secret // no need to filter out misconfiguration // it's VMAgent owner responsibility if apiserverConfig != nil { if apiserverConfig.BasicAuth != nil { - credentials, err := k8stools.LoadBasicAuthSecret(ctx, rclient, namespace, apiserverConfig.BasicAuth, nsSecretCache) + credentials, err := k8stools.LoadBasicAuthSecret(ctx, rclient, vmagentCRNamespace, apiserverConfig.BasicAuth, ssCache.nsSecretCache) if err != nil { return nil, fmt.Errorf("could not generate basicAuth for apiserver config. %w", err) } - baSecrets["apiserver"] = &credentials + ssCache.baSecrets["apiserver"] = &credentials } if apiserverConfig.Authorization != nil { - secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, namespace, apiserverConfig.Authorization.Credentials, buildCacheKey(namespace, "apiserver"), nsSecretCache) + secretValue, err := k8stools.GetCredFromSecret(ctx, rclient, vmagentCRNamespace, apiserverConfig.Authorization.Credentials, buildCacheKey(vmagentCRNamespace, "apiserver"), ssCache.nsSecretCache) if err != nil { return nil, fmt.Errorf("cannot fetch authorization secret for apiserver config: %w", err) } - authorizationSecrets["apiserver"] = secretValue + ssCache.authorizationSecrets["apiserver"] = secretValue + } + if err := addAssetsToCache(ctx, rclient, vmagentCRNamespace, apiserverConfig.TLSConfig, ssCache); err != nil { + return nil, fmt.Errorf("cannot add tls asset for apiServerConfig %w", err) } } @@ -702,29 +664,34 @@ func loadScrapeSecrets( // it's VMAgent owner responsibility for _, rws := range remoteWriteSpecs { if rws.BasicAuth != nil { - credentials, err := k8stools.LoadBasicAuthSecret(ctx, rclient, namespace, rws.BasicAuth, nsSecretCache) + credentials, err := k8stools.LoadBasicAuthSecret(ctx, rclient, vmagentCRNamespace, rws.BasicAuth, ssCache.nsSecretCache) if err != nil { return nil, fmt.Errorf("could not generate basicAuth for remote write spec %s config. %w", rws.URL, err) } - baSecrets[rws.AsMapKey()] = &credentials + ssCache.baSecrets[rws.AsMapKey()] = &credentials } if rws.OAuth2 != nil { - oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, rws.OAuth2, namespace, nsSecretCache, nsCMCache) + oauth2, err := k8stools.LoadOAuthSecrets(ctx, rclient, rws.OAuth2, vmagentCRNamespace, ssCache.nsSecretCache, ssCache.nsCMCache) if err != nil { - return nil, fmt.Errorf("cannot load oauth2 creds for :%s, ns: %s, err: %w", "remoteWrite", namespace, err) + return nil, fmt.Errorf("cannot load oauth2 creds for :%s, ns: %s, err: %w", "remoteWrite", vmagentCRNamespace, err) } - oauth2Secret[rws.AsMapKey()] = oauth2 + ssCache.oauth2Secrets[rws.AsMapKey()] = oauth2 } if rws.BearerTokenSecret != nil && rws.BearerTokenSecret.Name != "" { - token, err := k8stools.GetCredFromSecret(ctx, rclient, namespace, rws.BearerTokenSecret, buildCacheKey(namespace, rws.BearerTokenSecret.Name), nsSecretCache) + token, err := k8stools.GetCredFromSecret(ctx, rclient, vmagentCRNamespace, rws.BearerTokenSecret, buildCacheKey(vmagentCRNamespace, rws.BearerTokenSecret.Name), ssCache.nsSecretCache) if err != nil { return nil, fmt.Errorf("cannot get bearer token for remoteWrite: %w", err) } - bearerSecrets[rws.AsMapKey()] = token + ssCache.bearerTokens[rws.AsMapKey()] = token + } + if err := addAssetsToCache(ctx, rclient, vmagentCRNamespace, rws.TLSConfig, ssCache); err != nil { + return nil, fmt.Errorf("cannot add asset for remote write target: %w", err) } + } + sos.badObjects = badObjects - return &scrapesSecretsCache{baSecrets: baSecrets, oauth2Secrets: oauth2Secret, bearerTokens: bearerSecrets, authorizationSecrets: authorizationSecrets}, errG.Err() + return ssCache, nil } func loadBasicAuthSecretFromAPI(ctx context.Context, rclient client.Client, basicAuth *vmv1beta1.BasicAuth, ns string, cache map[string]*corev1.Secret) (*k8stools.BasicAuthCredentials, error) { @@ -884,12 +851,7 @@ var invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`) func generateConfig( ctx context.Context, cr *vmv1beta1.VMAgent, - sMons map[string]*vmv1beta1.VMServiceScrape, - pMons map[string]*vmv1beta1.VMPodScrape, - probes map[string]*vmv1beta1.VMProbe, - nodes map[string]*vmv1beta1.VMNodeScrape, - statics map[string]*vmv1beta1.VMStaticScrape, - scrapeConfs map[string]*vmv1beta1.VMScrapeConfig, + sos *scrapeObjects, secretsCache *scrapesSecretsCache, additionalScrapeConfigs []byte, ) ([]byte, error) { @@ -916,74 +878,16 @@ func generateConfig( cfg = append(cfg, yaml.MapItem{Key: "global", Value: globalItems}) - sMonIdentifiers := make([]string, len(sMons)) - i := 0 - for k := range sMons { - sMonIdentifiers[i] = k - i++ - } - - // Sorting ensures, that we always generate the config in the same order. - sort.Strings(sMonIdentifiers) - - pMonIdentifiers := make([]string, len(pMons)) - i = 0 - for k := range pMons { - pMonIdentifiers[i] = k - i++ - } - - // Sorting ensures, that we always generate the config in the same order. - sort.Strings(pMonIdentifiers) - - probeIdentifiers := make([]string, len(probes)) - i = 0 - for k := range probes { - probeIdentifiers[i] = k - i++ - } - // Sorting ensures, that we always generate the config in the same order. - sort.Strings(probeIdentifiers) - - nodeIdentifiers := make([]string, len(nodes)) - i = 0 - for k := range nodes { - nodeIdentifiers[i] = k - i++ - } - // Sorting ensures, that we always generate the config in the same order. - sort.Strings(nodeIdentifiers) - - staticsIdentifiers := make([]string, len(statics)) - i = 0 - for k := range statics { - staticsIdentifiers[i] = k - i++ - } - - // Sorting ensures, that we always generate the config in the same order. - sort.Strings(staticsIdentifiers) - - scrapeConfigIdentifiers := make([]string, len(scrapeConfs)) - i = 0 - for k := range scrapeConfs { - scrapeConfigIdentifiers[i] = k - i++ - } - - // Sorting ensures, that we always generate the config in the same order. - sort.Strings(scrapeConfigIdentifiers) - apiserverConfig := cr.Spec.APIServerConfig var scrapeConfigs []yaml.MapSlice - for _, identifier := range sMonIdentifiers { - for i, ep := range sMons[identifier].Spec.Endpoints { + for _, ss := range sos.sss { + for i, ep := range ss.Spec.Endpoints { scrapeConfigs = append(scrapeConfigs, generateServiceScrapeConfig( ctx, cr, - sMons[identifier], + ss, ep, i, apiserverConfig, secretsCache, @@ -991,13 +895,13 @@ func generateConfig( )) } } - for _, identifier := range pMonIdentifiers { - for i, ep := range pMons[identifier].Spec.PodMetricsEndpoints { + for _, identifier := range sos.pss { + for i, ep := range identifier.Spec.PodMetricsEndpoints { scrapeConfigs = append(scrapeConfigs, generatePodScrapeConfig( ctx, cr, - pMons[identifier], ep, i, + identifier, ep, i, apiserverConfig, secretsCache, cr.Spec.VMAgentSecurityEnforcements, @@ -1005,24 +909,24 @@ func generateConfig( } } - for i, identifier := range probeIdentifiers { + for i, identifier := range sos.prss { scrapeConfigs = append(scrapeConfigs, generateProbeConfig( ctx, cr, - probes[identifier], + identifier, i, apiserverConfig, secretsCache, cr.Spec.VMAgentSecurityEnforcements, )) } - for i, identifier := range nodeIdentifiers { + for i, identifier := range sos.nss { scrapeConfigs = append(scrapeConfigs, generateNodeScrapeConfig( ctx, cr, - nodes[identifier], + identifier, i, apiserverConfig, secretsCache, @@ -1030,13 +934,13 @@ func generateConfig( )) } - for _, identifier := range staticsIdentifiers { - for i, ep := range statics[identifier].Spec.TargetEndpoints { + for _, identifier := range sos.stss { + for i, ep := range identifier.Spec.TargetEndpoints { scrapeConfigs = append(scrapeConfigs, generateStaticScrapeConfig( ctx, cr, - statics[identifier], + identifier, ep, i, secretsCache, cr.Spec.VMAgentSecurityEnforcements, @@ -1044,12 +948,12 @@ func generateConfig( } } - for _, identifier := range scrapeConfigIdentifiers { + for _, identifier := range sos.scss { scrapeConfigs = append(scrapeConfigs, generateScrapeConfig( ctx, cr, - scrapeConfs[identifier], + identifier, secretsCache, cr.Spec.VMAgentSecurityEnforcements, )) @@ -1075,8 +979,8 @@ func generateConfig( return yaml.Marshal(cfg) } -func makeConfigSecret(cr *vmv1beta1.VMAgent, config *config.BaseOperatorConf, ssCache *scrapesSecretsCache) *v1.Secret { - s := &v1.Secret{ +func makeConfigSecret(cr *vmv1beta1.VMAgent, config *config.BaseOperatorConf, ssCache *scrapesSecretsCache) *corev1.Secret { + s := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: cr.PrefixedName(), Annotations: cr.AnnotationsFiltered(), diff --git a/internal/controller/operator/factory/vmagent/vmagent_scrapeconfig_test.go b/internal/controller/operator/factory/vmagent/vmagent_scrapeconfig_test.go index aab06ad1..0b755650 100644 --- a/internal/controller/operator/factory/vmagent/vmagent_scrapeconfig_test.go +++ b/internal/controller/operator/factory/vmagent/vmagent_scrapeconfig_test.go @@ -965,46 +965,6 @@ scrape_configs: replacement: ${1} - target_label: endpoint replacement: "8011" -- job_name: podScrape/default/test-vps-mixed/0 - kubernetes_sd_configs: - - role: pod - namespaces: - names: - - default - honor_labels: false - metrics_path: /metrics-5-good - sample_limit: 10 - relabel_configs: - - action: drop - source_labels: - - __meta_kubernetes_pod_phase - regex: (Failed|Succeeded) - - action: keep - source_labels: - - __meta_kubernetes_pod_label_app - regex: prod - - action: keep - source_labels: - - __meta_kubernetes_pod_container_port_name - regex: "801" - - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container - - source_labels: - - __meta_kubernetes_pod_name - target_label: pod - - target_label: job - replacement: default/test-vps-mixed - - source_labels: - - __meta_kubernetes_pod_label_app - target_label: job - regex: (.+) - replacement: ${1} - - target_label: endpoint - replacement: "801" - job_name: nodeScrape/default/test-good/0 kubernetes_sd_configs: - role: node diff --git a/internal/controller/operator/factory/vmagent/vmagent_test.go b/internal/controller/operator/factory/vmagent/vmagent_test.go index 18d0f57f..1496bd48 100644 --- a/internal/controller/operator/factory/vmagent/vmagent_test.go +++ b/internal/controller/operator/factory/vmagent/vmagent_test.go @@ -458,12 +458,12 @@ func TestCreateOrUpdateVMAgent(t *testing.T) { func Test_loadTLSAssets(t *testing.T) { type args struct { - monitors map[string]*vmv1beta1.VMServiceScrape - pods map[string]*vmv1beta1.VMPodScrape - statics map[string]*vmv1beta1.VMStaticScrape - nodes map[string]*vmv1beta1.VMNodeScrape - probes map[string]*vmv1beta1.VMProbe - cr *vmv1beta1.VMAgent + servicescrapes []*vmv1beta1.VMServiceScrape + podscrapes []*vmv1beta1.VMPodScrape + statics []*vmv1beta1.VMStaticScrape + nodes []*vmv1beta1.VMNodeScrape + probes []*vmv1beta1.VMProbe + cr *vmv1beta1.VMAgent } tests := []struct { name string @@ -478,8 +478,8 @@ func Test_loadTLSAssets(t *testing.T) { cr: &vmv1beta1.VMAgent{ Spec: vmv1beta1.VMAgentSpec{}, }, - monitors: map[string]*vmv1beta1.VMServiceScrape{ - "vmagent-monitor": { + servicescrapes: []*vmv1beta1.VMServiceScrape{ + { ObjectMeta: metav1.ObjectMeta{Name: "vmagent-monitor", Namespace: "default"}, Spec: vmv1beta1.VMServiceScrapeSpec{ Endpoints: []vmv1beta1.Endpoint{ @@ -511,6 +511,96 @@ func Test_loadTLSAssets(t *testing.T) { }, want: map[string]string{"default_tls-secret_cert": "cert-data"}, }, + { + name: "load tls asset from secret for proxy tls", + args: args{ + cr: &vmv1beta1.VMAgent{ + Spec: vmv1beta1.VMAgentSpec{}, + }, + podscrapes: []*vmv1beta1.VMPodScrape{ + { + ObjectMeta: metav1.ObjectMeta{Name: "single-pod", Namespace: "ns-1"}, + Spec: vmv1beta1.VMPodScrapeSpec{ + PodMetricsEndpoints: []vmv1beta1.PodMetricsEndpoint{ + { + Port: "8080", + EndpointScrapeParams: vmv1beta1.EndpointScrapeParams{ + VMScrapeParams: &vmv1beta1.VMScrapeParams{ + ProxyClientConfig: &vmv1beta1.ProxyAuth{ + TLSConfig: &vmv1beta1.TLSConfig{ + CA: vmv1beta1.SecretOrConfigMap{ + Secret: &corev1.SecretKeySelector{ + Key: "ca", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tls-access", + }, + }, + }, + Cert: vmv1beta1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + Key: "cert", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tls-cm", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + servicescrapes: []*vmv1beta1.VMServiceScrape{ + { + ObjectMeta: metav1.ObjectMeta{Name: "vmagent-monitor", Namespace: "default"}, + Spec: vmv1beta1.VMServiceScrapeSpec{ + Endpoints: []vmv1beta1.Endpoint{ + { + EndpointAuth: vmv1beta1.EndpointAuth{ + TLSConfig: &vmv1beta1.TLSConfig{ + KeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tls-secret", + }, + Key: "cert", + }, + }, + }, + }, + }, + }, + }, + }, + }, + predefinedObjects: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-secret", + Namespace: "default", + }, + Data: map[string][]byte{"cert": []byte(`cert-data`)}, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-access", + Namespace: "ns-1", + }, + Data: map[string][]byte{"ca": []byte(`cert-data`)}, + }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-cm", + Namespace: "ns-1", + }, + Data: map[string]string{"cert": `cert-data`}, + }, + }, + want: map[string]string{"default_tls-secret_cert": "cert-data", "ns-1_tls-access_ca": "cert-data", "ns-1_tls-cm_cert": "cert-data"}, + }, + { name: "load tls asset from secret with remoteWrite tls", args: args{ @@ -554,12 +644,13 @@ func Test_loadTLSAssets(t *testing.T) { }, }, }, - monitors: map[string]*vmv1beta1.VMServiceScrape{ - "vmagent-monitor": { + servicescrapes: []*vmv1beta1.VMServiceScrape{ + { ObjectMeta: metav1.ObjectMeta{Name: "vmagent-monitor", Namespace: "default"}, Spec: vmv1beta1.VMServiceScrapeSpec{ Endpoints: []vmv1beta1.Endpoint{ { + Port: "8080", EndpointAuth: vmv1beta1.EndpointAuth{ TLSConfig: &vmv1beta1.TLSConfig{ KeySecret: &corev1.SecretKeySelector{ @@ -571,18 +662,54 @@ func Test_loadTLSAssets(t *testing.T) { }, }, }, + { + Port: "8081", + EndpointAuth: vmv1beta1.EndpointAuth{ + // check for secret and configmap naming clash + TLSConfig: &vmv1beta1.TLSConfig{ + Cert: vmv1beta1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + Key: "clash-key", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "name-clash", + }, + }, + }, + KeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "name-clash", + }, + Key: "clash-key", + }, + }, + }, + }, }, }, }, }, }, predefinedObjects: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name-clash", + Namespace: "default", + }, + Data: map[string][]byte{"clash-key": []byte(`value-1`)}, + }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name-clash", + Namespace: "default", + }, + Data: map[string]string{"clash-key": `value-2`}, + }, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "tls-secret", Namespace: "default", }, - Data: map[string][]byte{"cert": []byte(`cert-data`)}, + Data: map[string][]byte{"cert": []byte(`cert-data`), "clash-key": []byte(`value-1`)}, }, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -592,21 +719,29 @@ func Test_loadTLSAssets(t *testing.T) { Data: map[string][]byte{"cert": []byte(`cert-data`), "key": []byte(`cert-key`), "ca": []byte(`cert-ca`)}, }, }, - want: map[string]string{"default_tls-secret_cert": "cert-data", "default_remote1-write-spec_ca": "cert-ca", "default_remote1-write-spec_cert": "cert-data", "default_remote1-write-spec_key": "cert-key"}, + want: map[string]string{ + "default_tls-secret_cert": "cert-data", "default_remote1-write-spec_ca": "cert-ca", "default_remote1-write-spec_cert": "cert-data", "default_remote1-write-spec_key": "cert-key", + "default_name-clash_clash-key": "value-2", + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fclient := k8stools.GetTestClientWithObjects(tt.predefinedObjects) - got, err := loadTLSAssets(context.TODO(), fclient, tt.args.cr, tt.args.monitors, tt.args.pods, tt.args.probes, tt.args.nodes, tt.args.statics) + sos := &scrapeObjects{ + sss: tt.args.servicescrapes, + pss: tt.args.podscrapes, + prss: tt.args.probes, + nss: tt.args.nodes, + stss: tt.args.statics, + } + got, err := loadScrapeSecrets(context.TODO(), fclient, sos, tt.args.cr.Namespace, tt.args.cr.Spec.APIServerConfig, tt.args.cr.Spec.RemoteWrite) if (err != nil) != tt.wantErr { t.Errorf("loadTLSAssets() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("loadTLSAssets() got = %v, want %v", got, tt.want) - } + assert.Equal(t, tt.want, got.tlsAssets) }) } }