From d91b4245d6a110be46fb6d1a1906c9e02d392487 Mon Sep 17 00:00:00 2001 From: Amit Schendel <58078857+amitschendel@users.noreply.github.com> Date: Sun, 7 Jan 2024 17:01:46 +0200 Subject: [PATCH] Feature/multiple am (#128) * Updating kubecop chart to support endpoints and adding a new demo chart Signed-off-by: Amit Schendel * Adding support for multiple alert managers Signed-off-by: Amit Schendel * Removing kubecop from helm chart Signed-off-by: Amit Schendel * Fixing conflicts Signed-off-by: Amit Schendel * Adding fix for system test Signed-off-by: Amit Schendel --------- Signed-off-by: Amit Schendel --- README.md | 2 +- chart/demo/.helmignore | 23 +++++++ chart/demo/Chart.yaml | 9 +++ chart/demo/templates/_helpers.tpl | 62 +++++++++++++++++++ .../templates/alertmanager-configmap.yaml | 24 +++++++ .../demo/templates/alertmanager-service.yaml | 13 ++++ chart/demo/templates/alertmanager.yaml | 30 +++++++++ chart/demo/values.yaml | 13 ++++ chart/kubecop/templates/deamonset.yaml | 4 +- chart/kubecop/values.yaml | 2 +- pkg/exporters/README.md | 3 +- pkg/exporters/alert_manager.go | 6 -- pkg/exporters/alert_manager_test.go | 1 - pkg/exporters/exporters_bus.go | 40 +++++++++--- resources/latest/kubecop-values.yaml | 2 +- resources/system-tests/kubecop-values.yaml | 2 +- 16 files changed, 214 insertions(+), 22 deletions(-) create mode 100644 chart/demo/.helmignore create mode 100644 chart/demo/Chart.yaml create mode 100644 chart/demo/templates/_helpers.tpl create mode 100644 chart/demo/templates/alertmanager-configmap.yaml create mode 100644 chart/demo/templates/alertmanager-service.yaml create mode 100644 chart/demo/templates/alertmanager.yaml create mode 100644 chart/demo/values.yaml diff --git a/README.md b/README.md index a5479f4..ecb812d 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ They can be enabled with Helm (Stdout is on by default) Currently supported: * Alert manager * Enable: `kubecop.alertmanager.enabled` - * Endpoint: `kubecop.alertmanager.endpoint` (example `localhost:9093`) + * Endpoint: `kubecop.alertmanager.endpoints` (example `localhost:9093` or `localhost:9093,localhost:9094`) * Syslog (RFC 5424) * Enable: `kubecop.syslog.enabled` * Endpoint: `kubecop.syslog.endpoint` (example `localhost:514`) diff --git a/chart/demo/.helmignore b/chart/demo/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/chart/demo/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/chart/demo/Chart.yaml b/chart/demo/Chart.yaml new file mode 100644 index 0000000..00a1857 --- /dev/null +++ b/chart/demo/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: kubecop-alertmanager +description: A Helm chart for kubecop-alertmanager + +type: application + +version: 0.0.1 + +appVersion: "0.0.1" diff --git a/chart/demo/templates/_helpers.tpl b/chart/demo/templates/_helpers.tpl new file mode 100644 index 0000000..7fc608d --- /dev/null +++ b/chart/demo/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "..name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "..fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "..chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "..labels" -}} +helm.sh/chart: {{ include "..chart" . }} +{{ include "..selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "..selectorLabels" -}} +app.kubernetes.io/name: {{ include "..name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "..serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "..fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/chart/demo/templates/alertmanager-configmap.yaml b/chart/demo/templates/alertmanager-configmap.yaml new file mode 100644 index 0000000..087535b --- /dev/null +++ b/chart/demo/templates/alertmanager-configmap.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: alertmanager-config + namespace: kubescape +data: + alertmanager.yml: | + global: + smtp_smarthost: {{ .Values.kubecop.alertmanager.smtp_endpoint }} + smtp_from: '{{ .Values.kubecop.alertmanager.smtp_from }}' + smtp_auth_username: '{{ .Values.kubecop.alertmanager.smtp_auth_username }}' + smtp_auth_password: '{{ .Values.kubecop.alertmanager.smtp_auth_password }}' + smtp_auth_secret: '{{ .Values.kubecop.alertmanager.smtp_auth_secret }}' + smtp_auth_identity: '{{ .Values.kubecop.alertmanager.smtp_auth_identity }}' + smtp_require_tls: {{ .Values.kubecop.alertmanager.smtp_require_tls }} + + route: + group_by: ['alertname'] + receiver: 'email-notifications' + + receivers: + - name: 'email-notifications' + email_configs: + - to: '{{ .Values.kubecop.alertmanager.email_to }}' diff --git a/chart/demo/templates/alertmanager-service.yaml b/chart/demo/templates/alertmanager-service.yaml new file mode 100644 index 0000000..ddc8d28 --- /dev/null +++ b/chart/demo/templates/alertmanager-service.yaml @@ -0,0 +1,13 @@ +# alertmanager-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: alertmanager-kubecop-service + namespace: kubescape +spec: + selector: + app: alertmanager-kubecop # This should match the labels of your Alertmanager pods + ports: + - protocol: TCP + port: 9093 # The port on which Alertmanager is running + targetPort: 9093 # The port to forward traffic to on the selected pods diff --git a/chart/demo/templates/alertmanager.yaml b/chart/demo/templates/alertmanager.yaml new file mode 100644 index 0000000..587e266 --- /dev/null +++ b/chart/demo/templates/alertmanager.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alertmanager-kubecop + namespace: kubescape + labels: + app: alertmanager-kubecop +spec: + replicas: 1 + selector: + matchLabels: + app: alertmanager-kubecop + template: + metadata: + labels: + app: alertmanager-kubecop + spec: + containers: + - name: alertmanager + image: quay.io/prometheus/alertmanager:latest + imagePullPolicy: Always + ports: + - containerPort: 9093 + volumeMounts: # Mount the Alertmanager configuration + - name: alertmanager-config + mountPath: /etc/alertmanager + volumes: + - name: alertmanager-config + configMap: + name: alertmanager-config diff --git a/chart/demo/values.yaml b/chart/demo/values.yaml new file mode 100644 index 0000000..c9854de --- /dev/null +++ b/chart/demo/values.yaml @@ -0,0 +1,13 @@ +# Default values for .. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +kubecop: + alertmanager: + smtp_endpoint: "" + smtp_from: "" + smtp_auth_username: "" + smtp_auth_password: "" + smtp_require_tls: false + email_to: "" + diff --git a/chart/kubecop/templates/deamonset.yaml b/chart/kubecop/templates/deamonset.yaml index d003982..93eace1 100644 --- a/chart/kubecop/templates/deamonset.yaml +++ b/chart/kubecop/templates/deamonset.yaml @@ -63,8 +63,8 @@ spec: - name: HOST_ROOT value: "/host" {{- if .Values.kubecop.alertmanager.enabled }} - - name: ALERTMANAGER_URL - value: {{ .Values.kubecop.alertmanager.endpoint }} + - name: ALERTMANAGER_URLS + value: {{ .Values.kubecop.alertmanager.endpoints }} {{- end }} {{- if .Values.kubecop.syslog.enabled }} - name: SYSLOG_HOST diff --git a/chart/kubecop/values.yaml b/chart/kubecop/values.yaml index 7bb63ad..1817fb6 100644 --- a/chart/kubecop/values.yaml +++ b/chart/kubecop/values.yaml @@ -18,7 +18,7 @@ kubecop: finalizationJitter: 120s alertmanager: enabled: false - endpoint: "localhost:9093" + endpoints: "localhost:9093" syslog: enabled: false endpoint: "localhost:514" diff --git a/pkg/exporters/README.md b/pkg/exporters/README.md index 47404d3..baa3ad3 100644 --- a/pkg/exporters/README.md +++ b/pkg/exporters/README.md @@ -10,8 +10,9 @@ The following exporters are available: ### Alertmanager The Alertmanager exporter is used to send alerts to the Alertmanager. The Alertmanager will then send the alerts to the configured receivers. +This exporter supports multiple Alertmanagers. The alerts will be sent to all configured Alertmanagers. To enable the Alertmanager exporter, set the following environment variables: -- `ALERTMANAGER_URL`: The URL of the Alertmanager. Example: `localhost:9093` +- `ALERTMANAGER_URLS`: The URLs of the Alertmanagers. Example: `localhost:9093` or `localhost:9093,localhost:9094` ### STD OUT The STD OUT exporter is used to print the alerts to the standard output. This exporter is enabled by default. diff --git a/pkg/exporters/alert_manager.go b/pkg/exporters/alert_manager.go index 172a263..33274d8 100644 --- a/pkg/exporters/alert_manager.go +++ b/pkg/exporters/alert_manager.go @@ -25,12 +25,6 @@ type AlertManagerExporter struct { } func InitAlertManagerExporter(alertmanagerURL string) *AlertManagerExporter { - if alertmanagerURL == "" { - alertmanagerURL = os.Getenv("ALERTMANAGER_URL") - if alertmanagerURL == "" { - return nil - } - } // Create a new Alertmanager client cfg := client.DefaultTransportConfig().WithHost(alertmanagerURL) amClient := client.NewHTTPClientWithConfig(nil, cfg) diff --git a/pkg/exporters/alert_manager_test.go b/pkg/exporters/alert_manager_test.go index f57efb3..95ad894 100644 --- a/pkg/exporters/alert_manager_test.go +++ b/pkg/exporters/alert_manager_test.go @@ -28,7 +28,6 @@ func TestSendAlert(t *testing.T) { recievedData <- bodyData })) defer server.Close() - // os.Setenv("ALERTMANAGER_URL", "localhost:9093") // Create a new Alertmanager exporter exporter := InitAlertManagerExporter(strings.Replace(server.URL, "http://", "", 1)) diff --git a/pkg/exporters/exporters_bus.go b/pkg/exporters/exporters_bus.go index 129b9b3..7934aef 100644 --- a/pkg/exporters/exporters_bus.go +++ b/pkg/exporters/exporters_bus.go @@ -2,22 +2,29 @@ package exporters import ( "log" + "os" + "strings" "github.com/armosec/kubecop/pkg/engine/rule" "github.com/armosec/kubecop/pkg/scan" ) type ExportersConfig struct { - StdoutExporter *bool `yaml:"stdoutExporter"` - AlertManagerExporterURL string `yaml:"alertManagerExporterURL"` - SyslogExporter string `yaml:"syslogExporterURL"` - CsvRuleExporterPath string `yaml:"csvRuleExporterPath"` - CsvMalwareExporterPath string `yaml:"csvMalwareExporterPath"` + StdoutExporter *bool `yaml:"stdoutExporter"` + AlertManagerExporterUrls string `yaml:"alertManagerExporterUrls"` + SyslogExporter string `yaml:"syslogExporterURL"` + CsvRuleExporterPath string `yaml:"CsvRuleExporterPath"` + CsvMalwareExporterPath string `yaml:"CsvMalwareExporterPath"` } // This file will contain the single point of contact for all exporters, // it will be used by the engine to send alerts to all exporters. +const ( + // AlertManagerURLs separator delimiter. + AlertManagerSepartorDelimiter = "," +) + var ( // Exporters is a list of all exporters. exporters []Exporter @@ -25,9 +32,12 @@ var ( // InitExporters initializes all exporters. func InitExporters(exportersConfig ExportersConfig) { - alertMan := InitAlertManagerExporter(exportersConfig.AlertManagerExporterURL) - if alertMan != nil { - exporters = append(exporters, alertMan) + alertManagerUrls := parseAlertManagerUrls(exportersConfig.AlertManagerExporterUrls) + for _, url := range alertManagerUrls { + alertMan := InitAlertManagerExporter(url) + if alertMan != nil { + exporters = append(exporters, alertMan) + } } stdoutExp := InitStdoutExporter(exportersConfig.StdoutExporter) if stdoutExp != nil { @@ -48,6 +58,20 @@ func InitExporters(exportersConfig ExportersConfig) { log.Print("exporters initialized") } +// ParseAlertManagerUrls parses the alert manager urls from the given string. +func parseAlertManagerUrls(urls string) []string { + if urls == "" { + urls = os.Getenv("ALERTMANAGER_URLS") + if urls == "" { + return nil + } + + return strings.Split(urls, AlertManagerSepartorDelimiter) + + } + return strings.Split(urls, AlertManagerSepartorDelimiter) +} + func SendRuleAlert(failedRule rule.RuleFailure) { for _, exporter := range exporters { exporter.SendRuleAlert(failedRule) diff --git a/resources/latest/kubecop-values.yaml b/resources/latest/kubecop-values.yaml index d7707cc..45c38f0 100644 --- a/resources/latest/kubecop-values.yaml +++ b/resources/latest/kubecop-values.yaml @@ -5,4 +5,4 @@ image: kubecop: alertmanager: enabled: true - endpoint: "alertmanager-operated.monitoring.svc.cluster.local:9093" \ No newline at end of file + endpoints: "alertmanager-operated.monitoring.svc.cluster.local:9093" \ No newline at end of file diff --git a/resources/system-tests/kubecop-values.yaml b/resources/system-tests/kubecop-values.yaml index bca2ae3..8edb4bc 100644 --- a/resources/system-tests/kubecop-values.yaml +++ b/resources/system-tests/kubecop-values.yaml @@ -9,4 +9,4 @@ kubecop: finalizationJitter: 1s alertmanager: enabled: true - endpoint: "alertmanager-operated.monitoring.svc.cluster.local:9093" \ No newline at end of file + endpoints: "alertmanager-operated.monitoring.svc.cluster.local:9093" \ No newline at end of file