From c9407b9a70f4117a66e6f09846c78beec1132349 Mon Sep 17 00:00:00 2001 From: Magyari Sandor Szilard Date: Wed, 8 Sep 2021 14:49:23 +0200 Subject: [PATCH 1/2] remove logging & monitoring integrated services --- cmd/pipeline/BUILD.plz | 6 - cmd/pipeline/capabilities.go | 6 - cmd/pipeline/main.go | 25 - cmd/worker/BUILD.plz | 6 - cmd/worker/main.go | 24 - config/config.yaml.dist | 45 - internal/cmd/BUILD.plz | 4 - internal/cmd/config.go | 52 -- .../services/logging/BUILD.plz | 73 -- .../services/logging/common.go | 68 -- .../services/logging/common_test.go | 204 ----- .../services/logging/config.go | 105 --- .../services/logging/error.go | 25 - .../services/logging/kubernetes.go | 39 - .../services/logging/manager.go | 189 ---- .../services/logging/manager_test.go | 291 ------ .../services/logging/operator.go | 499 ----------- .../logging/operator_flow_resource.go | 73 -- .../logging/operator_output_definition.go | 118 --- .../services/logging/operator_test.go | 157 ---- .../services/logging/output_definition.go | 90 -- .../logging/output_definition_azure.go | 62 -- .../logging/output_definition_base.go | 38 - .../logging/output_definition_bucket.go | 71 -- .../services/logging/output_definition_gcs.go | 54 -- .../logging/output_definition_loki.go | 45 - .../services/logging/output_definition_s3.go | 66 -- .../logging/output_definition_secret.go | 52 -- .../logging/output_definition_secret_azure.go | 54 -- .../logging/output_definition_secret_gcs.go | 45 - .../logging/output_definition_secret_s3.go | 36 - .../services/logging/spec.go | 164 ---- .../services/logging/values.go | 37 - .../services/monitoring/BUILD.plz | 74 -- .../services/monitoring/common.go | 77 -- .../services/monitoring/common_test.go | 197 ----- .../services/monitoring/config.go | 137 --- .../services/monitoring/error.go | 41 - .../services/monitoring/kubernetes.go | 29 - .../services/monitoring/manager.go | 155 ---- .../services/monitoring/manager_test.go | 382 -------- .../services/monitoring/migrator.go | 81 -- .../services/monitoring/migrator_test.go | 104 --- .../services/monitoring/operator.go | 826 ------------------ .../services/monitoring/operator_secret.go | 151 ---- .../services/monitoring/operator_test.go | 174 ---- .../services/monitoring/output.go | 149 ---- .../monitoring/output_alertmanager.go | 53 -- .../services/monitoring/output_grafana.go | 53 -- .../services/monitoring/output_prometheus.go | 53 -- .../services/monitoring/output_pushgateway.go | 51 -- .../monitoring/secret_alertmanager.go | 25 - .../services/monitoring/secret_base.go | 19 - .../services/monitoring/secret_prometheus.go | 25 - .../services/monitoring/spec.go | 281 ------ .../services/monitoring/values.go | 165 ---- 56 files changed, 6125 deletions(-) delete mode 100644 internal/integratedservices/services/logging/BUILD.plz delete mode 100644 internal/integratedservices/services/logging/common.go delete mode 100644 internal/integratedservices/services/logging/common_test.go delete mode 100644 internal/integratedservices/services/logging/config.go delete mode 100644 internal/integratedservices/services/logging/error.go delete mode 100644 internal/integratedservices/services/logging/kubernetes.go delete mode 100644 internal/integratedservices/services/logging/manager.go delete mode 100644 internal/integratedservices/services/logging/manager_test.go delete mode 100644 internal/integratedservices/services/logging/operator.go delete mode 100644 internal/integratedservices/services/logging/operator_flow_resource.go delete mode 100644 internal/integratedservices/services/logging/operator_output_definition.go delete mode 100644 internal/integratedservices/services/logging/operator_test.go delete mode 100644 internal/integratedservices/services/logging/output_definition.go delete mode 100644 internal/integratedservices/services/logging/output_definition_azure.go delete mode 100644 internal/integratedservices/services/logging/output_definition_base.go delete mode 100644 internal/integratedservices/services/logging/output_definition_bucket.go delete mode 100644 internal/integratedservices/services/logging/output_definition_gcs.go delete mode 100644 internal/integratedservices/services/logging/output_definition_loki.go delete mode 100644 internal/integratedservices/services/logging/output_definition_s3.go delete mode 100644 internal/integratedservices/services/logging/output_definition_secret.go delete mode 100644 internal/integratedservices/services/logging/output_definition_secret_azure.go delete mode 100644 internal/integratedservices/services/logging/output_definition_secret_gcs.go delete mode 100644 internal/integratedservices/services/logging/output_definition_secret_s3.go delete mode 100644 internal/integratedservices/services/logging/spec.go delete mode 100644 internal/integratedservices/services/logging/values.go delete mode 100644 internal/integratedservices/services/monitoring/BUILD.plz delete mode 100644 internal/integratedservices/services/monitoring/common.go delete mode 100644 internal/integratedservices/services/monitoring/common_test.go delete mode 100644 internal/integratedservices/services/monitoring/config.go delete mode 100644 internal/integratedservices/services/monitoring/error.go delete mode 100644 internal/integratedservices/services/monitoring/kubernetes.go delete mode 100644 internal/integratedservices/services/monitoring/manager.go delete mode 100644 internal/integratedservices/services/monitoring/manager_test.go delete mode 100644 internal/integratedservices/services/monitoring/migrator.go delete mode 100644 internal/integratedservices/services/monitoring/migrator_test.go delete mode 100644 internal/integratedservices/services/monitoring/operator.go delete mode 100644 internal/integratedservices/services/monitoring/operator_secret.go delete mode 100644 internal/integratedservices/services/monitoring/operator_test.go delete mode 100644 internal/integratedservices/services/monitoring/output.go delete mode 100644 internal/integratedservices/services/monitoring/output_alertmanager.go delete mode 100644 internal/integratedservices/services/monitoring/output_grafana.go delete mode 100644 internal/integratedservices/services/monitoring/output_prometheus.go delete mode 100644 internal/integratedservices/services/monitoring/output_pushgateway.go delete mode 100644 internal/integratedservices/services/monitoring/secret_alertmanager.go delete mode 100644 internal/integratedservices/services/monitoring/secret_base.go delete mode 100644 internal/integratedservices/services/monitoring/secret_prometheus.go delete mode 100644 internal/integratedservices/services/monitoring/spec.go delete mode 100644 internal/integratedservices/services/monitoring/values.go diff --git a/cmd/pipeline/BUILD.plz b/cmd/pipeline/BUILD.plz index 4df8c88aa5..4c223a6b92 100644 --- a/cmd/pipeline/BUILD.plz +++ b/cmd/pipeline/BUILD.plz @@ -42,7 +42,6 @@ go_binary( "//internal/cluster/distribution/eks/eksprovider/driver", "//internal/cluster/distribution/pke", "//internal/cluster/distribution/pke/pkeaws/pkeawsadapter", - "//internal/cluster/endpoints", "//internal/cluster/infrastructure/aws/awsworkflow", "//internal/cluster/metrics/adapters/prometheus", "//internal/clustergroup", @@ -68,8 +67,6 @@ go_binary( "//internal/integratedservices/services/dns/dnsadapter", "//internal/integratedservices/services/expiry", "//internal/integratedservices/services/ingress", - "//internal/integratedservices/services/logging", - "//internal/integratedservices/services/monitoring", "//internal/integratedservices/services/securityscan", "//internal/integratedservices/services/securityscan/securityscanadapter", "//internal/integratedservices/services/vault", @@ -210,7 +207,6 @@ go_test( "//internal/cluster/distribution/eks/eksprovider/driver", "//internal/cluster/distribution/pke", "//internal/cluster/distribution/pke/pkeaws/pkeawsadapter", - "//internal/cluster/endpoints", "//internal/cluster/infrastructure/aws/awsworkflow", "//internal/cluster/metrics/adapters/prometheus", "//internal/clustergroup", @@ -236,8 +232,6 @@ go_test( "//internal/integratedservices/services/dns/dnsadapter", "//internal/integratedservices/services/expiry", "//internal/integratedservices/services/ingress", - "//internal/integratedservices/services/logging", - "//internal/integratedservices/services/monitoring", "//internal/integratedservices/services/securityscan", "//internal/integratedservices/services/securityscan/securityscanadapter", "//internal/integratedservices/services/vault", diff --git a/cmd/pipeline/capabilities.go b/cmd/pipeline/capabilities.go index 16c1d16a08..089397fbc4 100644 --- a/cmd/pipeline/capabilities.go +++ b/cmd/pipeline/capabilities.go @@ -26,12 +26,6 @@ func mapCapabilities(config configuration) cap.Capabilities { "enabled": config.Cluster.Vault.Enabled, "managed": config.Cluster.Vault.Managed.Enabled, }, - "monitoring": cap.Cap{ - "enabled": config.Cluster.Monitoring.Enabled, - }, - "logging": cap.Cap{ - "enabled": config.Cluster.Logging.Enabled, - }, "dns": cap.Cap{ "enabled": config.Cluster.DNS.Enabled, "baseDomain": config.Cluster.DNS.BaseDomain, diff --git a/cmd/pipeline/main.go b/cmd/pipeline/main.go index e5581ba46f..b34259ea04 100644 --- a/cmd/pipeline/main.go +++ b/cmd/pipeline/main.go @@ -88,7 +88,6 @@ import ( eksDriver "github.com/banzaicloud/pipeline/internal/cluster/distribution/eks/eksprovider/driver" pkeDistribution "github.com/banzaicloud/pipeline/internal/cluster/distribution/pke" "github.com/banzaicloud/pipeline/internal/cluster/distribution/pke/pkeaws/pkeawsadapter" - "github.com/banzaicloud/pipeline/internal/cluster/endpoints" "github.com/banzaicloud/pipeline/internal/cluster/infrastructure/aws/awsworkflow" prometheusMetrics "github.com/banzaicloud/pipeline/internal/cluster/metrics/adapters/prometheus" "github.com/banzaicloud/pipeline/internal/clustergroup" @@ -113,8 +112,6 @@ import ( "github.com/banzaicloud/pipeline/internal/integratedservices/services/dns/dnsadapter" "github.com/banzaicloud/pipeline/internal/integratedservices/services/expiry" "github.com/banzaicloud/pipeline/internal/integratedservices/services/ingress" - integratedServiceLogging "github.com/banzaicloud/pipeline/internal/integratedservices/services/logging" - featureMonitoring "github.com/banzaicloud/pipeline/internal/integratedservices/services/monitoring" "github.com/banzaicloud/pipeline/internal/integratedservices/services/securityscan" "github.com/banzaicloud/pipeline/internal/integratedservices/services/securityscan/securityscanadapter" integratedServiceVault "github.com/banzaicloud/pipeline/internal/integratedservices/services/vault" @@ -881,7 +878,6 @@ func main() { // common setup (for both legacy and V2 IS) clusterGetter := integratedserviceadapter.MakeClusterGetter(clusterManager) clusterPropertyGetter := dnsadapter.NewClusterPropertyGetter(clusterManager) - endpointManager := endpoints.NewEndpointManager(commonLogger) integratedServiceManagers := make([]integratedservices.IntegratedServiceManager, 0) // integrated service service V2 setup @@ -924,27 +920,6 @@ func main() { integratedServiceManagers = append(integratedServiceManagers, integratedServiceVault.MakeIntegratedServiceManager(clusterGetter, commonSecretStore, config.Cluster.Vault.Config, commonLogger)) } - if config.Cluster.Monitoring.Enabled { - integratedServiceManagers = append(integratedServiceManagers, featureMonitoring.MakeIntegratedServiceManager( - clusterGetter, - commonSecretStore, - endpointManager, - unifiedHelmReleaser, - config.Cluster.Monitoring.Config, - commonLogger, - )) - } - - if config.Cluster.Logging.Enabled { - integratedServiceManagers = append(integratedServiceManagers, integratedServiceLogging.MakeIntegratedServiceManager( - clusterGetter, - commonSecretStore, - endpointManager, - config.Cluster.Logging.Config, - commonLogger, - )) - } - if config.Cluster.SecurityScan.Enabled { customAnchoreConfigProvider := securityscan.NewCustomAnchoreConfigProvider( featureRepository, diff --git a/cmd/worker/BUILD.plz b/cmd/worker/BUILD.plz index 86d1c696c0..fa700ceec9 100644 --- a/cmd/worker/BUILD.plz +++ b/cmd/worker/BUILD.plz @@ -31,7 +31,6 @@ go_binary( "//internal/cluster/distribution/pke/pkeaws/pkeawsprovider/workflow", "//internal/cluster/distribution/pke/pkeaws/pkeawsworkflow", "//internal/cluster/dns", - "//internal/cluster/endpoints", "//internal/cluster/infrastructure/aws/awsworkflow", "//internal/cluster/kubernetes", "//internal/cluster/workflow", @@ -56,8 +55,6 @@ go_binary( "//internal/integratedservices/services/expiry/adapter/workflow", "//internal/integratedservices/services/ingress", "//internal/integratedservices/services/ingress/ingressadapter", - "//internal/integratedservices/services/logging", - "//internal/integratedservices/services/monitoring", "//internal/integratedservices/services/securityscan", "//internal/integratedservices/services/securityscan/securityscanadapter", "//internal/integratedservices/services/vault", @@ -155,7 +152,6 @@ go_test( "//internal/cluster/distribution/pke/pkeaws/pkeawsprovider/workflow", "//internal/cluster/distribution/pke/pkeaws/pkeawsworkflow", "//internal/cluster/dns", - "//internal/cluster/endpoints", "//internal/cluster/infrastructure/aws/awsworkflow", "//internal/cluster/kubernetes", "//internal/cluster/workflow", @@ -180,8 +176,6 @@ go_test( "//internal/integratedservices/services/expiry/adapter/workflow", "//internal/integratedservices/services/ingress", "//internal/integratedservices/services/ingress/ingressadapter", - "//internal/integratedservices/services/logging", - "//internal/integratedservices/services/monitoring", "//internal/integratedservices/services/securityscan", "//internal/integratedservices/services/securityscan/securityscanadapter", "//internal/integratedservices/services/vault", diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 12bcfb3a09..dfeeebe575 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -58,7 +58,6 @@ import ( "github.com/banzaicloud/pipeline/internal/cluster/distribution/pke/pkeaws" "github.com/banzaicloud/pipeline/internal/cluster/distribution/pke/pkeaws/pkeawsadapter" intClusterDNS "github.com/banzaicloud/pipeline/internal/cluster/dns" - "github.com/banzaicloud/pipeline/internal/cluster/endpoints" intClusterK8s "github.com/banzaicloud/pipeline/internal/cluster/kubernetes" intClusterWorkflow "github.com/banzaicloud/pipeline/internal/cluster/workflow" "github.com/banzaicloud/pipeline/internal/clustergroup" @@ -82,8 +81,6 @@ import ( expiryWorkflow "github.com/banzaicloud/pipeline/internal/integratedservices/services/expiry/adapter/workflow" intsvcingress "github.com/banzaicloud/pipeline/internal/integratedservices/services/ingress" intsvcingressadapter "github.com/banzaicloud/pipeline/internal/integratedservices/services/ingress/ingressadapter" - integratedServiceLogging "github.com/banzaicloud/pipeline/internal/integratedservices/services/logging" - integratedServiceMonitoring "github.com/banzaicloud/pipeline/internal/integratedservices/services/monitoring" "github.com/banzaicloud/pipeline/internal/integratedservices/services/securityscan" "github.com/banzaicloud/pipeline/internal/integratedservices/services/securityscan/securityscanadapter" integratedServiceVault "github.com/banzaicloud/pipeline/internal/integratedservices/services/vault" @@ -668,7 +665,6 @@ func main() { clusterGetter := integratedserviceadapter.MakeClusterGetter(clusterManager) clusterService := integratedserviceadapter.NewClusterService(clusterManager) - endpointManager := endpoints.NewEndpointManager(logger) orgDomainService := dnsadapter.NewOrgDomainService( config.Cluster.DNS.BaseDomain, dnsSvc, @@ -736,26 +732,6 @@ func main() { config.Cluster.Vault.Config, logger, ), - integratedServiceMonitoring.MakeIntegratedServiceOperator( - clusterGetter, - clusterService, - unifiedHelmReleaser, - kubernetesService, - config.Cluster.Monitoring.Config, - logger, - commonSecretStore, - integratedServiceMonitoring.Migrate, - ), - integratedServiceLogging.MakeIntegratedServicesOperator( - clusterGetter, - clusterService, - unifiedHelmReleaser, - kubernetesService, - endpointManager, - config.Cluster.Logging.Config, - logger, - commonSecretStore, - ), expiry.NewExpiryServiceOperator(expirerService, services.BindIntegratedServiceSpec, logger), intsvcingress.NewOperator( intsvcingressadapter.NewOperatorClusterStore(clusterStore), diff --git a/config/config.yaml.dist b/config/config.yaml.dist index 4249132fd9..8c5a67329a 100644 --- a/config/config.yaml.dist +++ b/config/config.yaml.dist @@ -184,51 +184,6 @@ dex: # # See https://github.com/banzaicloud/bank-vaults/tree/master/charts/vault-secrets-webhook for details # values: {} # -# monitoring: -# enabled: true -# -# # Inherited from cluster.namespace when empty -# namespace: "" -# -# grafana: -# adminUser: admin -# -# charts: -# operator: -# chart: "prometheus-community/kube-prometheus-stack" -# version: "12.11.3" -# -# # See https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack for details -# values: {} -# -# pushgateway: -# chart: "prometheus-community/prometheus-pushgateway" -# version: "1.5.1" -# -# # See https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-pushgateway for details -# values: {} -# -# logging: -# enabled: true -# -# # Inherited from cluster.namespace when empty -# namespace: "" -# -# charts: -# operator: -# chart: "banzaicloud-stable/logging-operator" -# version: "2.7.2" -# -# # See https://github.com/banzaicloud/logging-operator/tree/master/charts/logging-operator for details -# values: {} -# -# loki: -# chart: "banzaicloud-stable/loki" -# version: "0.17.4" -# -# # See https://github.com/banzaicloud/banzai-charts/tree/master/loki for details -# values: {} -# # dns: # enabled: true # diff --git a/internal/cmd/BUILD.plz b/internal/cmd/BUILD.plz index 77f57bd1e3..0b13baeb41 100644 --- a/internal/cmd/BUILD.plz +++ b/internal/cmd/BUILD.plz @@ -13,8 +13,6 @@ go_library( "//internal/integratedservices/operator", "//internal/integratedservices/services/dns", "//internal/integratedservices/services/ingress", - "//internal/integratedservices/services/logging", - "//internal/integratedservices/services/monitoring", "//internal/integratedservices/services/securityscan", "//internal/integratedservices/services/vault", "//internal/platform/cadence", @@ -43,8 +41,6 @@ go_test( "//internal/integratedservices/operator", "//internal/integratedservices/services/dns", "//internal/integratedservices/services/ingress", - "//internal/integratedservices/services/logging", - "//internal/integratedservices/services/monitoring", "//internal/integratedservices/services/securityscan", "//internal/integratedservices/services/vault", "//internal/platform/cadence", diff --git a/internal/cmd/config.go b/internal/cmd/config.go index 803b126d4e..281e6f2318 100644 --- a/internal/cmd/config.go +++ b/internal/cmd/config.go @@ -27,8 +27,6 @@ import ( "github.com/banzaicloud/pipeline/internal/integratedservices/operator" "github.com/banzaicloud/pipeline/internal/integratedservices/services/dns" "github.com/banzaicloud/pipeline/internal/integratedservices/services/ingress" - "github.com/banzaicloud/pipeline/internal/integratedservices/services/logging" - "github.com/banzaicloud/pipeline/internal/integratedservices/services/monitoring" "github.com/banzaicloud/pipeline/internal/integratedservices/services/securityscan" "github.com/banzaicloud/pipeline/internal/integratedservices/services/vault" "github.com/banzaicloud/pipeline/internal/platform/cadence" @@ -206,10 +204,6 @@ type ClusterConfig struct { // Initial manifest Manifest string - Monitoring ClusterMonitoringConfig - - Logging ClusterLoggingConfig - // Namespace to install Pipeline components to Namespace string @@ -233,8 +227,6 @@ func (c ClusterConfig) Validate() error { errs = errors.Append(errs, c.Labels.Validate()) - errs = errors.Append(errs, c.Logging.Validate()) - if c.Manifest != "" { file, err := os.OpenFile(c.Manifest, os.O_RDONLY, 0666) _ = file.Close() @@ -243,8 +235,6 @@ func (c ClusterConfig) Validate() error { } } - errs = errors.Append(errs, c.Monitoring.Validate()) - if c.Namespace == "" { errs = errors.Append(errs, errors.New("cluster namespace is required")) } @@ -278,14 +268,6 @@ func (c *ClusterConfig) Process() error { c.Labels.Namespace = c.Namespace } - if c.Logging.Namespace == "" { - c.Logging.Namespace = c.Namespace - } - - if c.Monitoring.Namespace == "" { - c.Monitoring.Namespace = c.Namespace - } - if c.SecurityScan.PipelineNamespace == "" { c.SecurityScan.PipelineNamespace = c.Namespace } @@ -389,40 +371,6 @@ func (c ClusterIngressConfig) Validate() error { return errs } -// ClusterLoggingConfig contains cluster logging configuration. -type ClusterLoggingConfig struct { - Enabled bool - - logging.Config `mapstructure:",squash"` -} - -func (c ClusterLoggingConfig) Validate() error { - var errs error - - if c.Enabled { - errs = errors.Append(errs, c.Config.Validate()) - } - - return errs -} - -// ClusterMonitoringConfig contains cluster monitoring configuration. -type ClusterMonitoringConfig struct { - Enabled bool - - monitoring.Config `mapstructure:",squash"` -} - -func (c ClusterMonitoringConfig) Validate() error { - var errs error - - if c.Enabled { - errs = errors.Append(errs, c.Config.Validate()) - } - - return errs -} - // ClusterSecurityScanConfig contains cluster security scan configuration. type ClusterSecurityScanConfig struct { Enabled bool diff --git a/internal/integratedservices/services/logging/BUILD.plz b/internal/integratedservices/services/logging/BUILD.plz deleted file mode 100644 index 9741ff4ce2..0000000000 --- a/internal/integratedservices/services/logging/BUILD.plz +++ /dev/null @@ -1,73 +0,0 @@ -go_library( - name = "logging", - srcs = glob( - ["*.go"], - exclude = ["*_test.go"], - ), - visibility = ["PUBLIC"], - deps = [ - "//internal/cluster/endpoints", - "//internal/common", - "//internal/integratedservices", - "//internal/integratedservices/integratedserviceadapter", - "//internal/integratedservices/services", - "//internal/providers", - "//internal/secret/secrettype", - "//pkg/any", - "//pkg/cluster", - "//pkg/helm", - "//pkg/jsonstructure", - "//pkg/providers/azure", - "//pkg/providers/azure/objectstore", - "//src/auth", - "//src/cluster", - "//src/dns", - "//src/secret", - "//third_party/go:emperror.dev__errors", - "//third_party/go:github.com__banzaicloud__logging-operator__pkg__sdk__api__v1beta1", - "//third_party/go:github.com__banzaicloud__logging-operator__pkg__sdk__model__output", - "//third_party/go:github.com__banzaicloud__operator-tools__pkg__secret", - "//third_party/go:github.com__mitchellh__copystructure", - "//third_party/go:github.com__mitchellh__mapstructure", - "//third_party/go:k8s.io__api__core__v1", - "//third_party/go:k8s.io__apimachinery__pkg__api__errors", - "//third_party/go:k8s.io__apimachinery__pkg__apis__meta__v1", - "//third_party/go:k8s.io__apimachinery__pkg__runtime", - ], -) - -go_test( - name = "test", - srcs = glob(["*.go"]), - deps = [ - "//internal/cluster/endpoints", - "//internal/common", - "//internal/common/commonadapter", - "//internal/integratedservices", - "//internal/integratedservices/integratedserviceadapter", - "//internal/integratedservices/services", - "//internal/providers", - "//internal/secret/secrettype", - "//pkg/any", - "//pkg/cluster", - "//pkg/helm", - "//pkg/jsonstructure", - "//pkg/providers/azure", - "//pkg/providers/azure/objectstore", - "//src/auth", - "//src/cluster", - "//src/dns", - "//src/secret", - "//third_party/go:emperror.dev__errors", - "//third_party/go:github.com__banzaicloud__logging-operator__pkg__sdk__api__v1beta1", - "//third_party/go:github.com__banzaicloud__logging-operator__pkg__sdk__model__output", - "//third_party/go:github.com__banzaicloud__operator-tools__pkg__secret", - "//third_party/go:github.com__mitchellh__copystructure", - "//third_party/go:github.com__mitchellh__mapstructure", - "//third_party/go:github.com__stretchr__testify__assert", - "//third_party/go:k8s.io__api__core__v1", - "//third_party/go:k8s.io__apimachinery__pkg__api__errors", - "//third_party/go:k8s.io__apimachinery__pkg__apis__meta__v1", - "//third_party/go:k8s.io__apimachinery__pkg__runtime", - ], -) diff --git a/internal/integratedservices/services/logging/common.go b/internal/integratedservices/services/logging/common.go deleted file mode 100644 index fcae36f9eb..0000000000 --- a/internal/integratedservices/services/logging/common.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "fmt" -) - -const ( - integratedServiceName = "logging" - - providerAmazonS3 = "s3" - providerGoogleGCS = "gcs" - providerAzure = "azure" - providerLoki = "loki" - - tlsSecretName = "logging-tls-secret" - loggingOperatorReleaseName = "logging-operator" - lokiReleaseName = "loki" - lokiServiceName = "loki" - releaseSecretTag = "release:logging" - integratedServiceSecretTag = "feature:logging" - lokiSecretTag = "app:loki" - generatedSecretUsername = "admin" - fluentSharedSecretName = "logging-operator-fluent-shared-secret" - - outputDefinitionSecretKeyS3AccessKeyID = "awsAccessKeyId" - outputDefinitionSecretKeyS3AccessKey = "awsSecretAccessKey" - outputDefinitionSecretKeyGCS = "credentials.json" - outputDefinitionSecretKeyAzureStorageAccount = "azureStorageAccount" - outputDefinitionSecretKeyAzureStorageAccess = "azureStorageAccessKey" - - lokiOutputDefinitionName = "loki-output" - flowResourceName = "banzai-logging-flow" - resourceLabelKey = "banzaicloud.io/service" - loggingResourceName = "banzai-logging" -) - -func getLokiSecretName(clusterID uint) string { - return fmt.Sprintf("cluster-%d-loki", clusterID) -} - -func generateClusterUIDSecretTag(clusterUID string) string { - return fmt.Sprintf("clusterUID:%s", clusterUID) -} - -func generateClusterNameSecretTag(clusterName string) string { - return fmt.Sprintf("cluster:%s", clusterName) -} - -func generateAnnotations(secretName string) map[string]interface{} { - return map[string]interface{}{ - "traefik.ingress.kubernetes.io/auth-type": "basic", - "traefik.ingress.kubernetes.io/auth-secret": secretName, - } -} diff --git a/internal/integratedservices/services/logging/common_test.go b/internal/integratedservices/services/logging/common_test.go deleted file mode 100644 index 5d01f5d0b0..0000000000 --- a/internal/integratedservices/services/logging/common_test.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - - "emperror.dev/errors" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/pkg/helm" - "github.com/banzaicloud/pipeline/src/secret" -) - -type obj = map[string]interface{} - -const ( - lokiPath = "/loki" - lokiURL = "http://logging.io/loki" - lokiServiceUrl = "dummyServiceUrl:9090" - lokiSecretID = "lokiSecretID" -) - -type dummyClusterGetter struct { - Clusters map[uint]dummyCluster -} - -func (d dummyClusterGetter) GetClusterByIDOnly(ctx context.Context, clusterID uint) (integratedserviceadapter.Cluster, error) { - return d.Clusters[clusterID], nil -} - -func (d dummyClusterGetter) GetClusterStatus(ctx context.Context, clusterID uint) (string, error) { - if c, ok := d.Clusters[clusterID]; ok { - return c.Status, nil - } - return "", errors.New("cluster not found") -} - -type dummyCluster struct { - K8sConfig []byte - Name string - OrgID uint - ID uint - UID string - NodePools map[string]bool - Rbac bool - Status string -} - -func (d dummyCluster) GetK8sConfig() ([]byte, error) { - return d.K8sConfig, nil -} - -func (d dummyCluster) GetName() string { - return d.Name -} - -func (d dummyCluster) GetOrganizationId() uint { - return d.OrgID -} - -func (d dummyCluster) GetUID() string { - return d.UID -} - -func (d dummyCluster) GetID() uint { - return d.ID -} - -func (d dummyCluster) NodePoolExists(nodePoolName string) bool { - return d.NodePools[nodePoolName] -} - -func (d dummyCluster) RbacEnabled() bool { - return d.Rbac -} - -type dummyOrganizationalSecretStore struct { - Secrets map[uint]map[string]*secret.SecretItemResponse -} - -func (d dummyOrganizationalSecretStore) Get(orgID uint, secretID string) (*secret.SecretItemResponse, error) { - if orgSecrets, ok := d.Secrets[orgID]; ok { - if sir, ok := orgSecrets[secretID]; ok { - return sir, nil - } - } - return nil, secret.ErrSecretNotExists -} - -func (d dummyOrganizationalSecretStore) Store(organizationID uint, request *secret.CreateSecretRequest) (string, error) { - return "customsecretid", nil -} - -func (d dummyOrganizationalSecretStore) GetByName(organizationID uint, name string) (*secret.SecretItemResponse, error) { - if orgSecrets, ok := d.Secrets[organizationID]; ok { - for n, sir := range orgSecrets { - if n == name { - return sir, nil - } - } - } - return nil, secret.ErrSecretNotExists -} - -func (d dummyOrganizationalSecretStore) Delete(organizationID uint, secretID string) error { - return nil -} - -type dummyEndpointService struct{} - -func (dummyEndpointService) List(ctx context.Context, kubeConfig []byte, releaseName string) ([]*helm.EndpointItem, error) { - return []*helm.EndpointItem{ - { - Name: "ingress-traefik", - EndPointURLs: []*helm.EndPointURLs{ - { - Path: lokiPath, - URL: lokiURL, - ReleaseName: releaseName, - }, - }, - }, - }, nil -} - -func (dummyEndpointService) GetServiceURL(ctx context.Context, kubeConfig []byte, serviceName string, namespace string) (string, error) { - return lokiServiceUrl, nil -} - -type dummyHelmService struct{} - -func (d dummyHelmService) ApplyDeployment( - ctx context.Context, - clusterID uint, - namespace string, - deploymentName string, - releaseName string, - values []byte, - chartVersion string, -) error { - return nil -} - -func (d dummyHelmService) ApplyDeploymentSkipCRDs( - ctx context.Context, - clusterID uint, - namespace string, - deploymentName string, - releaseName string, - values []byte, - chartVersion string, -) error { - return nil -} - -func (d dummyHelmService) DeleteDeployment(ctx context.Context, clusterID uint, releaseName, namespace string) error { - return nil -} - -func (d dummyHelmService) GetDeployment(ctx context.Context, clusterID uint, releaseName, namespace string) (*helm.GetDeploymentResponse, error) { - return &helm.GetDeploymentResponse{ - ReleaseName: releaseName, - }, nil -} - -type dummyKubernetesService struct { -} - -// DeleteObject deletes an Object from a specific cluster. -func (s *dummyKubernetesService) DeleteObject(ctx context.Context, clusterID uint, o runtime.Object) error { - return nil -} - -// EnsureObject makes sure that a given Object is on the cluster and returns it. -func (s *dummyKubernetesService) EnsureObject(ctx context.Context, clusterID uint, o runtime.Object) error { - return nil -} - -func (s *dummyKubernetesService) List(ctx context.Context, clusterID uint, labels map[string]string, o runtime.Object) error { - return nil -} - -func (s *dummyKubernetesService) Update(ctx context.Context, clusterID uint, o runtime.Object) error { - return nil -} - -func (s *dummyKubernetesService) GetObject(ctx context.Context, clusterID uint, objRef corev1.ObjectReference, obj runtime.Object) error { - return nil -} diff --git a/internal/integratedservices/services/logging/config.go b/internal/integratedservices/services/logging/config.go deleted file mode 100644 index db3d32d6b4..0000000000 --- a/internal/integratedservices/services/logging/config.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "emperror.dev/errors" -) - -// Config contains configuration for the logging integrated service. -type Config struct { - Namespace string - Charts ChartsConfig - Images ImagesConfig -} - -func (c Config) Validate() error { - if c.Namespace == "" { - return errors.New("logging namespace is required") - } - - if err := c.Charts.Operator.Validate(); err != nil { - return errors.WrapIf(err, "error during validation logging operator config") - } - - if err := c.Charts.Loki.Validate(); err != nil { - return errors.WrapIf(err, "error during validation loki chart config") - } - - if err := c.Images.Operator.Validate(); err != nil { - return errors.WrapIf(err, "error during validation operator image config") - } - - if err := c.Images.Loki.Validate(); err != nil { - return errors.WrapIf(err, "error during validation loki image config") - } - - if err := c.Images.Fluentbit.Validate(); err != nil { - return errors.WrapIf(err, "error during validation fluentbit image config") - } - - if err := c.Images.Fluentd.Validate(); err != nil { - return errors.WrapIf(err, "error during validation fluentd image config") - } - - return nil -} - -type ChartsConfig struct { - Operator ChartConfig - Loki ChartConfig -} - -type ChartConfig struct { - Chart string - Version string - Values map[string]interface{} -} - -func (c ChartConfig) Validate() error { - if c.Chart == "" { - return errors.New("chart is required") - } - - if c.Version == "" { - return errors.New("chart version is required") - } - - return nil -} - -type ImagesConfig struct { - Operator ImageConfig - Loki ImageConfig - Fluentbit ImageConfig - Fluentd ImageConfig -} - -type ImageConfig struct { - Repository string - Tag string -} - -func (c ImageConfig) Validate() error { - if c.Repository == "" { - return errors.New("repository is required") - } - - if c.Tag == "" { - return errors.New("tag is required") - } - - return nil -} diff --git a/internal/integratedservices/services/logging/error.go b/internal/integratedservices/services/logging/error.go deleted file mode 100644 index 2c05586c4c..0000000000 --- a/internal/integratedservices/services/logging/error.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import "fmt" - -type requiredFieldError struct { - name string -} - -func (e requiredFieldError) Error() string { - return fmt.Sprintf("%q cannot be empty", e.name) -} diff --git a/internal/integratedservices/services/logging/kubernetes.go b/internal/integratedservices/services/logging/kubernetes.go deleted file mode 100644 index 51088a5aae..0000000000 --- a/internal/integratedservices/services/logging/kubernetes.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -type KubernetesService interface { - // EnsureObject makes sure that a given Object is on the cluster and returns it. - EnsureObject(ctx context.Context, clusterID uint, o runtime.Object) error - - // Update updates a given Object on the cluster and returns it. - Update(ctx context.Context, clusterID uint, o runtime.Object) error - - // DeleteObject deletes an Object from a specific cluster. - DeleteObject(ctx context.Context, clusterID uint, o runtime.Object) error - - // GetObject gets an Object from a specific cluster. - GetObject(ctx context.Context, clusterID uint, objRef corev1.ObjectReference, obj runtime.Object) error - - // List lists Objects on specific cluster. - List(ctx context.Context, clusterID uint, labels map[string]string, o runtime.Object) error -} diff --git a/internal/integratedservices/services/logging/manager.go b/internal/integratedservices/services/logging/manager.go deleted file mode 100644 index 702741bf86..0000000000 --- a/internal/integratedservices/services/logging/manager.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - "fmt" - - "emperror.dev/errors" - - "github.com/banzaicloud/pipeline/internal/cluster/endpoints" - "github.com/banzaicloud/pipeline/internal/common" - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - pkgHelm "github.com/banzaicloud/pipeline/pkg/helm" -) - -// IntegratedServiceManager implements the Logging integrated service manager -type IntegratedServicesManager struct { - integratedservices.PassthroughIntegratedServiceSpecPreparer - - clusterGetter integratedserviceadapter.ClusterGetter - secretStore services.SecretStore - endpointsService endpoints.EndpointService - config Config - logger common.Logger -} - -func MakeIntegratedServiceManager( - clusterGetter integratedserviceadapter.ClusterGetter, - secretStore services.SecretStore, - endpointsService endpoints.EndpointService, - config Config, - logger common.Logger, -) IntegratedServicesManager { - return IntegratedServicesManager{ - clusterGetter: clusterGetter, - secretStore: secretStore, - endpointsService: endpointsService, - config: config, - logger: logger, - } -} - -// Name returns the integrated service' name -func (IntegratedServicesManager) Name() string { - return integratedServiceName -} - -func (m IntegratedServicesManager) GetOutput(ctx context.Context, clusterID uint, spec integratedservices.IntegratedServiceSpec) (integratedservices.IntegratedServiceOutput, error) { - boundSpec, err := bindIntegratedServiceSpec(spec) - if err != nil { - return nil, integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: err.Error(), - } - } - - cluster, err := m.clusterGetter.GetClusterByIDOnly(ctx, clusterID) - if err != nil { - return nil, errors.WrapIf(err, "failed to get cluster") - } - - kubeConfig, err := cluster.GetK8sConfig() - if err != nil { - return nil, errors.WrapIf(err, "failed to get K8S config") - } - - endpoints, err := m.endpointsService.List(ctx, kubeConfig, lokiReleaseName) - if err != nil { - m.logger.Warn(fmt.Sprintf("failed to list endpoints: %s", err.Error())) - } - - return integratedservices.IntegratedServiceOutput{ - "logging": map[string]interface{}{ - "operatorVersion": m.config.Charts.Operator.Version, - "fluentdVersion": m.config.Images.Fluentd.Tag, - "fluentbitVersion": m.config.Images.Fluentbit.Tag, - }, - "loki": m.getLokiOutput(ctx, boundSpec, endpoints, kubeConfig, clusterID), - }, nil -} - -func (m IntegratedServicesManager) getLokiOutput( - ctx context.Context, - spec integratedServiceSpec, - endpoints []*pkgHelm.EndpointItem, - kubeConfig []byte, - clusterID uint, -) map[string]interface{} { - if spec.Loki.Enabled { - serviceUrl, err := getLokiServiceURL(ctx, spec.Loki, kubeConfig, m.endpointsService, m.config.Namespace) - if err != nil { - m.logger.Warn("failed to get Loki service url") - } - return map[string]interface{}{ - "url": getLokiEndpoint(endpoints, spec.Loki), - "version": m.config.Images.Loki.Tag, - "serviceUrl": serviceUrl, - "secretId": m.getLokiSecretID(ctx, spec.Loki, clusterID), - } - } - return nil -} - -func getLokiEndpoint(endpoints []*pkgHelm.EndpointItem, spec lokiSpec) string { - if spec.Ingress.Enabled && endpoints != nil { - return getEndpointUrl(endpoints, spec.Ingress.Path, lokiReleaseName) - } - return "" -} - -func getEndpointUrl(endpoints []*pkgHelm.EndpointItem, path, releaseName string) string { - for _, ep := range endpoints { - for _, url := range ep.EndPointURLs { - if url.Path == path && url.ReleaseName == releaseName { - return url.URL - } - } - } - return "" -} - -func getLokiServiceURL( - ctx context.Context, - spec lokiSpec, - k8sConfig []byte, - service endpoints.EndpointService, - pipelineSystemNamespace string, -) (string, error) { - if spec.Enabled { - url, err := service.GetServiceURL(ctx, k8sConfig, lokiServiceName, pipelineSystemNamespace) - if err != nil { - return "", errors.WrapIf(err, "failed to get service") - } - return url, nil - } - - return "", nil -} - -func (m IntegratedServicesManager) getLokiSecretID(ctx context.Context, spec lokiSpec, clusterID uint) string { - if spec.Enabled && spec.Ingress.Enabled { - generatedSecretName := getLokiSecretName(clusterID) - if spec.Ingress.SecretID == "" && generatedSecretName != "" { - secretID, err := m.secretStore.GetIDByName(ctx, generatedSecretName) - if err != nil { - m.logger.Warn("failed to get generated Loki secret") - return "" - } - - return secretID - } - } - return "" -} - -func (IntegratedServicesManager) ValidateSpec(ctx context.Context, spec integratedservices.IntegratedServiceSpec) error { - vaultSpec, err := bindIntegratedServiceSpec(spec) - if err != nil { - return err - } - - if err := vaultSpec.Validate(); err != nil { - return integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: err.Error(), - } - } - - return nil -} - -func (IntegratedServicesManager) PrepareSpec(ctx context.Context, clusterID uint, spec integratedservices.IntegratedServiceSpec) (integratedservices.IntegratedServiceSpec, error) { - return spec, nil -} diff --git a/internal/integratedservices/services/logging/manager_test.go b/internal/integratedservices/services/logging/manager_test.go deleted file mode 100644 index 57a02567d1..0000000000 --- a/internal/integratedservices/services/logging/manager_test.go +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - "testing" - - "emperror.dev/errors" - "github.com/stretchr/testify/assert" - - "github.com/banzaicloud/pipeline/internal/common/commonadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - "github.com/banzaicloud/pipeline/src/auth" - "github.com/banzaicloud/pipeline/src/secret" -) - -func TestIntegratedServiceManager_Name(t *testing.T) { - mng := MakeIntegratedServiceManager(nil, nil, nil, Config{}, nil) - - assert.Equal(t, "logging", mng.Name()) -} - -func TestIntegratedServiceManager_GetOutput(t *testing.T) { - orgID := uint(13) - clusterID := uint(42) - clusterName := "the-cluster" - - clusterGetter := dummyClusterGetter{ - Clusters: map[uint]dummyCluster{ - clusterID: { - Name: clusterName, - OrgID: orgID, - ID: clusterID, - }, - }, - } - - orgSecretStore := dummyOrganizationalSecretStore{ - Secrets: map[uint]map[string]*secret.SecretItemResponse{ - orgID: { - lokiSecretID: { - ID: lokiSecretID, - Name: getLokiSecretName(clusterID), - Type: secrettype.HtpasswdSecretType, - Values: map[string]string{secrettype.Username: "admin", secrettype.Password: "pass"}, - Tags: []string{secret.TagBanzaiReadonly}, - Version: 1, - }, - }, - }, - } - - config := Config{ - Charts: ChartsConfig{ - Operator: ChartConfig{ - Version: "1.0.0", - }, - }, - Images: ImagesConfig{ - Operator: ImageConfig{ - Tag: "v2.0.0", - }, - Loki: ImageConfig{ - Tag: "v2.0.0", - }, - Fluentbit: ImageConfig{ - Tag: "v3.0.1", - }, - Fluentd: ImageConfig{ - Tag: "v3.0.2", - }, - }, - } - - secretStore := commonadapter.NewSecretStore(orgSecretStore, commonadapter.OrgIDContextExtractorFunc(auth.GetCurrentOrganizationID)) - endpointService := dummyEndpointService{} - logger := services.NoopLogger{} - mng := MakeIntegratedServiceManager(clusterGetter, secretStore, endpointService, config, logger) - ctx := auth.SetCurrentOrganizationID(context.Background(), orgID) - - spec := obj{ - "loki": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "/loki", - }, - }, - "logging": obj{ - "metrics": true, - "tls": true, - }, - "clusterOutput": obj{ - "enabled": false, - }, - } - - output, err := mng.GetOutput(ctx, clusterID, spec) - assert.NoError(t, err) - - assert.Equal(t, integratedservices.IntegratedServiceOutput{ - "logging": obj{ - "operatorVersion": "1.0.0", - "fluentdVersion": "v3.0.2", - "fluentbitVersion": "v3.0.1", - }, - "loki": obj{ - "secretId": "", - "version": "v2.0.0", - "url": lokiURL, - "serviceUrl": lokiServiceUrl, - }, - }, output) -} - -func TestIntegratedServiceManager_ValidateSpec(t *testing.T) { - mng := MakeIntegratedServiceManager(nil, nil, nil, Config{}, nil) - - cases := map[string]struct { - Spec integratedservices.IntegratedServiceSpec - Error interface{} - }{ - "empty spec": { - Spec: integratedservices.IntegratedServiceSpec{}, - Error: false, - }, - "valid spec": { - Spec: integratedservices.IntegratedServiceSpec{ - "loki": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "/loki", - }, - }, - "logging": obj{ - "metrics": true, - "tls": true, - }, - "clusterOutput": obj{ - "enabled": true, - "provider": obj{ - "name": "s3", - "secretId": "asdasd", - "bucket": obj{ - "name": "testbucket", - }, - }, - }, - }, - Error: false, - }, - "required bucket secret": { - Spec: integratedservices.IntegratedServiceSpec{ - "loki": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "/loki", - }, - }, - "logging": obj{ - "metrics": true, - "tls": true, - }, - "clusterOutput": obj{ - "enabled": true, - "provider": obj{ - "name": "oss", - "secretId": "", - "bucket": obj{ - "name": "testbucket", - }, - }, - }, - }, - Error: true, - }, - "storageaccount required": { - Spec: integratedservices.IntegratedServiceSpec{ - "loki": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "/loki", - }, - }, - "logging": obj{ - "metrics": true, - "tls": true, - }, - "clusterOutput": obj{ - "enabled": true, - "provider": obj{ - "name": "azure", - "secretId": "asdasd", - "bucket": obj{ - "name": "testbucket", - "resourceGroup": "testrg", - }, - }, - }, - }, - Error: true, - }, - "resourcegroup required": { - Spec: integratedservices.IntegratedServiceSpec{ - "loki": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "/loki", - }, - }, - "logging": obj{ - "metrics": true, - "tls": true, - }, - "clusterOutput": obj{ - "enabled": true, - "provider": obj{ - "name": "azure", - "secretId": "asdasd", - "bucket": obj{ - "name": "testbucket", - "storageAccount": "testsa", - }, - }, - }, - }, - Error: true, - }, - "invalid bucket provider": { - Spec: integratedservices.IntegratedServiceSpec{ - "loki": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "/loki", - }, - }, - "logging": obj{ - "metrics": true, - "tls": true, - }, - "clusterOutput": obj{ - "enabled": true, - "provider": obj{ - "name": "amazon", - "secretId": "asdasd", - "bucket": obj{ - "name": "testbucket", - }, - }, - }, - }, - Error: true, - }, - } - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - ctx := context.Background() - - err := mng.ValidateSpec(ctx, tc.Spec) - switch tc.Error { - case true: - assert.True(t, integratedservices.IsInputValidationError(err)) - case false, nil: - assert.NoError(t, err) - default: - assert.Equal(t, tc.Error, errors.Cause(err)) - } - }) - } -} diff --git a/internal/integratedservices/services/logging/operator.go b/internal/integratedservices/services/logging/operator.go deleted file mode 100644 index 3f76d23ba4..0000000000 --- a/internal/integratedservices/services/logging/operator.go +++ /dev/null @@ -1,499 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - "encoding/json" - "path" - - "emperror.dev/errors" - "github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1" - "github.com/mitchellh/copystructure" - corev1 "k8s.io/api/core/v1" - k8sapierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/banzaicloud/pipeline/internal/cluster/endpoints" - "github.com/banzaicloud/pipeline/internal/common" - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - "github.com/banzaicloud/pipeline/pkg/any" - "github.com/banzaicloud/pipeline/pkg/jsonstructure" - "github.com/banzaicloud/pipeline/src/auth" - pkgCluster "github.com/banzaicloud/pipeline/src/cluster" - "github.com/banzaicloud/pipeline/src/secret" -) - -// IntegratedServiceOperator implements the Logging integrated service operator -type IntegratedServiceOperator struct { - clusterGetter integratedserviceadapter.ClusterGetter - clusterService integratedservices.ClusterService - helmService services.HelmService - kubernetesService KubernetesService - endpointsService endpoints.EndpointService - config Config - logger common.Logger - secretStore services.SecretStore -} - -// MakeIntegratedServicesOperator returns a Logging integrated service operator -func MakeIntegratedServicesOperator( - clusterGetter integratedserviceadapter.ClusterGetter, - clusterService integratedservices.ClusterService, - helmService services.HelmService, - kubernetesService KubernetesService, - endpointsService endpoints.EndpointService, - config Config, - logger common.Logger, - secretStore services.SecretStore, -) IntegratedServiceOperator { - return IntegratedServiceOperator{ - clusterGetter: clusterGetter, - clusterService: clusterService, - helmService: helmService, - kubernetesService: kubernetesService, - endpointsService: endpointsService, - config: config, - logger: logger, - secretStore: secretStore, - } -} - -// Name returns the name of the Logging integrated service -func (IntegratedServiceOperator) Name() string { - return integratedServiceName -} - -// Apply applies the provided specification to the integrated service -func (op IntegratedServiceOperator) Apply(ctx context.Context, clusterID uint, spec integratedservices.IntegratedServiceSpec) error { - if err := op.clusterService.CheckClusterReady(ctx, clusterID); err != nil { - return err - } - - ctx, err := op.ensureOrgIDInContext(ctx, clusterID) - if err != nil { - return err - } - - boundSpec, err := bindIntegratedServiceSpec(spec) - if err != nil { - return integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: err.Error(), - } - } - - cl, err := op.clusterGetter.GetClusterByIDOnly(ctx, clusterID) - if err != nil { - return errors.WrapIf(err, "failed to get cluster") - } - - if err := op.processTLS(ctx, boundSpec, cl); err != nil { - return errors.WrapIf(err, "failed to generate and install TLS secret to the cluster") - } - - if err := op.installLoggingOperator(ctx, cl.GetID()); err != nil { - return errors.WrapIf(err, "failed to install logging-operator") - } - - if err := op.processLoki(ctx, boundSpec.Loki, cl); err != nil { - return errors.WrapIf(err, "failed to install Loki") - } - - if err := op.createLoggingResource(ctx, clusterID, boundSpec); err != nil { - return errors.WrapIf(err, "failed to create logging resource") - } - - outputManagers, err := op.createClusterOutputDefinitions(ctx, boundSpec, cl) - if err != nil { - return errors.WrapIf(err, "failed to create cluster output definitions") - } - - if err := op.createClusterFlowResource(ctx, outputManagers, cl.GetID()); err != nil { - return errors.WrapIf(err, "failed to create cluster flow resource") - } - - return nil -} - -// Deactivate deactivates the integrated service -func (op IntegratedServiceOperator) Deactivate(ctx context.Context, clusterID uint, spec integratedservices.IntegratedServiceSpec) error { - if err := op.clusterService.CheckClusterReady(ctx, clusterID); err != nil { - return err - } - - ctx, err := op.ensureOrgIDInContext(ctx, clusterID) - if err != nil { - return err - } - - // delete Loki deployment - if err := op.helmService.DeleteDeployment(ctx, clusterID, lokiReleaseName, op.config.Namespace); err != nil { - return errors.WrapIfWithDetails(err, "failed to delete deployment", "release", lokiReleaseName) - } - - // delete Logging-operator deployment - if err := op.helmService.DeleteDeployment(ctx, clusterID, loggingOperatorReleaseName, op.config.Namespace); err != nil { - return errors.WrapIfWithDetails(err, "failed to delete deployment", "release", loggingOperatorReleaseName) - } - - return nil -} - -func (op IntegratedServiceOperator) ensureOrgIDInContext(ctx context.Context, clusterID uint) (context.Context, error) { - if _, ok := auth.GetCurrentOrganizationID(ctx); !ok { - cluster, err := op.clusterGetter.GetClusterByIDOnly(ctx, clusterID) - if err != nil { - return ctx, errors.WrapIf(err, "failed to get cluster by ID") - } - ctx = auth.SetCurrentOrganizationID(ctx, cluster.GetOrganizationId()) - } - return ctx, nil -} - -func (op IntegratedServiceOperator) processTLS(ctx context.Context, spec integratedServiceSpec, cl integratedserviceadapter.Cluster) error { - if spec.Logging.TLS { - // generate TLS secret and save to Vault - if err := op.generateTLSSecret(cl); err != nil { - return errors.WrapIf(err, "failed to generate TLS secret") - } - - // install secret to cluster - if err := op.installTLSSecretsToCluster(ctx, cl); err != nil { - return errors.WrapIf(err, "failed to install TLS secret to the cluster") - } - } - return nil -} - -func (op IntegratedServiceOperator) generateTLSSecret(cl integratedserviceadapter.Cluster) error { - namespace := op.config.Namespace - clusterUIDSecretTag := generateClusterUIDSecretTag(cl.GetUID()) - clusterNameSecretTag := generateClusterNameSecretTag(cl.GetName()) - tlsHost := "fluentd." + namespace + ".svc.cluster.local" - - req := &secret.CreateSecretRequest{ - Name: tlsSecretName, - Type: secrettype.TLSSecretType, - Tags: []string{ - clusterNameSecretTag, - clusterUIDSecretTag, - secret.TagBanzaiReadonly, - releaseSecretTag, - integratedServiceSecretTag, - }, - Values: map[string]string{ - secrettype.TLSHosts: tlsHost, - }, - } - - _, err := secret.Store.GetOrCreate(cl.GetOrganizationId(), req) - if err != nil { - return errors.WrapIf(err, "failed to create secret") - } - - return nil -} - -func (op IntegratedServiceOperator) generateHTPasswordSecretForLoki(ctx context.Context, cl integratedserviceadapter.Cluster) error { - clusterNameSecretTag := generateClusterNameSecretTag(cl.GetName()) - clusterUIDSecretTag := generateClusterUIDSecretTag(cl.GetUID()) - - secretTags := []string{ - clusterNameSecretTag, - clusterUIDSecretTag, - releaseSecretTag, - integratedServiceSecretTag, - lokiSecretTag, - } - - adminPass, err := secret.RandomString("randAlphaNum", 12) - if err != nil { - return errors.WrapIf(err, "Loki password generation failed") - } - - secretRequest := &secret.CreateSecretRequest{ - Name: getLokiSecretName(cl.GetID()), - Type: secrettype.HtpasswdSecretType, - Values: map[string]string{ - secrettype.Username: generatedSecretUsername, - secrettype.Password: adminPass, - }, - Tags: secretTags, - } - _, err = secret.Store.CreateOrUpdate(cl.GetOrganizationId(), secretRequest) - if err != nil { - return errors.WrapIf(err, "failed to store Loki secret") - } - - return nil -} - -func (op IntegratedServiceOperator) installTLSSecretsToCluster(ctx context.Context, cl integratedserviceadapter.Cluster) error { - const kubeCaCertKey = "ca.crt" - const kubeTlsCertKey = "tls.crt" - const kubeTlsKeyKey = "tls.key" - - namespace := op.config.Namespace - installSecretRequest := pkgCluster.InstallSecretRequest{ - SourceSecretName: tlsSecretName, - Namespace: namespace, - Update: true, - Spec: map[string]pkgCluster.InstallSecretRequestSpecItem{ - kubeCaCertKey: {Source: secrettype.CACert}, - kubeTlsCertKey: {Source: secrettype.ClientCert}, - kubeTlsKeyKey: {Source: secrettype.ClientKey}, - }, - } - - // install TLS shared secret - if _, err := op.installSecret(ctx, cl, fluentSharedSecretName, installSecretRequest); err != nil { - return errors.WrapIfWithDetails(err, - "failed to install fluent shared secret to the cluster", - "clusterID", cl.GetID()) - } - - return nil -} - -func (op IntegratedServiceOperator) processLoki(ctx context.Context, spec lokiSpec, cl integratedserviceadapter.Cluster) error { - if spec.Enabled { - chartName := op.config.Charts.Loki.Chart - chartVersion := op.config.Charts.Loki.Version - - var annotations map[string]interface{} - if spec.Ingress.Enabled { - secretName, err := op.getLokiSecret(ctx, spec.Ingress, cl) - if err != nil { - return errors.WrapIf(err, "failed to get Loki secret") - } - - if err := op.installLokiSecret(ctx, secretName, cl); err != nil { - return errors.WrapIf(err, "failed to install Loki secret to cluster") - } - - annotations = generateAnnotations(secretName) - } - - domain := spec.Ingress.Domain - if domain == "" { - domain = "/" - } - - chartValues := &lokiValues{ - Ingress: ingressValues{ - Enabled: spec.Ingress.Enabled, - Hosts: []string{path.Join(domain, spec.Ingress.Path)}, - Annotations: annotations, - }, - Image: imageValues{ - Repository: op.config.Images.Loki.Repository, - Tag: op.config.Images.Loki.Tag, - }, - } - - lokiConfigValues, err := copystructure.Copy(op.config.Charts.Loki.Values) - if err != nil { - return errors.WrapIf(err, "failed to copy loki values") - } - valuesBytes, err := mergeValuesWithConfig(chartValues, lokiConfigValues) - if err != nil { - return errors.WrapIf(err, "failed to merge loki values with config") - } - - if err := op.helmService.ApplyDeployment( - ctx, - cl.GetID(), - op.config.Namespace, - chartName, - lokiReleaseName, - valuesBytes, - chartVersion, - ); err != nil { - return errors.WrapIf(err, "failed to apply Loki deployment") - } - } - - return nil -} - -func (op IntegratedServiceOperator) installLokiSecret(ctx context.Context, secretName string, cl integratedserviceadapter.Cluster) error { - installSecretRequest := pkgCluster.InstallSecretRequest{ - SourceSecretName: secretName, - Namespace: op.config.Namespace, - Spec: map[string]pkgCluster.InstallSecretRequestSpecItem{ - "auth": {Source: secrettype.HtpasswdFile}, - }, - Update: true, - } - - if _, err := op.installSecret(ctx, cl, secretName, installSecretRequest); err != nil { - return errors.WrapIfWithDetails(err, "failed to install Loki secret to cluster") - } - - return nil -} - -func (op IntegratedServiceOperator) getLokiSecret( - ctx context.Context, - ingress ingressSpec, - cl integratedserviceadapter.Cluster, -) (string, error) { - var secretName string - if ingress.SecretID == "" { - // get secret by name, this necessary in case of integrated service update - secretName = getLokiSecretName(cl.GetID()) - existingSecretID, err := op.secretStore.GetIDByName(ctx, secretName) - if existingSecretID != "" { - return secretName, nil - } else if isSecretNotFoundError(err) { - // generate and store secret - err = op.generateHTPasswordSecretForLoki(ctx, cl) - if err != nil { - return "", errors.WrapIf(err, "failed to generate Loki secret") - } - } else { - return "", errors.WrapIf(err, "error during getting Loki secret") - } - } else { - var err error - secretName, err = op.secretStore.GetNameByID(ctx, ingress.SecretID) - if err != nil { - return "", errors.WrapIfWithDetails(err, - "failed to get Loki secret", - "secretID", ingress.SecretID) - } - } - return secretName, nil -} - -func isSecretNotFoundError(err error) bool { - errCause := errors.Cause(err) - if errCause == secret.ErrSecretNotExists { - return true - } - return false -} - -func (op IntegratedServiceOperator) installSecret(ctx context.Context, cl integratedserviceadapter.Cluster, secretName string, secretRequest pkgCluster.InstallSecretRequest) (string, error) { - k8sSecName, err := pkgCluster.InstallSecret(cl, secretName, secretRequest) - if err != nil { - return "", errors.WrapIfWithDetails(err, "failed to install secret to the cluster", "clusterID", cl.GetID()) - } - - return k8sSecName, nil -} - -func (op IntegratedServiceOperator) installLoggingOperator(ctx context.Context, clusterID uint) error { - chartValues := loggingOperatorValues{ - Image: imageValues{ - Repository: op.config.Images.Operator.Repository, - Tag: op.config.Images.Operator.Tag, - }, - } - - operatorConfigValues, err := copystructure.Copy(op.config.Charts.Operator.Values) - if err != nil { - return errors.WrapIf(err, "failed to copy operator values") - } - valuesBytes, err := mergeValuesWithConfig(chartValues, operatorConfigValues) - if err != nil { - return errors.WrapIf(err, "failed to merge operator values with config") - } - - return op.helmService.ApplyDeploymentSkipCRDs( - ctx, clusterID, op.config.Namespace, op.config.Charts.Operator.Chart, loggingOperatorReleaseName, valuesBytes, op.config.Charts.Operator.Version) -} - -func mergeValuesWithConfig(chartValues interface{}, configValues interface{}) ([]byte, error) { - out, err := jsonstructure.Encode(chartValues) - if err != nil { - return nil, errors.WrapIf(err, "failed to encode chart values") - } - - result, err := any.Merge(configValues, out, jsonstructure.DefaultMergeOptions()) - if err != nil { - return nil, errors.WrapIf(err, "failed to merge values") - } - - return json.Marshal(result) -} - -func (op IntegratedServiceOperator) createLoggingResource(ctx context.Context, clusterID uint, spec integratedServiceSpec) error { - tlsEnabled := spec.Logging.TLS - loggingResource := &v1beta1.Logging{ - ObjectMeta: metav1.ObjectMeta{ - Name: loggingResourceName, - Namespace: op.config.Namespace, - Labels: map[string]string{resourceLabelKey: integratedServiceName}, - }, - Spec: v1beta1.LoggingSpec{ - FluentbitSpec: &v1beta1.FluentbitSpec{ - Image: v1beta1.ImageSpec{ - Repository: op.config.Images.Fluentbit.Repository, - Tag: op.config.Images.Fluentbit.Tag, - PullPolicy: "IfNotPresent", - }, - TLS: v1beta1.FluentbitTLS{ - Enabled: tlsEnabled, - }, - Metrics: &v1beta1.Metrics{ - ServiceMonitor: spec.Logging.Metrics, - }, - }, - FluentdSpec: &v1beta1.FluentdSpec{ - TLS: v1beta1.FluentdTLS{ - Enabled: tlsEnabled, - }, - Image: v1beta1.ImageSpec{ - Repository: op.config.Images.Fluentd.Repository, - Tag: op.config.Images.Fluentd.Tag, - PullPolicy: "IfNotPresent", - }, - Metrics: &v1beta1.Metrics{ - ServiceMonitor: spec.Logging.Metrics, - }, - }, - ControlNamespace: op.config.Namespace, - }, - } - - if tlsEnabled { - sharedKey := "fluentSharedKey" - loggingResource.Spec.FluentdSpec.TLS.SecretName = fluentSharedSecretName - loggingResource.Spec.FluentdSpec.TLS.SharedKey = sharedKey - loggingResource.Spec.FluentbitSpec.TLS.SecretName = fluentSharedSecretName - loggingResource.Spec.FluentbitSpec.TLS.SharedKey = sharedKey - } - - var oldLoggingResource v1beta1.Logging - if err := op.kubernetesService.GetObject(ctx, clusterID, corev1.ObjectReference{ - Namespace: op.config.Namespace, - Name: loggingResourceName, - }, &oldLoggingResource); err != nil { - if k8sapierrors.IsNotFound(err) { - // Logging resource is not found, create it - return op.kubernetesService.EnsureObject(ctx, clusterID, loggingResource) - } - - return errors.WrapIf(err, "failed to get Logging resource") - } - - loggingResource.ResourceVersion = oldLoggingResource.ResourceVersion - return op.kubernetesService.Update(ctx, clusterID, loggingResource) -} diff --git a/internal/integratedservices/services/logging/operator_flow_resource.go b/internal/integratedservices/services/logging/operator_flow_resource.go deleted file mode 100644 index a603566e5f..0000000000 --- a/internal/integratedservices/services/logging/operator_flow_resource.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - - "emperror.dev/errors" - "github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1" - corev1 "k8s.io/api/core/v1" - k8sapierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func (op IntegratedServiceOperator) createClusterFlowResource(ctx context.Context, managers []outputDefinitionManager, clusterID uint) error { - if len(managers) == 0 { - // create flow only in case of non empty output list - return nil - } - - flowResource := op.generateFlowResource(managers) - - var oldFlow v1beta1.ClusterFlow - if err := op.kubernetesService.GetObject(ctx, clusterID, corev1.ObjectReference{ - Namespace: op.config.Namespace, - Name: flowResourceName, - }, &oldFlow); err != nil { - if k8sapierrors.IsNotFound(err) { - // ClusterFlow resource is not found, create it - return op.kubernetesService.EnsureObject(ctx, clusterID, flowResource) - } - - return errors.WrapIf(err, "failed to get ClusterFlow resource") - } - - flowResource.ResourceVersion = oldFlow.ResourceVersion - return op.kubernetesService.Update(ctx, clusterID, flowResource) -} - -func (op IntegratedServiceOperator) generateFlowResource(definitions []outputDefinitionManager) *v1beta1.ClusterFlow { - var outputRefs []string - for _, d := range definitions { - outputRefs = append(outputRefs, d.getName()) - } - - return &v1beta1.ClusterFlow{ - ObjectMeta: metav1.ObjectMeta{ - Name: flowResourceName, - Namespace: op.config.Namespace, - Labels: map[string]string{resourceLabelKey: integratedServiceName}, - }, - Spec: v1beta1.ClusterFlowSpec{ - Match: []v1beta1.ClusterMatch{ - { - ClusterSelect: &v1beta1.ClusterSelect{}, - }, - }, - OutputRefs: outputRefs, - }, - } -} diff --git a/internal/integratedservices/services/logging/operator_output_definition.go b/internal/integratedservices/services/logging/operator_output_definition.go deleted file mode 100644 index 0638230d66..0000000000 --- a/internal/integratedservices/services/logging/operator_output_definition.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - - "emperror.dev/errors" - "github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1" - - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" -) - -func (op IntegratedServiceOperator) createClusterOutputDefinitions(ctx context.Context, spec integratedServiceSpec, cl integratedserviceadapter.Cluster) ([]outputDefinitionManager, error) { - var creators []outputManagerCreator - if spec.ClusterOutput.Enabled { - // install secrets to cluster - sourceSecretName, err := op.secretStore.GetNameByID(ctx, spec.ClusterOutput.Provider.SecretID) - if err != nil { - return nil, errors.WrapIfWithDetails(err, "failed to get secret name", "secretID", spec.ClusterOutput.Provider.SecretID) - } - - if err := op.installSecretForOutput(ctx, spec.ClusterOutput, sourceSecretName, cl); err != nil { - return nil, errors.WrapIf(err, "failed to install secret to cluster for cluster output") - } - - creators = append(creators, outputManagerCreator{ - name: spec.ClusterOutput.Provider.Name, - sourceSecretName: sourceSecretName, - providerSpec: spec.ClusterOutput.Provider, - }) - } - - if spec.Loki.Enabled { - serviceURL, err := op.getLokiServiceURL(ctx, cl) - if err != nil { - return nil, errors.WrapIf(err, "failed to get Loki service url") - } - - creators = append(creators, outputManagerCreator{ - name: providerLoki, - serviceURL: serviceURL, - }) - } - - // remove old output definitions with integrated service labels - var outputList v1beta1.ClusterOutputList - if err := op.kubernetesService.List(ctx, cl.GetID(), map[string]string{resourceLabelKey: integratedServiceName}, &outputList); err != nil { - return nil, errors.WrapIf(err, "failed to list output definitions") - } - - for _, item := range outputList.Items { - if err := op.kubernetesService.DeleteObject(ctx, cl.GetID(), &item); err != nil { - return nil, errors.WrapIfWithDetails(err, "failed to delete output definition", "name", item.Name) - } - } - - // create output definition managers - managers := newOutputDefinitionManager(creators) - for _, m := range managers { - // generate output definition - outputDefinition, err := generateOutputDefinition(ctx, m, op.secretStore, op.config.Namespace, cl.GetOrganizationId()) - if err != nil { - return nil, errors.WrapIf(err, "failed to generate output definition") - } - - // create new output definition - if err := op.kubernetesService.EnsureObject(ctx, cl.GetID(), outputDefinition); err != nil { - return nil, errors.WrapIf(err, "failed to create output definition") - } - } - - return managers, nil -} - -func (op IntegratedServiceOperator) getLokiServiceURL(ctx context.Context, cl integratedserviceadapter.Cluster) (string, error) { - k8sConfig, err := cl.GetK8sConfig() - if err != nil { - return "", errors.WrapIfWithDetails(err, "failed to get kubeconfig", "cluster", cl.GetID()) - } - - return op.endpointsService.GetServiceURL(ctx, k8sConfig, lokiServiceName, op.config.Namespace) -} - -func (op IntegratedServiceOperator) installSecretForOutput(ctx context.Context, spec clusterOutputSpec, sourceSecretName string, cl integratedserviceadapter.Cluster) error { - secretManager, err := newOutputSecretInstallManager(spec.Provider.Name, sourceSecretName, op.config.Namespace) - if err != nil { - return errors.WrapIf(err, "failed to create output secret installer") - } - - secretValues, err := op.secretStore.GetSecretValues(ctx, spec.Provider.SecretID) - if err != nil { - return errors.WrapIfWithDetails(err, "failed to get secret values", "secretID", spec.Provider.SecretID) - } - - installSecretRequest, err := secretManager.generateSecretRequest(secretValues, spec.Provider.Bucket) - if err != nil { - return errors.WrapIf(err, "failed to generate install secret request") - } - - if _, err := op.installSecret(ctx, cl, sourceSecretName, *installSecretRequest); err != nil { - return errors.WrapIf(err, "failed to install secret to cluster") - } - - return nil -} diff --git a/internal/integratedservices/services/logging/operator_test.go b/internal/integratedservices/services/logging/operator_test.go deleted file mode 100644 index 74f11b4a66..0000000000 --- a/internal/integratedservices/services/logging/operator_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/banzaicloud/pipeline/internal/common/commonadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - pkgCluster "github.com/banzaicloud/pipeline/pkg/cluster" - "github.com/banzaicloud/pipeline/src/auth" - "github.com/banzaicloud/pipeline/src/secret" -) - -func TestIntegratedServiceOperator_Name(t *testing.T) { - op := MakeIntegratedServicesOperator(nil, nil, nil, nil, nil, Config{}, nil, nil) - - assert.Equal(t, "logging", op.Name()) -} - -func TestIntegratedServiceOperator_Apply(t *testing.T) { - clusterID := uint(42) - orgID := uint(13) - - clusterGetter := dummyClusterGetter{ - Clusters: map[uint]dummyCluster{}, - } - clusterService := integratedserviceadapter.NewClusterService(clusterGetter) - helmService := dummyHelmService{} - - orgSecretStore := dummyOrganizationalSecretStore{ - Secrets: map[uint]map[string]*secret.SecretItemResponse{ - orgID: { - lokiSecretID: { - ID: lokiSecretID, - Name: getLokiSecretName(clusterID), - Type: secrettype.HtpasswdSecretType, - Values: map[string]string{secrettype.Username: "admin", secrettype.Password: "pass"}, - Tags: []string{secret.TagBanzaiReadonly}, - Version: 1, - }, - }, - }, - } - - logger := services.NoopLogger{} - secretStore := commonadapter.NewSecretStore(orgSecretStore, commonadapter.OrgIDContextExtractorFunc(auth.GetCurrentOrganizationID)) - kubernetesService := dummyKubernetesService{} - endpointService := dummyEndpointService{} - op := MakeIntegratedServicesOperator(clusterGetter, clusterService, helmService, &kubernetesService, endpointService, Config{}, logger, secretStore) - - cases := map[string]struct { - Spec integratedservices.IntegratedServiceSpec - Cluster dummyCluster - Error interface{} - }{ - "cluster not ready": { - Spec: integratedservices.IntegratedServiceSpec{}, - Cluster: dummyCluster{ - OrgID: orgID, - Status: pkgCluster.Creating, - ID: clusterID, - }, - Error: integratedservices.ClusterIsNotReadyError{ - ClusterID: clusterID, - }, - }, - "enable Loki": { - Spec: integratedservices.IntegratedServiceSpec{ - "loki": obj{ - "enabled": true, - "ingress": obj{ - "enabled": false, - }, - }, - "logging": obj{ - "metrics": true, - "tls": false, - }, - "clusterOutput": obj{ - "enabled": false, - }, - }, - Cluster: dummyCluster{ - OrgID: orgID, - Status: pkgCluster.Running, - ID: clusterID, - }, - Error: false, - }, - } - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - clusterGetter.Clusters[clusterID] = tc.Cluster - - ctx := auth.SetCurrentOrganizationID(context.Background(), orgID) - - err := op.Apply(ctx, clusterID, tc.Spec) - switch tc.Error { - case nil, false: - assert.NoError(t, err) - case true: - assert.Error(t, err) - default: - assert.Equal(t, tc.Error, err) - } - }) - } -} - -func TestIntegratedServiceOperator_Deactivate(t *testing.T) { - clusterID := uint(42) - orgID := uint(13) - - clusterGetter := dummyClusterGetter{ - Clusters: map[uint]dummyCluster{ - clusterID: { - Status: pkgCluster.Running, - ID: clusterID, - }, - }, - } - clusterService := integratedserviceadapter.NewClusterService(clusterGetter) - helmService := dummyHelmService{} - endpointService := dummyEndpointService{} - orgSecretStore := dummyOrganizationalSecretStore{ - Secrets: map[uint]map[string]*secret.SecretItemResponse{ - orgID: nil, - }, - } - secretStore := commonadapter.NewSecretStore(orgSecretStore, commonadapter.OrgIDContextExtractorFunc(auth.GetCurrentOrganizationID)) - logger := services.NoopLogger{} - kubernetesService := dummyKubernetesService{} - op := MakeIntegratedServicesOperator(clusterGetter, clusterService, helmService, &kubernetesService, endpointService, Config{}, logger, secretStore) - - ctx := context.Background() - - _ = op.Deactivate(ctx, clusterID, nil) -} diff --git a/internal/integratedservices/services/logging/output_definition.go b/internal/integratedservices/services/logging/output_definition.go deleted file mode 100644 index 6405f405b5..0000000000 --- a/internal/integratedservices/services/logging/output_definition.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "context" - - "emperror.dev/errors" - "github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/banzaicloud/pipeline/internal/common" -) - -type outputManagerCreator struct { - name string - sourceSecretName string - serviceURL string - providerSpec providerSpec -} - -type outputDefinitionManager interface { - getOutputSpec(bucketSpec, bucketOptions) v1beta1.ClusterOutputSpec - getProviderSpec() providerSpec - getName() string -} - -func newOutputDefinitionManager(creators []outputManagerCreator) (managers []outputDefinitionManager) { - for _, creator := range creators { - baseManager := baseOutputManager{ - sourceSecretName: creator.sourceSecretName, - providerSpec: creator.providerSpec, - } - switch creator.name { - case providerAmazonS3: - managers = append(managers, outputDefinitionManagerS3{baseOutputManager: baseManager}) - case providerGoogleGCS: - managers = append(managers, outputDefinitionManagerGCS{baseOutputManager: baseManager}) - case providerAzure: - managers = append(managers, outputDefinitionManagerAzure{baseOutputManager: baseManager}) - case providerLoki: - managers = append(managers, outputDefinitionManagerLoki{serviceURL: creator.serviceURL}) - } - } - - return -} - -func generateOutputDefinition( - ctx context.Context, - m outputDefinitionManager, - secretStore common.SecretStore, - namespace string, - orgID uint, -) (*v1beta1.ClusterOutput, error) { - spec := m.getProviderSpec() - bucketOptions := &bucketOptions{} - if spec.SecretID != "" { - secretValues, err := secretStore.GetSecretValues(ctx, spec.SecretID) - if err != nil { - return nil, errors.WrapIfWithDetails(err, "failed to get secret", "secretID", spec.SecretID) - } - - bucketOptions, err = generateBucketOptions(spec, secretValues, orgID) - if err != nil { - return nil, errors.WrapIf(err, "failed to generate bucket options") - } - } - - return &v1beta1.ClusterOutput{ - ObjectMeta: metav1.ObjectMeta{ - Name: m.getName(), - Namespace: namespace, - Labels: map[string]string{resourceLabelKey: integratedServiceName}, - }, - Spec: m.getOutputSpec(spec.Bucket, *bucketOptions), - }, nil -} diff --git a/internal/integratedservices/services/logging/output_definition_azure.go b/internal/integratedservices/services/logging/output_definition_azure.go deleted file mode 100644 index d4d8779aa9..0000000000 --- a/internal/integratedservices/services/logging/output_definition_azure.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1" - "github.com/banzaicloud/logging-operator/pkg/sdk/model/output" - loggingSecret "github.com/banzaicloud/operator-tools/pkg/secret" - v1 "k8s.io/api/core/v1" -) - -type outputDefinitionManagerAzure struct { - baseOutputManager -} - -func (outputDefinitionManagerAzure) getName() string { - return "azure-output" -} - -func (m outputDefinitionManagerAzure) getOutputSpec(spec bucketSpec, _ bucketOptions) v1beta1.ClusterOutputSpec { - return v1beta1.ClusterOutputSpec{ - OutputSpec: v1beta1.OutputSpec{ - AzureStorage: &output.AzureStorage{ - Path: m.getPathSpec(), - AzureStorageAccount: &loggingSecret.Secret{ - ValueFrom: &loggingSecret.ValueFrom{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: m.sourceSecretName, - }, - Key: outputDefinitionSecretKeyAzureStorageAccount, - }, - }, - }, - AzureStorageAccessKey: &loggingSecret.Secret{ - ValueFrom: &loggingSecret.ValueFrom{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: m.sourceSecretName, - }, - Key: outputDefinitionSecretKeyAzureStorageAccess, - }, - }, - }, - AzureContainer: spec.Name, - Buffer: m.getBufferSpec(), - }, - }, - } -} diff --git a/internal/integratedservices/services/logging/output_definition_base.go b/internal/integratedservices/services/logging/output_definition_base.go deleted file mode 100644 index c18773f78c..0000000000 --- a/internal/integratedservices/services/logging/output_definition_base.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import "github.com/banzaicloud/logging-operator/pkg/sdk/model/output" - -type baseOutputManager struct { - sourceSecretName string - providerSpec providerSpec -} - -func (baseOutputManager) getBufferSpec() *output.Buffer { - return &output.Buffer{ - Timekey: "1m", - TimekeyWait: "10s", - TimekeyUseUtc: true, - } -} - -func (baseOutputManager) getPathSpec() string { - return "logs/${tag}/%Y/%m/%d/" -} - -func (b baseOutputManager) getProviderSpec() providerSpec { - return b.providerSpec -} diff --git a/internal/integratedservices/services/logging/output_definition_bucket.go b/internal/integratedservices/services/logging/output_definition_bucket.go deleted file mode 100644 index d879d86017..0000000000 --- a/internal/integratedservices/services/logging/output_definition_bucket.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "emperror.dev/errors" - - "github.com/banzaicloud/pipeline/internal/providers" - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - pkgCluster "github.com/banzaicloud/pipeline/pkg/cluster" - "github.com/banzaicloud/pipeline/src/secret" -) - -type bucketOptions struct { - s3 *struct { - region string - } - gcs *struct { - project string - } -} - -func generateBucketOptions(spec providerSpec, secretValues map[string]string, orgID uint) (*bucketOptions, error) { - secretItems := &secret.SecretItemResponse{ - Values: secretValues, - } - switch spec.Name { - case providerAmazonS3: - return generateS3BucketOptions(spec, secretItems, orgID) - case providerGoogleGCS: - return generateGCSBucketOptions(secretValues), nil - default: - return &bucketOptions{}, nil - } -} - -func generateS3BucketOptions(spec providerSpec, secretItems *secret.SecretItemResponse, orgID uint) (*bucketOptions, error) { - region, err := providers.GetBucketLocation(pkgCluster.Amazon, secretItems, spec.Bucket.Name, orgID, nil) - if err != nil { - return nil, errors.WrapIfWithDetails(err, "failed to get S3 bucket region", "bucket", spec.Bucket) - } - return &bucketOptions{ - s3: &struct { - region string - }{ - region: region, - }, - }, nil -} - -func generateGCSBucketOptions(secretValues map[string]string) *bucketOptions { - return &bucketOptions{ - gcs: &struct { - project string - }{ - project: secretValues[secrettype.ProjectId], - }, - } -} diff --git a/internal/integratedservices/services/logging/output_definition_gcs.go b/internal/integratedservices/services/logging/output_definition_gcs.go deleted file mode 100644 index 3ac92e041d..0000000000 --- a/internal/integratedservices/services/logging/output_definition_gcs.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1" - "github.com/banzaicloud/logging-operator/pkg/sdk/model/output" - loggingSecret "github.com/banzaicloud/operator-tools/pkg/secret" - v1 "k8s.io/api/core/v1" -) - -type outputDefinitionManagerGCS struct { - baseOutputManager -} - -func (outputDefinitionManagerGCS) getName() string { - return "gcs-output" -} - -func (m outputDefinitionManagerGCS) getOutputSpec(spec bucketSpec, op bucketOptions) v1beta1.ClusterOutputSpec { - return v1beta1.ClusterOutputSpec{ - OutputSpec: v1beta1.OutputSpec{ - GCSOutput: &output.GCSOutput{ - Project: op.gcs.project, - Keyfile: "", - CredentialsJson: &loggingSecret.Secret{ - ValueFrom: &loggingSecret.ValueFrom{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: m.sourceSecretName, - }, - Key: outputDefinitionSecretKeyGCS, - }, - }, - }, - Bucket: spec.Name, - Path: m.getPathSpec(), - Buffer: m.getBufferSpec(), - }, - }, - } -} diff --git a/internal/integratedservices/services/logging/output_definition_loki.go b/internal/integratedservices/services/logging/output_definition_loki.go deleted file mode 100644 index 78e2e8f01f..0000000000 --- a/internal/integratedservices/services/logging/output_definition_loki.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "fmt" - - "github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1" - "github.com/banzaicloud/logging-operator/pkg/sdk/model/output" -) - -type outputDefinitionManagerLoki struct { - serviceURL string -} - -func (o outputDefinitionManagerLoki) getOutputSpec(_ bucketSpec, _ bucketOptions) v1beta1.ClusterOutputSpec { - return v1beta1.ClusterOutputSpec{ - OutputSpec: v1beta1.OutputSpec{ - LokiOutput: &output.LokiOutput{ - Url: fmt.Sprintf("http://%s", o.serviceURL), - ConfigureKubernetesLabels: true, - }, - }, - } -} - -func (outputDefinitionManagerLoki) getName() string { - return lokiOutputDefinitionName -} - -func (outputDefinitionManagerLoki) getProviderSpec() providerSpec { - return providerSpec{} -} diff --git a/internal/integratedservices/services/logging/output_definition_s3.go b/internal/integratedservices/services/logging/output_definition_s3.go deleted file mode 100644 index ea8097fb27..0000000000 --- a/internal/integratedservices/services/logging/output_definition_s3.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1" - "github.com/banzaicloud/logging-operator/pkg/sdk/model/output" - loggingSecret "github.com/banzaicloud/operator-tools/pkg/secret" - v1 "k8s.io/api/core/v1" -) - -type outputDefinitionManagerS3 struct { - baseOutputManager -} - -func (outputDefinitionManagerS3) getName() string { - return "s3-output" -} - -func (m outputDefinitionManagerS3) getOutputSpec(spec bucketSpec, op bucketOptions) v1beta1.ClusterOutputSpec { - return v1beta1.ClusterOutputSpec{ - OutputSpec: v1beta1.OutputSpec{ - S3OutputConfig: &output.S3OutputConfig{ - AwsAccessKey: &loggingSecret.Secret{ - ValueFrom: &loggingSecret.ValueFrom{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: m.sourceSecretName, - }, - Key: outputDefinitionSecretKeyS3AccessKeyID, - }, - }, - }, - AwsSecretKey: &loggingSecret.Secret{ - ValueFrom: &loggingSecret.ValueFrom{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: m.sourceSecretName, - }, - Key: outputDefinitionSecretKeyS3AccessKey, - }, - }, - }, - Path: m.getPathSpec(), - S3Region: op.s3.region, - S3Bucket: spec.Name, - Buffer: m.getBufferSpec(), - Format: &output.Format{ - Type: "json", - }, - }, - }, - } -} diff --git a/internal/integratedservices/services/logging/output_definition_secret.go b/internal/integratedservices/services/logging/output_definition_secret.go deleted file mode 100644 index 548a292c26..0000000000 --- a/internal/integratedservices/services/logging/output_definition_secret.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "emperror.dev/errors" - - pkgCluster "github.com/banzaicloud/pipeline/src/cluster" -) - -type outputSecretInstallManager interface { - generateSecretRequest(map[string]string, bucketSpec) (*pkgCluster.InstallSecretRequest, error) -} - -type baseOutputSecretInstallManager struct { - sourceSecretName string - namespace string -} - -func newOutputSecretInstallManager(providerName, sourceSecretName, namespace string) (outputSecretInstallManager, error) { - switch providerName { - case providerAmazonS3: - return outputSecretInstallManagerS3{baseOutputSecretInstallManager{ - sourceSecretName: sourceSecretName, - namespace: namespace, - }}, nil - case providerGoogleGCS: - return outputSecretInstallManagerGCS{baseOutputSecretInstallManager{ - sourceSecretName: sourceSecretName, - namespace: namespace, - }}, nil - case providerAzure: - return outputSecretInstallManagerAzure{baseOutputSecretInstallManager{ - sourceSecretName: sourceSecretName, - namespace: namespace, - }}, nil - default: - return nil, errors.NewWithDetails("unsupported provider", "provider", providerName) - } -} diff --git a/internal/integratedservices/services/logging/output_definition_secret_azure.go b/internal/integratedservices/services/logging/output_definition_secret_azure.go deleted file mode 100644 index 50e242bdf8..0000000000 --- a/internal/integratedservices/services/logging/output_definition_secret_azure.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "emperror.dev/errors" - - "github.com/banzaicloud/pipeline/pkg/providers/azure" - azureObjectstore "github.com/banzaicloud/pipeline/pkg/providers/azure/objectstore" - pkgCluster "github.com/banzaicloud/pipeline/src/cluster" -) - -type outputSecretInstallManagerAzure struct { - baseOutputSecretInstallManager -} - -func (m outputSecretInstallManagerAzure) generateSecretRequest(secretValues map[string]string, spec bucketSpec) (*pkgCluster.InstallSecretRequest, error) { - credentials := *azure.NewCredentials(secretValues) - - storageAccountClient, err := azureObjectstore.NewAuthorizedStorageAccountClientFromSecret(credentials) - if err != nil { - return nil, errors.WrapIf(err, "failed to create storage account client") - } - sak, err := storageAccountClient.GetStorageAccountKey(spec.ResourceGroup, spec.StorageAccount) - if err != nil { - return nil, errors.WrapIf(err, "get storage account key failed") - } - - return &pkgCluster.InstallSecretRequest{ - SourceSecretName: m.sourceSecretName, - Namespace: m.namespace, - Spec: map[string]pkgCluster.InstallSecretRequestSpecItem{ - outputDefinitionSecretKeyAzureStorageAccount: { - Value: spec.StorageAccount, - }, - outputDefinitionSecretKeyAzureStorageAccess: { - Value: sak, - }, - }, - Update: true, - }, nil -} diff --git a/internal/integratedservices/services/logging/output_definition_secret_gcs.go b/internal/integratedservices/services/logging/output_definition_secret_gcs.go deleted file mode 100644 index de01f11c9b..0000000000 --- a/internal/integratedservices/services/logging/output_definition_secret_gcs.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "encoding/json" - - "emperror.dev/errors" - - pkgCluster "github.com/banzaicloud/pipeline/src/cluster" -) - -type outputSecretInstallManagerGCS struct { - baseOutputSecretInstallManager -} - -func (m outputSecretInstallManagerGCS) generateSecretRequest(secretValues map[string]string, _ bucketSpec) (*pkgCluster.InstallSecretRequest, error) { - raw, err := json.Marshal(secretValues) - if err != nil { - return nil, errors.Wrap(err, "failed to marshal secret values") - } - - return &pkgCluster.InstallSecretRequest{ - SourceSecretName: m.sourceSecretName, - Namespace: m.namespace, - Spec: map[string]pkgCluster.InstallSecretRequestSpecItem{ - outputDefinitionSecretKeyGCS: { - Value: string(raw), - }, - }, - Update: true, - }, nil -} diff --git a/internal/integratedservices/services/logging/output_definition_secret_s3.go b/internal/integratedservices/services/logging/output_definition_secret_s3.go deleted file mode 100644 index 8a7e75b9bd..0000000000 --- a/internal/integratedservices/services/logging/output_definition_secret_s3.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - pkgCluster "github.com/banzaicloud/pipeline/src/cluster" -) - -type outputSecretInstallManagerS3 struct { - baseOutputSecretInstallManager -} - -func (m outputSecretInstallManagerS3) generateSecretRequest(_ map[string]string, _ bucketSpec) (*pkgCluster.InstallSecretRequest, error) { - return &pkgCluster.InstallSecretRequest{ - SourceSecretName: m.sourceSecretName, - Namespace: m.namespace, - Spec: map[string]pkgCluster.InstallSecretRequestSpecItem{ - outputDefinitionSecretKeyS3AccessKeyID: {Source: secrettype.AwsAccessKeyId}, - outputDefinitionSecretKeyS3AccessKey: {Source: secrettype.AwsSecretAccessKey}, - }, - Update: true, - }, nil -} diff --git a/internal/integratedservices/services/logging/spec.go b/internal/integratedservices/services/logging/spec.go deleted file mode 100644 index a8a6b2e8e9..0000000000 --- a/internal/integratedservices/services/logging/spec.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -import ( - "emperror.dev/errors" - "github.com/mitchellh/mapstructure" - - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/src/dns" -) - -type integratedServiceSpec struct { - Loki lokiSpec `json:"loki" mapstructure:"loki"` - Logging loggingSpec `json:"logging" mapstructure:"logging"` - ClusterOutput clusterOutputSpec `json:"clusterOutput" mapstructure:"clusterOutput"` -} - -type lokiSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - Ingress ingressSpec `json:"ingress" mapstructure:"ingress"` -} - -type ingressSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - Domain string `json:"domain" mapstructure:"domain"` - Path string `json:"path" mapstructure:"path"` - SecretID string `json:"secretId" mapstructure:"secretId"` -} - -type loggingSpec struct { - Metrics bool `json:"metrics" mapstructure:"metrics"` - TLS bool `json:"tls" mapstructure:"tls"` -} - -type clusterOutputSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - Provider providerSpec `json:"provider" mapstructure:"provider"` -} - -type providerSpec struct { - Name string `json:"name" mapstructure:"name"` - Bucket bucketSpec `json:"bucket" mapstructure:"bucket"` - SecretID string `json:"secretId" mapstructure:"secretId"` -} - -type bucketSpec struct { - Name string `json:"name" mapstructure:"name"` - ResourceGroup string `json:"resourceGroup" mapstructure:"resourceGroup"` - StorageAccount string `json:"storageAccount" mapstructure:"storageAccount"` -} - -func bindIntegratedServiceSpec(spec integratedservices.IntegratedServiceSpec) (integratedServiceSpec, error) { - var integratedServiceSpec integratedServiceSpec - if err := mapstructure.Decode(spec, &integratedServiceSpec); err != nil { - return integratedServiceSpec, integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: "failed to bind integrated service spec", - } - } - - return integratedServiceSpec, nil -} - -func (s integratedServiceSpec) Validate() error { - if err := s.Loki.Validate(); err != nil { - return err - } - - if err := s.ClusterOutput.Validate(); err != nil { - return err - } - - return nil -} - -func (s lokiSpec) Validate() error { - if s.Enabled { - if err := s.Ingress.Validate(); err != nil { - return errors.WrapIf(err, "error during validating Loki ingress") - } - } - - return nil -} - -func (s ingressSpec) Validate() error { - if s.Enabled { - if s.Path == "" { - return requiredFieldError{name: "path"} - } - - if s.Domain != "" { - err := dns.ValidateSubdomain(s.Domain) - if err != nil { - return errors.New("invalid ingress domain") - } - } - } - - return nil -} - -func (s clusterOutputSpec) Validate() error { - if s.Enabled { - if err := s.Provider.Validate(); err != nil { - return errors.WrapIf(err, "error during validating provider") - } - } - - return nil -} - -func (s providerSpec) Validate() error { - if s.SecretID == "" { - return requiredFieldError{name: "secretId"} - } - - if s.Name == "" { - return requiredFieldError{name: "name"} - } - - switch s.Name { - case providerAmazonS3, providerAzure, providerGoogleGCS: - default: - return errors.New("invalid provider name") - } - - if err := s.Bucket.Validate(s.Name); err != nil { - return errors.WrapIf(err, "error during bucket validation") - } - - return nil -} - -func (s bucketSpec) Validate(provider string) error { - if s.Name == "" { - return requiredFieldError{name: "name"} - } - - if provider == providerAzure { - if s.ResourceGroup == "" { - return requiredFieldError{name: "resourceGroup"} - } - - if s.StorageAccount == "" { - return requiredFieldError{name: "storageAccount"} - } - } - - return nil -} diff --git a/internal/integratedservices/services/logging/values.go b/internal/integratedservices/services/logging/values.go deleted file mode 100644 index 9473f298cd..0000000000 --- a/internal/integratedservices/services/logging/values.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logging - -type loggingOperatorValues struct { - Image imageValues `json:"image" mapstructure:"image"` -} - -type imageValues struct { - Repository string `json:"repository" mapstructure:"repository"` - Tag string `json:"tag" mapstructure:"tag"` - PullPolicy string `json:"pullPolicy" mapstructure:"pullPolicy"` -} - -type lokiValues struct { - Ingress ingressValues `json:"ingress" mapstructure:"ingress"` - Image imageValues `json:"image"` -} - -type ingressValues struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - Hosts []string `json:"hosts" mapstructure:"hosts"` - Path string `json:"path,omitempty" mapstructure:"path"` - Annotations map[string]interface{} `json:"annotations,omitempty" mapstructure:"annotations"` -} diff --git a/internal/integratedservices/services/monitoring/BUILD.plz b/internal/integratedservices/services/monitoring/BUILD.plz deleted file mode 100644 index 0902c40bea..0000000000 --- a/internal/integratedservices/services/monitoring/BUILD.plz +++ /dev/null @@ -1,74 +0,0 @@ -go_library( - name = "monitoring", - srcs = glob( - ["*.go"], - exclude = ["*_test.go"], - ), - visibility = ["PUBLIC"], - deps = [ - "//internal/cluster/endpoints", - "//internal/common", - "//internal/helm", - "//internal/integratedservices", - "//internal/integratedservices/integratedserviceadapter", - "//internal/integratedservices/services", - "//internal/secret/secrettype", - "//pkg/any", - "//pkg/helm", - "//pkg/jsonstructure", - "//pkg/k8sclient", - "//src/auth", - "//src/cluster", - "//src/dns", - "//src/secret", - "//third_party/go:emperror.dev__errors", - "//third_party/go:github.com__Masterminds__semver__v3", - "//third_party/go:github.com__mitchellh__copystructure", - "//third_party/go:github.com__mitchellh__mapstructure", - "//third_party/go:k8s.io__api__storage__v1beta1", - "//third_party/go:k8s.io__apiextensions-apiserver__pkg__apis__apiextensions__v1beta1", - "//third_party/go:k8s.io__apimachinery__pkg__api__errors", - "//third_party/go:k8s.io__apimachinery__pkg__apis__meta__v1", - "//third_party/go:k8s.io__apimachinery__pkg__labels", - "//third_party/go:k8s.io__apimachinery__pkg__runtime", - "//third_party/go:k8s.io__client-go__kubernetes", - ], -) - -go_test( - name = "test", - srcs = glob(["*.go"]), - deps = [ - "//internal/cluster/endpoints", - "//internal/common", - "//internal/common/commonadapter", - "//internal/helm", - "//internal/integratedservices", - "//internal/integratedservices/integratedserviceadapter", - "//internal/integratedservices/services", - "//internal/secret/secrettype", - "//pkg/any", - "//pkg/cluster", - "//pkg/helm", - "//pkg/jsonstructure", - "//pkg/k8sclient", - "//src/auth", - "//src/cluster", - "//src/dns", - "//src/secret", - "//third_party/go:emperror.dev__errors", - "//third_party/go:github.com__Masterminds__semver__v3", - "//third_party/go:github.com__mitchellh__copystructure", - "//third_party/go:github.com__mitchellh__mapstructure", - "//third_party/go:github.com__stretchr__testify__assert", - "//third_party/go:k8s.io__api__extensions__v1beta1", - "//third_party/go:k8s.io__api__storage__v1beta1", - "//third_party/go:k8s.io__apiextensions-apiserver__pkg__apis__apiextensions__v1beta1", - "//third_party/go:k8s.io__apimachinery__pkg__api__errors", - "//third_party/go:k8s.io__apimachinery__pkg__apis__meta__v1", - "//third_party/go:k8s.io__apimachinery__pkg__labels", - "//third_party/go:k8s.io__apimachinery__pkg__runtime", - "//third_party/go:k8s.io__client-go__kubernetes", - "//third_party/go:k8s.io__client-go__kubernetes__fake", - ], -) diff --git a/internal/integratedservices/services/monitoring/common.go b/internal/integratedservices/services/monitoring/common.go deleted file mode 100644 index 863eedd113..0000000000 --- a/internal/integratedservices/services/monitoring/common.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "fmt" -) - -const ( - integratedServiceName = "monitoring" - prometheusOperatorReleaseName = "monitor" - prometheusPushgatewayReleaseName = "pushgateway" - grafanaSecretTag = "app:grafana" - prometheusSecretTag = "app:prometheus" - alertmanagerSecretTag = "app:alertmanager" - integratedServiceSecretTag = "feature:monitoring" - generatedSecretUsername = "admin" - alertManagerProviderConfigName = "default-receiver" - alertManagerNullReceiverName = "null" - - ingressTypeGrafana = "Grafana" - ingressTypePrometheus = "Prometheus" - ingressTypeAlertmanager = "Alertmanager" - - pagerDutyIntegrationEventApiV2 = "eventsApiV2" - pagerDutyIntegrationPrometheus = "prometheus" - - alertmanagerProviderSlack = "slack" - alertmanagerProviderPagerDuty = "pagerDuty" -) - -func getClusterNameSecretTag(clusterName string) string { - return fmt.Sprintf("cluster:%s", clusterName) -} - -func getClusterUIDSecretTag(clusterUID string) string { - return fmt.Sprintf("clusterUID:%s", clusterUID) -} - -func getReleaseSecretTag() string { - return fmt.Sprintf("release:%s", prometheusOperatorReleaseName) -} - -func getPrometheusSecretName(clusterID uint) string { - return fmt.Sprintf("cluster-%d-prometheus", clusterID) -} - -func getAlertmanagerSecretName(clusterID uint) string { - return fmt.Sprintf("cluster-%d-alertmanager", clusterID) -} - -func getPushgatewaySecretName(clusterID uint) string { - return fmt.Sprintf("cluster-%d-pushgateway", clusterID) -} - -func getGrafanaSecretName(clusterID uint) string { - return fmt.Sprintf("cluster-%d-grafana", clusterID) -} - -func generateAnnotations(secretName string) map[string]interface{} { - return map[string]interface{}{ - "traefik.ingress.kubernetes.io/auth-type": "basic", - "traefik.ingress.kubernetes.io/auth-secret": secretName, - } -} diff --git a/internal/integratedservices/services/monitoring/common_test.go b/internal/integratedservices/services/monitoring/common_test.go deleted file mode 100644 index 89625aab10..0000000000 --- a/internal/integratedservices/services/monitoring/common_test.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - - "emperror.dev/errors" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/pkg/helm" - "github.com/banzaicloud/pipeline/src/secret" -) - -type obj = map[string]interface{} - -const ( - grafanaSecretID = "grafanaSecretID" - prometheusSecretID = "prometheusSecretID" - grafanaPath = "/grafana" - prometheusPath = "/prometheus" - grafanaURL = "http://monitoring.io/grafana" - prometheusURL = "http://monitoring.io/prometheus" - serviceUrl = "dummyServiceUrl:9090" -) - -type dummyClusterGetter struct { - Clusters map[uint]dummyCluster -} - -func (d dummyClusterGetter) GetClusterByIDOnly(ctx context.Context, clusterID uint) (integratedserviceadapter.Cluster, error) { - return d.Clusters[clusterID], nil -} - -func (d dummyClusterGetter) GetClusterStatus(ctx context.Context, clusterID uint) (string, error) { - if c, ok := d.Clusters[clusterID]; ok { - return c.Status, nil - } - return "", errors.New("cluster not found") -} - -type dummyCluster struct { - K8sConfig []byte - Name string - OrgID uint - ID uint - UID string - NodePools map[string]bool - Rbac bool - Status string -} - -func (d dummyCluster) GetK8sConfig() ([]byte, error) { - return d.K8sConfig, nil -} - -func (d dummyCluster) GetName() string { - return d.Name -} - -func (d dummyCluster) GetOrganizationId() uint { - return d.OrgID -} - -func (d dummyCluster) GetUID() string { - return d.UID -} - -func (d dummyCluster) GetID() uint { - return d.ID -} - -func (d dummyCluster) NodePoolExists(nodePoolName string) bool { - return d.NodePools[nodePoolName] -} - -func (d dummyCluster) RbacEnabled() bool { - return d.Rbac -} - -type dummyOrganizationalSecretStore struct { - Secrets map[uint]map[string]*secret.SecretItemResponse -} - -func (d dummyOrganizationalSecretStore) Get(orgID uint, secretID string) (*secret.SecretItemResponse, error) { - if orgSecrets, ok := d.Secrets[orgID]; ok { - if sir, ok := orgSecrets[secretID]; ok { - return sir, nil - } - } - return nil, secret.ErrSecretNotExists -} - -func (d dummyOrganizationalSecretStore) Store(organizationID uint, request *secret.CreateSecretRequest) (string, error) { - return prometheusSecretID, nil -} - -func (d dummyOrganizationalSecretStore) GetByName(organizationID uint, name string) (*secret.SecretItemResponse, error) { - if orgSecrets, ok := d.Secrets[organizationID]; ok { - for n, sir := range orgSecrets { - if n == name { - return sir, nil - } - } - } - return nil, secret.ErrSecretNotExists -} - -func (d dummyOrganizationalSecretStore) Delete(organizationID uint, secretID string) error { - return nil -} - -type dummyEndpointService struct{} - -func (dummyEndpointService) List(ctx context.Context, kubeConfig []byte, releaseName string) ([]*helm.EndpointItem, error) { - return []*helm.EndpointItem{ - { - Name: "ingress-traefik", - EndPointURLs: []*helm.EndPointURLs{ - { - Path: grafanaPath, - URL: grafanaURL, - ReleaseName: releaseName, - }, - { - Path: prometheusPath, - URL: prometheusURL, - ReleaseName: releaseName, - }, - }, - }, - }, nil -} - -func (dummyEndpointService) GetServiceURL(ctx context.Context, kubeConfig []byte, serviceName string, namespace string) (string, error) { - return "dummyServiceUrl:9090", nil -} - -type dummyHelmService struct{} - -func (d dummyHelmService) ApplyDeployment( - ctx context.Context, - clusterID uint, - namespace string, - deploymentName string, - releaseName string, - values []byte, - chartVersion string, -) error { - return nil -} - -func (d dummyHelmService) ApplyDeploymentSkipCRDs( - ctx context.Context, - clusterID uint, - namespace string, - deploymentName string, - releaseName string, - values []byte, - chartVersion string, -) error { - return nil -} - -func (d dummyHelmService) DeleteDeployment(ctx context.Context, clusterID uint, releaseName, namespace string) error { - return nil -} - -func (d dummyHelmService) GetDeployment(ctx context.Context, clusterID uint, releaseName, namespace string) (*helm.GetDeploymentResponse, error) { - return &helm.GetDeploymentResponse{ - ReleaseName: releaseName, - }, nil -} - -type dummyKubernetesService struct { -} - -func (s *dummyKubernetesService) DeleteObject(ctx context.Context, clusterID uint, o runtime.Object) error { - return nil -} - -func (s *dummyKubernetesService) List(ctx context.Context, clusterID uint, labels map[string]string, o runtime.Object) error { - return nil -} diff --git a/internal/integratedservices/services/monitoring/config.go b/internal/integratedservices/services/monitoring/config.go deleted file mode 100644 index 3ec00b4ca8..0000000000 --- a/internal/integratedservices/services/monitoring/config.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "emperror.dev/errors" -) - -// Config contains configuration for the monitoring integrated service. -type Config struct { - Namespace string - Grafana GrafanaConfig - Charts ChartsConfig - Images ImagesConfig -} - -func (c Config) Validate() error { - if c.Namespace == "" { - return errors.New("monitoring namespace is required") - } - - if err := c.Grafana.Validate(); err != nil { - return err - } - - if err := c.Charts.Operator.Validate(); err != nil { - return errors.WrapIf(err, "error during validation Prometheus operator config") - } - - if err := c.Charts.Pushgateway.Validate(); err != nil { - return errors.WrapIf(err, "error during validation Pushgateway config") - } - - if err := c.Images.Prometheus.Validate(); err != nil { - return errors.WrapIf(err, "error during validate Prometheus images config") - } - - if err := c.Images.Alertmanager.Validate(); err != nil { - return errors.WrapIf(err, "error during validate Alertmanager images config") - } - - if err := c.Images.Grafana.Validate(); err != nil { - return errors.WrapIf(err, "error during validate Grafana images config") - } - - if err := c.Images.Nodeexporter.Validate(); err != nil { - return errors.WrapIf(err, "error during validate NodeExporter images config") - } - - if err := c.Images.Kubestatemetrics.Validate(); err != nil { - return errors.WrapIf(err, "error during validate KubeStateMetrics images config") - } - - if err := c.Images.Operator.Validate(); err != nil { - return errors.WrapIf(err, "error during validate Operator images config") - } - - if err := c.Images.Pushgateway.Validate(); err != nil { - return errors.WrapIf(err, "error during validate Pushgateway images config") - } - - return nil -} - -type GrafanaConfig struct { - AdminUser string -} - -func (c GrafanaConfig) Validate() error { - if c.AdminUser == "" { - return errors.New("monitoring grafana username is required") - } - - return nil -} - -type ChartsConfig struct { - Operator ChartConfig - Pushgateway ChartConfig -} - -type ChartConfig struct { - Chart string - Version string - Values map[string]interface{} -} - -func (c ChartConfig) Validate() error { - if c.Chart == "" { - return errors.New("chart is required") - } - - if c.Version == "" { - return errors.New("chart version is required") - } - - return nil -} - -type ImagesConfig struct { - Operator ImageConfig - Prometheus ImageConfig - Alertmanager ImageConfig - Grafana ImageConfig - Kubestatemetrics ImageConfig - Nodeexporter ImageConfig - Pushgateway ImageConfig -} - -type ImageConfig struct { - Repository string - Tag string -} - -func (c ImageConfig) Validate() error { - if c.Repository == "" { - return errors.New("repository is required") - } - - if c.Tag == "" { - return errors.New("tag is required") - } - - return nil -} diff --git a/internal/integratedservices/services/monitoring/error.go b/internal/integratedservices/services/monitoring/error.go deleted file mode 100644 index fb1dd6b335..0000000000 --- a/internal/integratedservices/services/monitoring/error.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import "fmt" - -type requiredFieldError struct { - fieldName string -} - -type invalidIngressHostError struct { - hostType string -} - -type canNotDisabledError struct { - fieldName string -} - -func (e invalidIngressHostError) Error() string { - return fmt.Sprintf("invalid %s ingress host", e.hostType) -} - -func (e requiredFieldError) Error() string { - return fmt.Sprintf("%q cannot be empty", e.fieldName) -} - -func (e canNotDisabledError) Error() string { - return fmt.Sprintf("%s cannot be disabled", e.fieldName) -} diff --git a/internal/integratedservices/services/monitoring/kubernetes.go b/internal/integratedservices/services/monitoring/kubernetes.go deleted file mode 100644 index 201254719b..0000000000 --- a/internal/integratedservices/services/monitoring/kubernetes.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - - "k8s.io/apimachinery/pkg/runtime" -) - -type KubernetesService interface { - // List lists Objects on specific cluster. - List(ctx context.Context, clusterID uint, labels map[string]string, o runtime.Object) error - - // DeleteObject deletes the passed in object from the cluster - DeleteObject(ctx context.Context, clusterID uint, o runtime.Object) error -} diff --git a/internal/integratedservices/services/monitoring/manager.go b/internal/integratedservices/services/monitoring/manager.go deleted file mode 100644 index 2e04fd912f..0000000000 --- a/internal/integratedservices/services/monitoring/manager.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - "fmt" - - "emperror.dev/errors" - - "github.com/banzaicloud/pipeline/internal/cluster/endpoints" - "github.com/banzaicloud/pipeline/internal/common" - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - pkgHelm "github.com/banzaicloud/pipeline/pkg/helm" -) - -// IntegratedServiceManager implements the Monitoring integrated service manager -type IntegratedServiceManager struct { - integratedservices.PassthroughIntegratedServiceSpecPreparer - - clusterGetter integratedserviceadapter.ClusterGetter - secretStore services.SecretStore - endpointsService endpoints.EndpointService - helmService services.HelmService - config Config - logger common.Logger -} - -func MakeIntegratedServiceManager( - clusterGetter integratedserviceadapter.ClusterGetter, - secretStore services.SecretStore, - endpointsService endpoints.EndpointService, - helmService services.HelmService, - config Config, - logger common.Logger, -) IntegratedServiceManager { - return IntegratedServiceManager{ - clusterGetter: clusterGetter, - secretStore: secretStore, - endpointsService: endpointsService, - helmService: helmService, - config: config, - logger: logger, - } -} - -// Name returns the integrated service' name -func (IntegratedServiceManager) Name() string { - return integratedServiceName -} - -// GetOutput returns the Monitoring integrated service'output -func (m IntegratedServiceManager) GetOutput(ctx context.Context, clusterID uint, spec integratedservices.IntegratedServiceSpec) (integratedservices.IntegratedServiceOutput, error) { - boundSpec, err := bindIntegratedServiceSpec(spec) - if err != nil { - return nil, integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: err.Error(), - } - } - - cluster, err := m.clusterGetter.GetClusterByIDOnly(ctx, clusterID) - if err != nil { - return nil, errors.WrapIf(err, "failed to get cluster") - } - - kubeConfig, err := cluster.GetK8sConfig() - if err != nil { - return nil, errors.WrapIf(err, "failed to get K8S config") - } - - endpoints, err := m.endpointsService.List(ctx, kubeConfig, prometheusOperatorReleaseName) - if err != nil { - m.logger.Warn(fmt.Sprintf("failed to list endpoints: %s", err.Error())) - } - - operatorValues := m.config.Charts.Operator.Values - pushgatewayValues := m.config.Charts.Pushgateway.Values - - out := integratedservices.IntegratedServiceOutput{ - "grafana": m.getComponentOutput(ctx, clusterID, newGrafanaOutputHelper(kubeConfig, boundSpec), endpoints, m.config.Namespace, prometheusOperatorReleaseName, operatorValues, m.config.Images.Grafana), - "prometheus": m.getComponentOutput(ctx, clusterID, newPrometheusOutputHelper(kubeConfig, boundSpec), endpoints, m.config.Namespace, prometheusOperatorReleaseName, operatorValues, m.config.Images.Prometheus), - "alertmanager": m.getComponentOutput(ctx, clusterID, newAlertmanagerOutputHelper(kubeConfig, boundSpec), endpoints, m.config.Namespace, prometheusOperatorReleaseName, operatorValues, m.config.Images.Alertmanager), - "pushgateway": m.getComponentOutput(ctx, clusterID, newPushgatewayOutputHelper(kubeConfig, boundSpec), endpoints, m.config.Namespace, prometheusPushgatewayReleaseName, pushgatewayValues, m.config.Images.Pushgateway), - "prometheusOperator": map[string]interface{}{ - "version": m.config.Charts.Operator.Version, - }, - } - - return out, nil -} - -// ValidateSpec validates a Monitoring integrated service specification -func (IntegratedServiceManager) ValidateSpec(ctx context.Context, spec integratedservices.IntegratedServiceSpec) error { - boundSpec, err := bindIntegratedServiceSpec(spec) - if err != nil { - return integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: err.Error(), - } - } - - if err := boundSpec.Validate(); err != nil { - return integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: err.Error(), - } - } - - return nil -} - -func (m IntegratedServiceManager) getComponentOutput( - ctx context.Context, - clusterID uint, - helper outputHelper, - endpoints []*pkgHelm.EndpointItem, - pipelineSystemNamespace string, - releaseName string, - values map[string]interface{}, - config ImageConfig, -) map[string]interface{} { - out := make(map[string]interface{}) - - o := outputManager{ - outputHelper: helper, - secretStore: m.secretStore, - logger: m.logger, - } - - writeSecretID(ctx, o, clusterID, out) - writeURL(o, endpoints, releaseName, out) - // TODO (colin): put back after the values can came from config - // writeVersion(o, values, out) - out[versionKey] = config.Tag - if err := writeServiceURL(ctx, o, m.endpointsService, pipelineSystemNamespace, out); err != nil { - m.logger.Warn(fmt.Sprintf("failed to get service url: %s", err.Error())) - } - - return out -} diff --git a/internal/integratedservices/services/monitoring/manager_test.go b/internal/integratedservices/services/monitoring/manager_test.go deleted file mode 100644 index d16528a8d5..0000000000 --- a/internal/integratedservices/services/monitoring/manager_test.go +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - "testing" - - "emperror.dev/errors" - "github.com/stretchr/testify/assert" - - "github.com/banzaicloud/pipeline/internal/common/commonadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - "github.com/banzaicloud/pipeline/src/auth" - "github.com/banzaicloud/pipeline/src/secret" -) - -func TestIntegratedServiceManager_Name(t *testing.T) { - mng := MakeIntegratedServiceManager(nil, nil, nil, nil, Config{}, nil) - - assert.Equal(t, "monitoring", mng.Name()) -} - -func TestIntegratedServiceManager_GetOutput(t *testing.T) { - orgID := uint(13) - clusterID := uint(42) - clusterName := "the-cluster" - - clusterGetter := dummyClusterGetter{ - Clusters: map[uint]dummyCluster{ - clusterID: { - Name: clusterName, - OrgID: orgID, - ID: clusterID, - }, - }, - } - - orgSecretStore := dummyOrganizationalSecretStore{ - Secrets: map[uint]map[string]*secret.SecretItemResponse{ - orgID: { - grafanaSecretID: { - ID: grafanaSecretID, - Name: getGrafanaSecretName(clusterID), - Type: secrettype.Password, - Values: map[string]string{secrettype.Username: "admin", secrettype.Password: "pass"}, - Tags: []string{secret.TagBanzaiReadonly}, - Version: 1, - }, - prometheusSecretID: { - ID: prometheusSecretID, - Name: getPrometheusSecretName(clusterID), - Type: secrettype.Password, - Values: map[string]string{secrettype.Username: "admin", secrettype.Password: "pass"}, - Tags: []string{secret.TagBanzaiReadonly}, - Version: 1, - }, - }, - }, - } - - config := Config{ - Grafana: GrafanaConfig{AdminUser: "admin"}, - Charts: ChartsConfig{ - Operator: ChartConfig{ - Version: "1.0.0", - }, - }, - Images: ImagesConfig{ - Operator: ImageConfig{ - Tag: "v0.1.1", - }, - Prometheus: ImageConfig{ - Tag: "v0.1.2", - }, - Alertmanager: ImageConfig{ - Tag: "v0.1.3", - }, - Grafana: ImageConfig{ - Tag: "v0.1.4", - }, - Kubestatemetrics: ImageConfig{ - Tag: "v0.1.5", - }, - Nodeexporter: ImageConfig{ - Tag: "v0.1.6", - }, - Pushgateway: ImageConfig{ - Tag: "v0.1.7", - }, - }, - } - - secretStore := commonadapter.NewSecretStore(orgSecretStore, commonadapter.OrgIDContextExtractorFunc(auth.GetCurrentOrganizationID)) - helmService := dummyHelmService{} - endpointService := dummyEndpointService{} - logger := services.NoopLogger{} - mng := MakeIntegratedServiceManager(clusterGetter, secretStore, endpointService, helmService, config, logger) - ctx := auth.SetCurrentOrganizationID(context.Background(), orgID) - - spec := obj{ - "grafana": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "/grafana", - }, - "secretId": grafanaSecretID, - }, - "alertmanager": obj{ - "enabled": true, - "ingress": obj{ - "enabled": false, - }, - }, - "prometheus": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "/prometheus", - }, - "secretId": prometheusSecretID, - }, - } - - output, err := mng.GetOutput(ctx, clusterID, spec) - assert.NoError(t, err) - - assert.Equal(t, integratedservices.IntegratedServiceOutput{ - "grafana": obj{ - "serviceUrl": serviceUrl, - "url": grafanaURL, - "version": "v0.1.4", - }, - "prometheus": obj{ - "serviceUrl": serviceUrl, - "url": prometheusURL, - "version": "v0.1.2", - }, - "prometheusOperator": obj{ - "version": config.Charts.Operator.Version, - }, - "alertmanager": obj{ - "serviceUrl": serviceUrl, - "version": "v0.1.3", - }, - "pushgateway": obj{ - "version": "v0.1.7", - }, - }, output) -} - -func TestIntegratedServiceManager_ValidateSpec(t *testing.T) { - mng := MakeIntegratedServiceManager(nil, nil, nil, nil, Config{}, nil) - - cases := map[string]struct { - Spec integratedservices.IntegratedServiceSpec - Error interface{} - }{ - "empty spec": { - Spec: integratedservices.IntegratedServiceSpec{}, - Error: true, - }, - "valid spec": { - Spec: obj{ - "grafana": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": grafanaPath, - }, - }, - "prometheus": obj{ - "enabled": true, - "storage": obj{ - "size": 100, - "retention": "10m", - }, - "ingress": obj{ - "enabled": true, - "path": prometheusPath, - }, - }, - "exporters": obj{ - "enabled": true, - "nodeExporter": obj{ - "enabled": true, - }, - "kubeStateMetrics": obj{ - "enabled": true, - }, - }, - }, - Error: false, - }, - "Grafana path empty": { - Spec: obj{ - "grafana": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": "", - }, - }, - "prometheus": obj{ - "enabled": true, - "storage": obj{ - "size": 100, - "retention": "10m", - }, - "ingress": obj{ - "enabled": true, - "path": prometheusPath, - }, - }, - "exporters": obj{ - "enabled": true, - "nodeExporter": obj{ - "enabled": true, - }, - "kubeStateMetrics": obj{ - "enabled": true, - }, - }, - }, - Error: true, - }, - "Grafana invalid domain": { - Spec: obj{ - "grafana": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "domain": "2342#@", - "path": grafanaPath, - }, - }, - "prometheus": obj{ - "enabled": true, - "storage": obj{ - "size": 100, - "retention": "10m", - }, - "ingress": obj{ - "enabled": true, - "path": prometheusPath, - }, - }, - "exporters": obj{ - "enabled": true, - "nodeExporter": obj{ - "enabled": true, - }, - "kubeStateMetrics": obj{ - "enabled": true, - }, - }, - }, - Error: true, - }, - "disabled exporters": { - Spec: obj{ - "grafana": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": grafanaPath, - }, - }, - "prometheus": obj{ - "enabled": true, - "storage": obj{ - "size": 100, - "retention": "10m", - }, - "ingress": obj{ - "enabled": true, - "path": prometheusPath, - }, - }, - "exporters": obj{ - "enabled": false, - }, - }, - Error: true, - }, - "disabled nodeExporter": { - Spec: obj{ - "grafana": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": grafanaPath, - }, - }, - "prometheus": obj{ - "enabled": true, - "storage": obj{ - "size": 100, - "retention": "10m", - }, - "ingress": obj{ - "enabled": true, - "path": prometheusPath, - }, - }, - "exporters": obj{ - "enabled": true, - "nodeExporter": obj{ - "enabled": false, - }, - "kubeStateMetrics": obj{ - "enabled": true, - }, - }, - }, - Error: true, - }, - "disabled kubeStateMetrics": { - Spec: obj{ - "grafana": obj{ - "enabled": true, - "ingress": obj{ - "enabled": true, - "path": grafanaPath, - }, - }, - "prometheus": obj{ - "enabled": true, - "storage": obj{ - "size": 100, - "retention": "10m", - }, - "ingress": obj{ - "enabled": true, - "path": prometheusPath, - }, - }, - "exporters": obj{ - "enabled": true, - "nodeExporter": obj{ - "enabled": true, - }, - "kubeStateMetrics": obj{ - "enabled": false, - }, - }, - }, - Error: true, - }, - } - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - ctx := context.Background() - - err := mng.ValidateSpec(ctx, tc.Spec) - switch tc.Error { - case true: - assert.True(t, integratedservices.IsInputValidationError(err)) - case false, nil: - assert.NoError(t, err) - default: - assert.Equal(t, tc.Error, errors.Cause(err)) - } - }) - } -} diff --git a/internal/integratedservices/services/monitoring/migrator.go b/internal/integratedservices/services/monitoring/migrator.go deleted file mode 100644 index 728567055b..0000000000 --- a/internal/integratedservices/services/monitoring/migrator.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright © 2020 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - "fmt" - - "emperror.dev/errors" - "github.com/Masterminds/semver/v3" - apierrors "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes" -) - -type ( - K8sClientFactory func() (kubernetes.Interface, error) - Migrator func(ctx context.Context, configFactory K8sClientFactory, namespace, oldChartVersion, newChartVersion string) error -) - -func Migrate(ctx context.Context, clientFactory K8sClientFactory, namespace, oldChartVersion, newChartVersion string) error { - referenceVersion, err := semver.NewVersion("8.13.0") - if err != nil { - return errors.WrapIf(err, "invalid reference version") - } - - oldSemVer, err := semver.NewVersion(oldChartVersion) - if err != nil { - return errors.WrapIf(err, "invalid old chart version") - } - - newSemVer, err := semver.NewVersion(newChartVersion) - if err != nil { - return errors.WrapIf(err, "invalid new chart version") - } - - if oldSemVer.LessThan(referenceVersion) && newSemVer.GreaterThan(referenceVersion) { - clientset, err := clientFactory() - if err != nil { - return errors.WrapIf(err, "unable to creates kubernetes config for migration") - } - - ingresses, err := clientset.ExtensionsV1beta1().Ingresses(namespace).List( - ctx, - v1.ListOptions{ - LabelSelector: labels.SelectorFromSet(map[string]string{"release": prometheusOperatorReleaseName}).String(), - }) - if err != nil { - return errors.WrapIf(err, "unable to remove legacy ingresses") - } - for _, i := range ingresses.Items { - err = clientset.ExtensionsV1beta1().Ingresses(namespace).Delete(ctx, i.Name, v1.DeleteOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - continue - } - return errors.WrapIf(err, "unable to remove ingress") - } - } - err = clientset.AppsV1().Deployments(namespace).Delete(ctx, fmt.Sprintf("%s-grafana", prometheusOperatorReleaseName), v1.DeleteOptions{}) - if err != nil { - if !apierrors.IsNotFound(err) { - return errors.WrapIf(err, "unable to remove grafana deployment") - } - } - } - return nil -} diff --git a/internal/integratedservices/services/monitoring/migrator_test.go b/internal/integratedservices/services/monitoring/migrator_test.go deleted file mode 100644 index e1d8ace1a7..0000000000 --- a/internal/integratedservices/services/monitoring/migrator_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright © 2020 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - "testing" - - "k8s.io/api/extensions/v1beta1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" -) - -func TestMigrateIngress(t *testing.T) { - releaseNamespace := "asd" - ingressNameToRemove := "aaa" - oldVersion := "8.5.14" - newVersion := "8.13.8" - - allIngresses := []runtime.Object{ - &v1beta1.Ingress{ - ObjectMeta: v1.ObjectMeta{ - Name: ingressNameToRemove, - Namespace: releaseNamespace, - Labels: map[string]string{ - "release": prometheusOperatorReleaseName, - }, - }, - }, - &v1beta1.Ingress{ - ObjectMeta: v1.ObjectMeta{ - Name: "bbb", - Namespace: releaseNamespace, - Labels: map[string]string{ - "does": "notmatch", - }, - }, - }, - } - - t.Run("migrate", func(t *testing.T) { - clientset := fake.NewSimpleClientset(allIngresses...) - - clientsetFactory := func() (kubernetes.Interface, error) { - return clientset, nil - } - - err := Migrate(context.Background(), clientsetFactory, releaseNamespace, oldVersion, newVersion) - if err != nil { - t.Fatalf("%+v", err) - } - - ingresses, err := clientset.ExtensionsV1beta1().Ingresses(releaseNamespace).List(context.Background(), v1.ListOptions{}) - if err != nil { - t.Fatalf("%+v", err) - } - - if len(ingresses.Items) != len(allIngresses)-1 { - t.Fatalf("invalid number of ingresses left, expected %d, got %d: %+v", len(allIngresses)-1, len(ingresses.Items), ingresses.Items) - } - - for _, i := range ingresses.Items { - if i.Name == ingressNameToRemove && i.Namespace == releaseNamespace { - t.Fatal("failed to remove the single ingress expected") - } - } - }) - - t.Run("nomigrate", func(t *testing.T) { - clientset := fake.NewSimpleClientset(allIngresses...) - - clientsetFactory := func() (kubernetes.Interface, error) { - return clientset, nil - } - - err := Migrate(context.Background(), clientsetFactory, releaseNamespace, oldVersion, oldVersion) - if err != nil { - t.Fatalf("%+v", err) - } - - ingresses, err := clientset.ExtensionsV1beta1().Ingresses(releaseNamespace).List(context.Background(), v1.ListOptions{}) - if err != nil { - t.Fatalf("%+v", err) - } - - if len(ingresses.Items) != len(allIngresses) { - t.Fatalf("expected that nobody get harmed") - } - }) -} diff --git a/internal/integratedservices/services/monitoring/operator.go b/internal/integratedservices/services/monitoring/operator.go deleted file mode 100644 index b1b8032778..0000000000 --- a/internal/integratedservices/services/monitoring/operator.go +++ /dev/null @@ -1,826 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - "encoding/json" - "fmt" - - "emperror.dev/errors" - "github.com/mitchellh/copystructure" - "github.com/mitchellh/mapstructure" - "k8s.io/api/storage/v1beta1" - v1beta12 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - - "github.com/banzaicloud/pipeline/internal/common" - "github.com/banzaicloud/pipeline/internal/helm" - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - "github.com/banzaicloud/pipeline/pkg/any" - "github.com/banzaicloud/pipeline/pkg/jsonstructure" - "github.com/banzaicloud/pipeline/pkg/k8sclient" - "github.com/banzaicloud/pipeline/src/auth" - pkgCluster "github.com/banzaicloud/pipeline/src/cluster" - "github.com/banzaicloud/pipeline/src/secret" -) - -// IntegratedServiceOperator implements the Monitoring integrated service operator -type IntegratedServiceOperator struct { - clusterGetter integratedserviceadapter.ClusterGetter - clusterService integratedservices.ClusterService - helmService services.HelmService - kubernetesService KubernetesService - config Config - logger common.Logger - secretStore services.SecretStore - migrator Migrator -} - -type chartValuesManager struct { - operator IntegratedServiceOperator - clusterID uint -} - -// MakeIntegratedServiceOperator returns a Monitoring integrated service operator -func MakeIntegratedServiceOperator( - clusterGetter integratedserviceadapter.ClusterGetter, - clusterService integratedservices.ClusterService, - helmService services.HelmService, - kubernetesService KubernetesService, - config Config, - logger common.Logger, - secretStore services.SecretStore, - migrator Migrator, -) IntegratedServiceOperator { - return IntegratedServiceOperator{ - clusterGetter: clusterGetter, - clusterService: clusterService, - helmService: helmService, - kubernetesService: kubernetesService, - config: config, - logger: logger, - secretStore: secretStore, - migrator: migrator, - } -} - -// Name returns the name of the DNS integrated service -func (IntegratedServiceOperator) Name() string { - return integratedServiceName -} - -// Apply applies the provided specification to the integrated service -func (op IntegratedServiceOperator) Apply(ctx context.Context, clusterID uint, spec integratedservices.IntegratedServiceSpec) error { - if err := op.clusterService.CheckClusterReady(ctx, clusterID); err != nil { - return err - } - - ctx, err := op.ensureOrgIDInContext(ctx, clusterID) - if err != nil { - return err - } - - logger := op.logger.WithContext(ctx).WithFields(map[string]interface{}{"cluster": clusterID, "integrated service": integratedServiceName}) - - boundSpec, err := bindIntegratedServiceSpec(spec) - if err != nil { - return integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: err.Error(), - } - } - - cluster, err := op.clusterGetter.GetClusterByIDOnly(ctx, clusterID) - if err != nil { - return errors.WrapIf(err, "failed to get cluster") - } - - var grafanaSecretID string - if boundSpec.Grafana.Enabled { - // get Grafana secret from spec or generate - grafanaSecretID, err = op.getGrafanaSecret(ctx, cluster, boundSpec, logger) - if err != nil { - return errors.WrapIf(err, "failed to get Grafana secret") - } - } - - baseSecretInfoer := baseSecretInfoer{ - clusterID: clusterID, - } - - // Prometheus - var prometheusSecretName string - if boundSpec.Prometheus.Enabled && boundSpec.Prometheus.Ingress.Enabled { - // get Prometheus secret from spec or generate - manager := secretManager{ - operator: op, - cluster: cluster, - tags: []string{prometheusSecretTag}, - infoer: prometheusSecretInfoer{baseSecretInfoer: baseSecretInfoer}, - } - prometheusSecretName, err = generateAndInstallSecret(ctx, boundSpec.Prometheus.Ingress, manager, logger) - if err != nil { - return errors.WrapIf(err, "failed to setup Prometheus ingress") - } - } - - // Alertmanager - var alertmanagerSecretName string - if boundSpec.Alertmanager.Enabled && boundSpec.Alertmanager.Ingress.Enabled { - // get Alertmanager secret from spec or generate - manager := secretManager{ - operator: op, - cluster: cluster, - tags: []string{alertmanagerSecretTag}, - infoer: alertmanagerSecretInfoer{baseSecretInfoer: baseSecretInfoer}, - } - alertmanagerSecretName, err = generateAndInstallSecret(ctx, boundSpec.Alertmanager.Ingress, manager, logger) - if err != nil { - return errors.WrapIf(err, "failed to setup Alertmanager ingress") - } - } - - // install Prometheus Operator - if err := op.installPrometheusOperator(ctx, cluster, logger, boundSpec, grafanaSecretID, prometheusSecretName, alertmanagerSecretName); err != nil { - return errors.WrapIf(err, "failed to install Prometheus operator") - } - - // Pushgateway - if boundSpec.Pushgateway.Enabled { - // install Prometheus Pushgateway - if err := op.installPrometheusPushGateway(ctx, cluster, boundSpec.Pushgateway, logger); err != nil { - return errors.WrapIf(err, "failed to install Prometheus Pushgateway") - } - } - - return nil -} - -// Deactivate deactivates the cluster integrated service -func (op IntegratedServiceOperator) Deactivate(ctx context.Context, clusterID uint, spec integratedservices.IntegratedServiceSpec) error { - if err := op.clusterService.CheckClusterReady(ctx, clusterID); err != nil { - return err - } - - ctx, err := op.ensureOrgIDInContext(ctx, clusterID) - if err != nil { - return err - } - - boundSpec, err := bindIntegratedServiceSpec(spec) - if err != nil { - return integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: err.Error(), - } - } - - if boundSpec.Grafana.SecretId == "" { - // Grafana secret generated in activation flow, delete it - if err := op.deleteGrafanaSecret(ctx, clusterID); err != nil && !isSecretNotFoundError(err) { - return errors.WrapIf(err, "failed to delete Grafana secret") - } - } - - if boundSpec.Prometheus.Ingress.SecretID == "" { - // Prometheus secret generated in activation flow, delete it - if err := op.deletePrometheusSecret(ctx, clusterID); err != nil && !isSecretNotFoundError(err) { - return errors.WrapIf(err, "failed to delete Prometheus secret") - } - } - - // delete prometheus operator deployment - if err := op.helmService.DeleteDeployment(ctx, clusterID, prometheusOperatorReleaseName, op.config.Namespace); err != nil { - return errors.WrapIfWithDetails(err, "failed to delete deployment", "release", prometheusOperatorReleaseName) - } - - // delete prometheus pushgateway deployment - if err := op.helmService.DeleteDeployment(ctx, clusterID, prometheusPushgatewayReleaseName, op.config.Namespace); err != nil { - return errors.WrapIfWithDetails(err, "failed to delete deployment", "release", prometheusPushgatewayReleaseName) - } - - // delete custom resources - if err := op.cleanupCRDs(ctx, clusterID); err != nil { - op.logger.Warn("failed to delete CRDs", map[string]interface{}{"failures": err}) - } - - return nil -} - -func (op IntegratedServiceOperator) installPrometheusPushGateway( - ctx context.Context, - cluster integratedserviceadapter.Cluster, - spec pushgatewaySpec, - logger common.Logger, -) error { - chartValues := &prometheusPushgatewayValues{ - Image: imageValues{ - Repository: op.config.Images.Pushgateway.Repository, - Tag: op.config.Images.Pushgateway.Tag, - }, - ServiceMonitor: serviceMonitorValues{ - Enabled: true, - Namespace: op.config.Namespace, - }, - } - - pushgatewayConfigValues, err := copystructure.Copy(op.config.Charts.Pushgateway.Values) - if err != nil { - return errors.WrapIf(err, "failed to copy pushgateway values") - } - valuesBytes, err := mergeOperatorValuesWithConfig(*chartValues, pushgatewayConfigValues) - if err != nil { - return errors.WrapIf(err, "failed to merge pushgateway values with config") - } - - return op.helmService.ApplyDeployment( - ctx, - cluster.GetID(), - op.config.Namespace, - op.config.Charts.Pushgateway.Chart, - prometheusPushgatewayReleaseName, - valuesBytes, - op.config.Charts.Pushgateway.Version, - ) -} - -func (op IntegratedServiceOperator) installPrometheusOperator( - ctx context.Context, - cluster integratedserviceadapter.Cluster, - logger common.Logger, - spec integratedServiceSpec, - grafanaSecretID string, - prometheusSecretName string, - alertmanagerSecretName string, -) error { - var grafanaUser string - var grafanaPass string - if spec.Grafana.Enabled { - grafanaSecret, err := op.secretStore.GetSecretValues(ctx, grafanaSecretID) - if err != nil { - return errors.WrapIf(err, "failed to get Grafana secret") - } - grafanaUser = grafanaSecret[secrettype.Username] - grafanaPass = grafanaSecret[secrettype.Password] - } - - valuesManager := chartValuesManager{ - operator: op, - clusterID: cluster.GetID(), - } - - alertmanagerValues, err := valuesManager.generateAlertmanagerChartValues(ctx, spec.Alertmanager, alertmanagerSecretName, op.config.Images.Alertmanager) - if err != nil { - return errors.WrapIf(err, "failed to generate Alertmanager chart values") - } - - // create chart values - chartValues := &prometheusOperatorValues{ - PrometheusOperator: operatorSpecValues{ - Image: imageValues{ - Repository: op.config.Images.Operator.Repository, - Tag: op.config.Images.Operator.Tag, - }, - CleanupCustomResource: false, - CreateCustomResource: true, - }, - Grafana: valuesManager.generateGrafanaChartValues(spec.Grafana, grafanaUser, grafanaPass, op.config.Images.Grafana), - Alertmanager: alertmanagerValues, - Prometheus: valuesManager.generatePrometheusChartValues(ctx, spec.Prometheus, prometheusSecretName, op.config.Images.Prometheus), - } - - // todo consider disabling cleanup in favor of installing crds from the chart's crds folder, but will need to take care of upgrades in that case - chartValues.PrometheusOperator.CreateCustomResource = false - - if spec.Exporters.Enabled { - chartValues.KubeStateMetrics = valuesManager.generateKubeStateMetricsChartValues(spec.Exporters.KubeStateMetrics) - if spec.Exporters.KubeStateMetrics.Enabled { - chartValues.KsmValues = &ksmValues{Image: imageValues{ - Repository: op.config.Images.Kubestatemetrics.Repository, - Tag: op.config.Images.Kubestatemetrics.Tag, - }} - } - - chartValues.NodeExporter = valuesManager.generateNodeExporterChartValues(spec.Exporters.NodeExporter) - if spec.Exporters.NodeExporter.Enabled { - chartValues.NeValues = &neValues{Image: imageValues{ - Repository: op.config.Images.Nodeexporter.Repository, - Tag: op.config.Images.Nodeexporter.Tag, - }} - } - } - - operatorConfigValues, err := copystructure.Copy(op.config.Charts.Operator.Values) - if err != nil { - return errors.WrapIf(err, "failed to copy operator values") - } - valuesBytes, err := mergeOperatorValuesWithConfig(*chartValues, operatorConfigValues) - if err != nil { - return errors.WrapIf(err, "failed to merge operator values with config") - } - - if op.migrator != nil { - release, err := op.helmService.GetDeployment(ctx, cluster.GetID(), prometheusOperatorReleaseName, op.config.Namespace) - if err != nil { - if !helm.ErrReleaseNotFound(err) { - return err - } - } else { - k8sClientFactory := func() (kubernetes.Interface, error) { - kubeConfig, err := cluster.GetK8sConfig() - if err != nil { - return nil, err - } - client, err := k8sclient.NewClientFromKubeConfig(kubeConfig) - if err != nil { - return nil, err - } - return client, nil - } - - err = op.migrator(ctx, k8sClientFactory, op.config.Namespace, release.ChartVersion, op.config.Charts.Operator.Version) - if err != nil { - return err - } - } - } - - return op.helmService.ApplyDeployment( - ctx, - cluster.GetID(), - op.config.Namespace, - op.config.Charts.Operator.Chart, - prometheusOperatorReleaseName, - valuesBytes, - op.config.Charts.Operator.Version, - ) -} - -func mergeOperatorValuesWithConfig(chartValues interface{}, configValues interface{}) ([]byte, error) { - out, err := jsonstructure.Encode(chartValues) - if err != nil { - return nil, errors.WrapIf(err, "failed to encode chart values") - } - - result, err := any.Merge(configValues, out, jsonstructure.DefaultMergeOptions()) - if err != nil { - return nil, errors.WrapIf(err, "failed to merge values") - } - - return json.Marshal(result) -} - -func (op IntegratedServiceOperator) generateGrafanaSecret( - ctx context.Context, - cluster integratedserviceadapter.Cluster, - logger common.Logger, -) (string, error) { - clusterNameSecretTag := getClusterNameSecretTag(cluster.GetName()) - clusterUIDSecretTag := getClusterUIDSecretTag(cluster.GetUID()) - releaseSecretTag := getReleaseSecretTag() - - // Generating Grafana credentials - username := op.config.Grafana.AdminUser - password, err := secret.RandomString("randAlphaNum", 12) - if err != nil { - return "", errors.WrapIf(err, "failed to generate Grafana admin user password") - } - - grafanaSecretRequest := secret.CreateSecretRequest{ - Name: getGrafanaSecretName(cluster.GetID()), - Type: secrettype.PasswordSecretType, - Values: map[string]string{ - secrettype.Username: username, - secrettype.Password: password, - }, - Tags: []string{ - clusterNameSecretTag, - clusterUIDSecretTag, - releaseSecretTag, - grafanaSecretTag, - }, - } - grafanaSecretID, err := secret.Store.CreateOrUpdate(cluster.GetOrganizationId(), &grafanaSecretRequest) - if err != nil { - return "", errors.WrapIf(err, "error store prometheus secret") - } - logger.Debug("grafana secret stored") - - return grafanaSecretID, nil -} - -func (op IntegratedServiceOperator) deleteGrafanaSecret(ctx context.Context, clusterID uint) error { - secretID, err := op.secretStore.GetIDByName(ctx, getGrafanaSecretName(clusterID)) - if err != nil { - return errors.WrapIf(err, "failed to get Grafana secret") - } - return op.secretStore.Delete(ctx, secretID) -} - -func (op IntegratedServiceOperator) deletePrometheusSecret(ctx context.Context, clusterID uint) error { - secretID, err := op.secretStore.GetIDByName(ctx, getPrometheusSecretName(clusterID)) - if err != nil { - return errors.WrapIf(err, "failed to get Prometheus secret") - } - return op.secretStore.Delete(ctx, secretID) -} - -func (op IntegratedServiceOperator) installSecret(ctx context.Context, clusterID uint, secretName string, secretRequest pkgCluster.InstallSecretRequest) (string, error) { - cl, err := op.clusterGetter.GetClusterByIDOnly(ctx, clusterID) - if err != nil { - return "", errors.WrapIfWithDetails(err, "failed to get cluster", "clusterID", clusterID) - } - - k8sSecName, err := pkgCluster.InstallSecret(cl, secretName, secretRequest) - if err != nil { - return "", errors.WrapIfWithDetails(err, "failed to install secret to the cluster", "clusterID", clusterID) - } - - return k8sSecName, nil -} - -func (op IntegratedServiceOperator) getGrafanaSecret( - ctx context.Context, - cluster integratedserviceadapter.Cluster, - spec integratedServiceSpec, - logger common.Logger, -) (string, error) { - secretID := spec.Grafana.SecretId - if secretID == "" { - // check Grafana secret exists - existingSecretID, err := op.secretStore.GetIDByName(ctx, getGrafanaSecretName(cluster.GetID())) - if existingSecretID != "" { - logger.Debug("Grafana secret already exists") - return existingSecretID, nil - } else if isSecretNotFoundError(err) { - // generate and store Grafana secret - secretID, err = op.generateGrafanaSecret(ctx, cluster, logger) - if err != nil { - return "", errors.WrapIf(err, "failed to create Grafana secret") - } - } else { - return "", errors.WrapIf(err, "error during getting Grafana secret") - } - } - - return secretID, nil -} - -func (op IntegratedServiceOperator) ensureOrgIDInContext(ctx context.Context, clusterID uint) (context.Context, error) { - if _, ok := auth.GetCurrentOrganizationID(ctx); !ok { - cluster, err := op.clusterGetter.GetClusterByIDOnly(ctx, clusterID) - if err != nil { - return ctx, errors.WrapIf(err, "failed to get cluster by ID") - } - ctx = auth.SetCurrentOrganizationID(ctx, cluster.GetOrganizationId()) - } - return ctx, nil -} - -func (op IntegratedServiceOperator) generateAlertManagerProvidersConfig(ctx context.Context, spec map[string]interface{}) (*configValues, error) { - var err error - var hasProvider bool - - // generate Slack configs - var slackConfigs []slackConfigValues - if slackProv, ok := spec[alertmanagerProviderSlack]; ok { - var slack slackSpec - if err := mapstructure.Decode(slackProv, &slack); err != nil { - return nil, errors.WrapIf(err, "failed to bind Slack config") - } - if slack.Enabled { - hasProvider = true - slackConfigs, err = op.generateSlackConfig(ctx, slack) - if err != nil { - return nil, errors.WrapIf(err, "failed to generate Slack config") - } - } - } - - // generate PagerDuty configs - var pageDutyConfigs []pagerdutyConfigValues - if pdProv, ok := spec[alertmanagerProviderPagerDuty]; ok { - var pd pagerDutySpec - if err := mapstructure.Decode(pdProv, &pd); err != nil { - return nil, errors.WrapIf(err, "failed to bind PagerDuty config") - } - if pd.Enabled { - hasProvider = true - pageDutyConfigs, err = op.generatePagerdutyConfig(ctx, pd) - if err != nil { - return nil, errors.WrapIf(err, "failed to generate PagerDuty config") - } - } - } - - receiverName := alertManagerNullReceiverName - if hasProvider { - receiverName = alertManagerProviderConfigName - } - result := &configValues{ - Receivers: []receiverItemValues{ - { - Name: receiverName, - }, - }, - Route: routeValues{ - Receiver: receiverName, - Routes: []interface{}{}, - }, - } - - if slackConfigs != nil { - result.Receivers[0].SlackConfigs = slackConfigs - } - - if pageDutyConfigs != nil { - result.Receivers[0].PagerdutyConfigs = pageDutyConfigs - } - - return result, nil -} - -func (op IntegratedServiceOperator) generateSlackConfig(ctx context.Context, config slackSpec) ([]slackConfigValues, error) { - if config.Enabled { - slackSecret, err := op.secretStore.GetSecretValues(ctx, config.SecretID) - if err != nil { - return nil, errors.WrapIf(err, "failed to get Slack secret") - } - - return []slackConfigValues{ - { - ApiUrl: slackSecret[secrettype.SlackApiUrl], - Channel: config.Channel, - SendResolved: config.SendResolved, - }, - }, nil - } - - return nil, nil -} - -func (op IntegratedServiceOperator) generatePagerdutyConfig(ctx context.Context, config pagerDutySpec) ([]pagerdutyConfigValues, error) { - if config.Enabled { - pdSecret, err := op.secretStore.GetSecretValues(ctx, config.SecretID) - if err != nil { - return nil, errors.WrapIf(err, "failed to get PagerDuty secret") - } - - pdConfig := pagerdutyConfigValues{ - Url: config.URL, - SendResolved: config.SendResolved, - } - - integrationKey := pdSecret[secrettype.PagerDutyIntegrationKey] - if config.IntegrationType == pagerDutyIntegrationEventApiV2 { - pdConfig.RoutingKey = integrationKey - } else { - pdConfig.ServiceKey = integrationKey - } - - return []pagerdutyConfigValues{pdConfig}, nil - } - - return nil, nil -} - -func isSecretNotFoundError(err error) bool { - errCause := errors.Cause(err) - if errCause == secret.ErrSecretNotExists { - return true - } - return false -} - -func (m chartValuesManager) generateGrafanaChartValues( - spec grafanaSpec, - username string, - password string, - config ImageConfig, -) *grafanaValues { - if spec.Enabled { - return &grafanaValues{ - baseValues: baseValues{ - Enabled: spec.Enabled, - Ingress: ingressValues{ - Enabled: spec.Ingress.Enabled, - Hosts: []string{spec.Ingress.Domain}, - Path: spec.Ingress.Path, - }, - }, - AdminUser: username, - AdminPassword: password, - GrafanaIni: grafanaIniValues{Server: grafanaIniServerValues{ - RootUrl: fmt.Sprintf("http://0.0.0.0:3000%s/", spec.Ingress.Path), - ServeFromSubPath: true, - }}, - DefaultDashboardsEnabled: spec.Dashboards, - Image: imageValues{ - Repository: config.Repository, - Tag: config.Tag, - }, - Persistence: persistenceValues{ - Enabled: true, - }, - Sidecar: sidecar{ - Datasources: datasources{ - Enabled: true, - Label: "grafana_datasource", - SearchNamespace: "ALL", - }, - }, - } - } - - return &grafanaValues{ - baseValues: baseValues{ - Enabled: false, - }, - } -} - -func (m chartValuesManager) generateAlertmanagerChartValues( - ctx context.Context, - spec alertmanagerSpec, - secretName string, - config ImageConfig, -) (*alertmanagerValues, error) { - if spec.Enabled { - var annotations map[string]interface{} - if spec.Ingress.Enabled { - annotations = generateAnnotations(secretName) - } - - alertmanagerConfig, err := m.operator.generateAlertManagerProvidersConfig(ctx, spec.Provider) - if err != nil { - return nil, errors.WrapIf(err, "failed to generate Alertmanager Provider config") - } - - return &alertmanagerValues{ - baseValues: baseValues{ - Enabled: spec.Enabled, - Ingress: ingressValues{ - Enabled: spec.Ingress.Enabled, - Hosts: []string{spec.Ingress.Domain}, - Paths: []string{spec.Ingress.Path}, - Annotations: annotations, - }, - }, - Spec: baseSpecValues{ - RoutePrefix: spec.Ingress.Path, - Image: imageValues{ - Repository: config.Repository, - Tag: config.Tag, - }, - }, - Config: alertmanagerConfig, - }, nil - } - - return &alertmanagerValues{ - baseValues: baseValues{ - Enabled: false, - }, - }, nil -} - -func (m chartValuesManager) generatePrometheusChartValues( - ctx context.Context, - spec prometheusSpec, - secretName string, - config ImageConfig, -) *prometheusValues { - if spec.Enabled { - defaultStorageClassName := spec.Storage.Class - if defaultStorageClassName == "" { - var err error - defaultStorageClassName, err = m.operator.getDefaultStorageClassName(ctx, m.clusterID) - if err != nil { - m.operator.logger.Warn("failed to get default storage class") - } - } - - var annotations map[string]interface{} - if spec.Ingress.Enabled { - annotations = generateAnnotations(secretName) - } - - return &prometheusValues{ - baseValues: baseValues{ - Enabled: spec.Enabled, - Ingress: ingressValues{ - Enabled: spec.Ingress.Enabled, - Hosts: []string{spec.Ingress.Domain}, - Paths: []string{spec.Ingress.Path}, - Annotations: annotations, - }, - }, - Spec: PrometheusSpecValues{ - baseSpecValues: baseSpecValues{ - RoutePrefix: spec.Ingress.Path, - Image: imageValues{ - Repository: config.Repository, - Tag: config.Tag, - }, - }, - RetentionSize: fmt.Sprintf("%.2fGiB", float64(spec.Storage.Size)*0.95), - Retention: spec.Storage.Retention, - StorageSpec: map[string]interface{}{ - "volumeClaimTemplate": map[string]interface{}{ - "spec": map[string]interface{}{ - "storageClassName": defaultStorageClassName, - "accessModes": []string{"ReadWriteOnce"}, - "resources": map[string]interface{}{ - "requests": map[string]interface{}{ - "storage": fmt.Sprintf("%dGi", spec.Storage.Size), - }, - }, - }, - }, - }, - ServiceMonitorSelectorNilUsesHelmValues: false, - }, - } - } - - return &prometheusValues{ - baseValues: baseValues{ - Enabled: false, - }, - } -} - -func (m chartValuesManager) generateKubeStateMetricsChartValues(spec exporterBaseSpec) kubeStateMetricsValues { - return kubeStateMetricsValues{ - Enabled: spec.Enabled, - } -} - -func (m chartValuesManager) generateNodeExporterChartValues(spec exporterBaseSpec) nodeExporterValues { - return nodeExporterValues{ - Enabled: spec.Enabled, - } -} - -func (op IntegratedServiceOperator) getDefaultStorageClassName(ctx context.Context, clusterID uint) (string, error) { - var storageClass v1beta1.StorageClassList - if err := op.kubernetesService.List(ctx, clusterID, nil, &storageClass); err != nil { - return "", errors.WrapIf(err, "failed to list storage classes") - } - - var defaultStorageClassName string - for _, sc := range storageClass.Items { - for key, value := range sc.Annotations { - if key == "storageclass.kubernetes.io/is-default-class" && value == "true" { - defaultStorageClassName = sc.ObjectMeta.Name - } - } - } - return defaultStorageClassName, nil -} - -// cleanupCRDs deletes CRDs after the release is deleted -func (op IntegratedServiceOperator) cleanupCRDs(ctx context.Context, clusterID uint) error { - // list with the monitoring related CRDs - crdNames := []string{ - "alertmanagerconfigs.monitoring.coreos.com", - "alertmanagers.monitoring.coreos.com", - "podmonitors.monitoring.coreos.com", - "probes.monitoring.coreos.com", - "prometheuses.monitoring.coreos.com", - "prometheusrules.monitoring.coreos.com", - "servicemonitors.monitoring.coreos.com", - "thanosrulers.monitoring.coreos.com", - } - - var failures error - for _, crdName := range crdNames { - crd := v1beta12.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: crdName, - }, - } - op.logger.Debug("deleting CRD", map[string]interface{}{"name": crdName}) - if err := op.kubernetesService.DeleteObject(ctx, clusterID, &crd); err != nil { - failures = errors.Append(failures, err) - } - } - return failures -} diff --git a/internal/integratedservices/services/monitoring/operator_secret.go b/internal/integratedservices/services/monitoring/operator_secret.go deleted file mode 100644 index 6cc769f662..0000000000 --- a/internal/integratedservices/services/monitoring/operator_secret.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - "fmt" - - "emperror.dev/errors" - - "github.com/banzaicloud/pipeline/internal/common" - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - pkgCluster "github.com/banzaicloud/pipeline/src/cluster" - "github.com/banzaicloud/pipeline/src/secret" -) - -type secretManager struct { - operator IntegratedServiceOperator - cluster integratedserviceadapter.Cluster - tags []string - infoer secretComponentInfoer -} - -type secretComponentInfoer interface { - name() string - generatedSecretName() string -} - -func (m secretManager) generateHTPasswordSecret(ctx context.Context) error { - clusterNameSecretTag := getClusterNameSecretTag(m.cluster.GetName()) - clusterUIDSecretTag := getClusterUIDSecretTag(m.cluster.GetUID()) - releaseSecretTag := getReleaseSecretTag() - - secretTags := []string{ - clusterNameSecretTag, - clusterUIDSecretTag, - releaseSecretTag, - integratedServiceSecretTag, - } - - secretTags = append(secretTags, m.tags...) - - adminPass, err := secret.RandomString("randAlphaNum", 12) - if err != nil { - return errors.WrapIf(err, fmt.Sprintf("%s password generation failed", m.infoer.name())) - } - - secretRequest := &secret.CreateSecretRequest{ - Name: m.infoer.generatedSecretName(), - Type: secrettype.HtpasswdSecretType, - Values: map[string]string{ - secrettype.Username: generatedSecretUsername, - secrettype.Password: adminPass, - }, - Tags: secretTags, - } - _, err = secret.Store.CreateOrUpdate(m.cluster.GetOrganizationId(), secretRequest) - if err != nil { - return errors.WrapIf(err, fmt.Sprintf("failed to store %s secret", m.infoer.name())) - } - - return nil -} - -func (m secretManager) getComponentSecret( - ctx context.Context, - ingress ingressSpecWithSecret, - logger common.Logger, -) (string, error) { - var secretName string - if ingress.SecretID == "" { - // get secret by name, this necessary in case of integrated service update - secretName = m.infoer.generatedSecretName() - existingSecretID, err := m.operator.secretStore.GetIDByName(ctx, secretName) - if existingSecretID != "" { - logger.Debug(fmt.Sprintf("%s secret already exists", m.infoer.name())) - return secretName, nil - } else if isSecretNotFoundError(err) { - // generate and store secret - err = m.generateHTPasswordSecret(ctx) - if err != nil { - return "", errors.WrapIf(err, fmt.Sprintf("failed to generate %s secret", m.infoer.name())) - } - } else { - return "", errors.WrapIf(err, fmt.Sprintf("error during getting %s secret", m.infoer.name())) - } - } else { - var err error - secretName, err = m.operator.secretStore.GetNameByID(ctx, ingress.SecretID) - if err != nil { - return "", errors.WrapIfWithDetails(err, "failed to get secret", - "secretID", ingress.SecretID, "component", m.infoer.name()) - } - } - return secretName, nil -} - -func (m secretManager) installSecret(ctx context.Context, clusterID uint, secretName string) error { - installSecretRequest := pkgCluster.InstallSecretRequest{ - SourceSecretName: secretName, - Namespace: m.operator.config.Namespace, - Spec: map[string]pkgCluster.InstallSecretRequestSpecItem{ - "auth": {Source: secrettype.HtpasswdFile}, - }, - Update: true, - } - - if _, err := m.operator.installSecret(ctx, clusterID, secretName, installSecretRequest); err != nil { - return errors.WrapIfWithDetails(err, fmt.Sprintf("failed to install %s secret to cluster", m.infoer.name()), "clusterID", clusterID) - } - - return nil -} - -func generateAndInstallSecret( - ctx context.Context, - ingressSpec ingressSpecWithSecret, - manager secretManager, - logger common.Logger, -) (string, error) { - var secretName string - var err error - if ingressSpec.Enabled { - // get secret from spec or generate - secretName, err = manager.getComponentSecret(ctx, ingressSpec, logger) - if err != nil { - return "", errors.WrapIfWithDetails(err, "failed to get secret", - "component", manager.infoer.name()) - } - - // install secret - if err := manager.installSecret(ctx, manager.cluster.GetID(), secretName); err != nil { - return "", errors.WrapIfWithDetails(err, "failed to install secret to cluster", - "clusterID", manager.cluster.GetID(), "component", manager.infoer.name()) - } - } - return secretName, err -} diff --git a/internal/integratedservices/services/monitoring/operator_test.go b/internal/integratedservices/services/monitoring/operator_test.go deleted file mode 100644 index df3b3f6ade..0000000000 --- a/internal/integratedservices/services/monitoring/operator_test.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/banzaicloud/pipeline/internal/common/commonadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/internal/integratedservices/integratedserviceadapter" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - "github.com/banzaicloud/pipeline/internal/secret/secrettype" - pkgCluster "github.com/banzaicloud/pipeline/pkg/cluster" - "github.com/banzaicloud/pipeline/src/auth" - "github.com/banzaicloud/pipeline/src/secret" -) - -func TestIntegratedServiceOperator_Name(t *testing.T) { - op := MakeIntegratedServiceOperator(nil, nil, nil, nil, Config{}, nil, nil, nil) - - assert.Equal(t, "monitoring", op.Name()) -} - -func TestIntegratedServiceOperator_Apply(t *testing.T) { - clusterID := uint(42) - orgID := uint(13) - - clusterGetter := dummyClusterGetter{ - Clusters: map[uint]dummyCluster{}, - } - clusterService := integratedserviceadapter.NewClusterService(clusterGetter) - helmService := dummyHelmService{} - - orgSecretStore := dummyOrganizationalSecretStore{ - Secrets: map[uint]map[string]*secret.SecretItemResponse{ - orgID: { - grafanaSecretID: { - ID: grafanaSecretID, - Name: getGrafanaSecretName(clusterID), - Type: secrettype.Password, - Values: map[string]string{secrettype.Username: "admin", secrettype.Password: "pass"}, - Tags: []string{secret.TagBanzaiReadonly}, - Version: 1, - }, - prometheusSecretID: { - ID: prometheusSecretID, - Name: getPrometheusSecretName(clusterID), - Type: secrettype.Password, - Values: map[string]string{secrettype.Username: "admin", secrettype.Password: "pass"}, - Tags: []string{secret.TagBanzaiReadonly}, - Version: 1, - }, - }, - }, - } - - logger := services.NoopLogger{} - secretStore := commonadapter.NewSecretStore(orgSecretStore, commonadapter.OrgIDContextExtractorFunc(auth.GetCurrentOrganizationID)) - kubernetesService := dummyKubernetesService{} - op := MakeIntegratedServiceOperator(clusterGetter, clusterService, helmService, &kubernetesService, Config{ - Charts: ChartsConfig{ - Operator: ChartConfig{ - Values: map[string]interface{}{}, - }, - Pushgateway: ChartConfig{ - Values: map[string]interface{}{}, - }, - }, - }, logger, secretStore, nil) - - cases := map[string]struct { - Spec integratedservices.IntegratedServiceSpec - Cluster dummyCluster - Error interface{} - }{ - "cluster not ready": { - Spec: integratedservices.IntegratedServiceSpec{}, - Cluster: dummyCluster{ - OrgID: orgID, - Status: pkgCluster.Creating, - ID: clusterID, - }, - Error: integratedservices.ClusterIsNotReadyError{ - ClusterID: clusterID, - }, - }, - "Enabled Grafana and Alertmanager": { - Spec: integratedservices.IntegratedServiceSpec{ - "grafana": obj{ - "enabled": true, - "public": obj{ - "enabled": true, - "path": "/grafana", - }, - "secretId": grafanaSecretID, - }, - "alertmanager": obj{ - "enabled": true, - "public": obj{ - "enabled": true, - "path": "/alertmanager", - }, - }, - }, - Cluster: dummyCluster{ - OrgID: orgID, - Status: pkgCluster.Running, - ID: clusterID, - }, - Error: false, - }, - } - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - clusterGetter.Clusters[clusterID] = tc.Cluster - - ctx := auth.SetCurrentOrganizationID(context.Background(), orgID) - - err := op.Apply(ctx, clusterID, tc.Spec) - switch tc.Error { - case nil, false: - assert.NoError(t, err) - case true: - assert.Error(t, err) - default: - assert.Equal(t, tc.Error, err) - } - }) - } -} - -func TestIntegratedServiceOperator_Deactivate(t *testing.T) { - clusterID := uint(42) - orgID := uint(13) - - clusterGetter := dummyClusterGetter{ - Clusters: map[uint]dummyCluster{ - clusterID: { - Status: pkgCluster.Running, - ID: clusterID, - }, - }, - } - clusterService := integratedserviceadapter.NewClusterService(clusterGetter) - helmService := dummyHelmService{} - orgSecretStore := dummyOrganizationalSecretStore{ - Secrets: map[uint]map[string]*secret.SecretItemResponse{ - orgID: nil, - }, - } - secretStore := commonadapter.NewSecretStore(orgSecretStore, commonadapter.OrgIDContextExtractorFunc(auth.GetCurrentOrganizationID)) - logger := services.NoopLogger{} - kubernetesService := dummyKubernetesService{} - op := MakeIntegratedServiceOperator(clusterGetter, clusterService, helmService, &kubernetesService, Config{}, logger, secretStore, nil) - - ctx := context.Background() - - _ = op.Deactivate(ctx, clusterID, nil) -} diff --git a/internal/integratedservices/services/monitoring/output.go b/internal/integratedservices/services/monitoring/output.go deleted file mode 100644 index 943009ebf9..0000000000 --- a/internal/integratedservices/services/monitoring/output.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "context" - "fmt" - - "emperror.dev/errors" - - "github.com/banzaicloud/pipeline/internal/cluster/endpoints" - "github.com/banzaicloud/pipeline/internal/common" - "github.com/banzaicloud/pipeline/internal/integratedservices/services" - pkgHelm "github.com/banzaicloud/pipeline/pkg/helm" -) - -const ( - urlKey = "url" - secretIDKey = "secretId" - versionKey = "version" - serviceURLKey = "serviceUrl" -) - -type baseOutput struct { - ingress baseIngressSpec - secretID string - enabled bool - k8sConfig []byte -} - -func (o baseOutput) getSecretID() string { - return o.secretID -} - -func (o baseOutput) isEnabled() bool { - return o.enabled -} - -func (o baseOutput) getIngress() baseIngressSpec { - return o.ingress -} - -func (o baseOutput) getK8SConfig() []byte { - return o.k8sConfig -} - -type outputHelper interface { - getOutputType() string - getDeploymentValueParentKey() string - getTopLevelDeploymentKey() string - getGeneratedSecretName(clusterID uint) string - getIngress() baseIngressSpec - isEnabled() bool - getSecretID() string - getServiceName() string - getK8SConfig() []byte -} - -type outputManager struct { - outputHelper - secretStore services.SecretStore - logger common.Logger -} - -//func writeVersion(m outputManager, deploymentValues map[string]interface{}, output map[string]interface{}) { -// if m.isEnabled() && deploymentValues != nil { -// var ok = true -// if m.getTopLevelDeploymentKey() != "" { -// deploymentValues, ok = deploymentValues[m.getTopLevelDeploymentKey()].(map[string]interface{}) -// } -// if ok { -// output[versionKey] = m.getVersionFromValues(deploymentValues) -// } -// } -//} - -func writeURL(m outputManager, endpoints []*pkgHelm.EndpointItem, releaseName string, output map[string]interface{}) { - if m.isEnabled() { - ingress := m.getIngress() - if ingress.Enabled && endpoints != nil { - output[urlKey] = getEndpointUrl(endpoints, ingress.Path, releaseName) - } - } -} - -func writeServiceURL(ctx context.Context, m outputManager, service endpoints.EndpointService, pipelineSystemNamespace string, output map[string]interface{}) error { - if m.isEnabled() { - url, err := service.GetServiceURL(ctx, m.getK8SConfig(), m.getServiceName(), pipelineSystemNamespace) - if err != nil { - return errors.WrapIf(err, "failed to get service") - } - output[serviceURLKey] = url - } - - return nil -} - -func writeSecretID(ctx context.Context, m outputManager, clusterID uint, output map[string]interface{}) { - if m.isEnabled() { - generatedSecretName := m.getGeneratedSecretName(clusterID) - if m.getSecretID() == "" && generatedSecretName != "" { - secretID, err := m.secretStore.GetIDByName(ctx, generatedSecretName) - if err != nil { - m.logger.Warn(fmt.Sprintf("failed to get generated %s secret", m.getOutputType())) - return - } - - output[secretIDKey] = secretID - } - } -} - -func (m *outputManager) getVersionFromValues(values map[string]interface{}) string { - specValues := values - parentKey := m.getDeploymentValueParentKey() - ok := true - if parentKey != "" { - specValues, ok = values[parentKey].(map[string]interface{}) - } - if ok { - if image, ok := specValues["image"].(map[string]interface{}); ok { - return image["tag"].(string) - } - } - return "" -} - -func getEndpointUrl(endpoints []*pkgHelm.EndpointItem, path, releaseName string) string { - for _, ep := range endpoints { - for _, url := range ep.EndPointURLs { - if url.Path == path && url.ReleaseName == releaseName { - return url.URL - } - } - } - return "" -} diff --git a/internal/integratedservices/services/monitoring/output_alertmanager.go b/internal/integratedservices/services/monitoring/output_alertmanager.go deleted file mode 100644 index d45e626880..0000000000 --- a/internal/integratedservices/services/monitoring/output_alertmanager.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -type outputAlertmanager struct { - baseOutput -} - -func newAlertmanagerOutputHelper( - kubeConfig []byte, - spec integratedServiceSpec, -) outputAlertmanager { - return outputAlertmanager{ - baseOutput: baseOutput{ - ingress: spec.Alertmanager.Ingress.baseIngressSpec, - secretID: spec.Alertmanager.Ingress.SecretID, - enabled: spec.Alertmanager.Enabled, - k8sConfig: kubeConfig, - }, - } -} - -func (outputAlertmanager) getOutputType() string { - return "Alertmanager" -} - -func (outputAlertmanager) getTopLevelDeploymentKey() string { - return "alertmanager" -} - -func (outputAlertmanager) getDeploymentValueParentKey() string { - return "alertmanagerSpec" -} - -func (outputAlertmanager) getGeneratedSecretName(clusterID uint) string { - return getAlertmanagerSecretName(clusterID) -} - -func (outputAlertmanager) getServiceName() string { - return "monitor-prometheus-operato-alertmanager" -} diff --git a/internal/integratedservices/services/monitoring/output_grafana.go b/internal/integratedservices/services/monitoring/output_grafana.go deleted file mode 100644 index 21173bb134..0000000000 --- a/internal/integratedservices/services/monitoring/output_grafana.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -type outputGrafana struct { - baseOutput -} - -func newGrafanaOutputHelper( - k8sConfig []byte, - spec integratedServiceSpec, -) outputGrafana { - return outputGrafana{ - baseOutput: baseOutput{ - ingress: spec.Grafana.Ingress, - secretID: spec.Grafana.SecretId, - enabled: spec.Grafana.Enabled, - k8sConfig: k8sConfig, - }, - } -} - -func (outputGrafana) getOutputType() string { - return "Grafana" -} - -func (outputGrafana) getTopLevelDeploymentKey() string { - return "" -} - -func (outputGrafana) getDeploymentValueParentKey() string { - return "grafana" -} - -func (outputGrafana) getGeneratedSecretName(clusterID uint) string { - return getGrafanaSecretName(clusterID) -} - -func (outputGrafana) getServiceName() string { - return "monitor-grafana" -} diff --git a/internal/integratedservices/services/monitoring/output_prometheus.go b/internal/integratedservices/services/monitoring/output_prometheus.go deleted file mode 100644 index 6a79893119..0000000000 --- a/internal/integratedservices/services/monitoring/output_prometheus.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -type outputPrometheus struct { - baseOutput -} - -func newPrometheusOutputHelper( - kubeConfig []byte, - spec integratedServiceSpec, -) outputPrometheus { - return outputPrometheus{ - baseOutput: baseOutput{ - ingress: spec.Prometheus.Ingress.baseIngressSpec, - secretID: spec.Prometheus.Ingress.SecretID, - enabled: spec.Prometheus.Enabled, - k8sConfig: kubeConfig, - }, - } -} - -func (outputPrometheus) getOutputType() string { - return "Prometheus" -} - -func (outputPrometheus) getTopLevelDeploymentKey() string { - return "prometheus" -} - -func (outputPrometheus) getDeploymentValueParentKey() string { - return "prometheusSpec" -} - -func (outputPrometheus) getGeneratedSecretName(clusterID uint) string { - return getPrometheusSecretName(clusterID) -} - -func (outputPrometheus) getServiceName() string { - return "monitor-prometheus-operato-prometheus" -} diff --git a/internal/integratedservices/services/monitoring/output_pushgateway.go b/internal/integratedservices/services/monitoring/output_pushgateway.go deleted file mode 100644 index 1474cd0932..0000000000 --- a/internal/integratedservices/services/monitoring/output_pushgateway.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -type outputPushgateway struct { - baseOutput -} - -func newPushgatewayOutputHelper( - kubeConfig []byte, - spec integratedServiceSpec, -) outputPushgateway { - return outputPushgateway{ - baseOutput: baseOutput{ - enabled: spec.Pushgateway.Enabled, - k8sConfig: kubeConfig, - }, - } -} - -func (outputPushgateway) getOutputType() string { - return "Pushgateway" -} - -func (outputPushgateway) getTopLevelDeploymentKey() string { - return "" -} - -func (outputPushgateway) getDeploymentValueParentKey() string { - return "" -} - -func (outputPushgateway) getGeneratedSecretName(clusterID uint) string { - return getPushgatewaySecretName(clusterID) -} - -func (outputPushgateway) getServiceName() string { - return "pushgateway-prometheus-pushgateway" -} diff --git a/internal/integratedservices/services/monitoring/secret_alertmanager.go b/internal/integratedservices/services/monitoring/secret_alertmanager.go deleted file mode 100644 index 9cd87fda03..0000000000 --- a/internal/integratedservices/services/monitoring/secret_alertmanager.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -type alertmanagerSecretInfoer struct{ baseSecretInfoer } - -func (alertmanagerSecretInfoer) name() string { - return "Alertmanager" -} - -func (i alertmanagerSecretInfoer) generatedSecretName() string { - return getAlertmanagerSecretName(i.clusterID) -} diff --git a/internal/integratedservices/services/monitoring/secret_base.go b/internal/integratedservices/services/monitoring/secret_base.go deleted file mode 100644 index ab0b3c4187..0000000000 --- a/internal/integratedservices/services/monitoring/secret_base.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -type baseSecretInfoer struct { - clusterID uint -} diff --git a/internal/integratedservices/services/monitoring/secret_prometheus.go b/internal/integratedservices/services/monitoring/secret_prometheus.go deleted file mode 100644 index 02b19cfac4..0000000000 --- a/internal/integratedservices/services/monitoring/secret_prometheus.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -type prometheusSecretInfoer struct{ baseSecretInfoer } - -func (prometheusSecretInfoer) name() string { - return "Prometheus" -} - -func (i prometheusSecretInfoer) generatedSecretName() string { - return getPrometheusSecretName(i.clusterID) -} diff --git a/internal/integratedservices/services/monitoring/spec.go b/internal/integratedservices/services/monitoring/spec.go deleted file mode 100644 index 92b7fd8441..0000000000 --- a/internal/integratedservices/services/monitoring/spec.go +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -import ( - "fmt" - "regexp" - - "emperror.dev/errors" - "github.com/mitchellh/mapstructure" - - "github.com/banzaicloud/pipeline/internal/integratedservices" - "github.com/banzaicloud/pipeline/src/dns" -) - -type integratedServiceSpec struct { - Prometheus prometheusSpec `json:"prometheus" mapstructure:"prometheus"` - Grafana grafanaSpec `json:"grafana" mapstructure:"grafana"` - Exporters exportersSpec `json:"exporters" mapstructure:"exporters"` - Alertmanager alertmanagerSpec `json:"alertmanager" mapstructure:"alertmanager"` - Pushgateway pushgatewaySpec `json:"pushgateway" mapstructure:"pushgateway"` -} - -type prometheusSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - Storage storageSpec `json:"storage" mapstructure:"storage"` - Ingress ingressSpecWithSecret `json:"ingress" mapstructure:"ingress"` -} - -type grafanaSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - SecretId string `json:"secretId" mapstructure:"secretId"` - Dashboards bool `json:"defaultDashboards" mapstructure:"defaultDashboards"` - Ingress baseIngressSpec `json:"ingress" mapstructure:"ingress"` -} - -type storageSpec struct { - Class string `json:"class" mapstructure:"class"` - Size uint `json:"size" mapstructure:"size"` - Retention string `json:"retention" mapstructure:"retention"` -} - -type ingressSpecWithSecret struct { - baseIngressSpec `mapstructure:",squash"` - SecretID string `json:"secretId" mapstructure:"secretId"` -} - -type baseIngressSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - Domain string `json:"domain" mapstructure:"domain"` - Path string `json:"path" mapstructure:"path"` -} - -type exportersSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - NodeExporter exporterBaseSpec `json:"nodeExporter" mapstructure:"nodeExporter"` - KubeStateMetrics exporterBaseSpec `json:"kubeStateMetrics" mapstructure:"kubeStateMetrics"` -} - -type exporterBaseSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` -} - -type alertmanagerSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - Provider map[string]interface{} `json:"provider" mapstructure:"provider"` - Ingress ingressSpecWithSecret `json:"ingress" mapstructure:"ingress"` -} - -type pushgatewaySpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` -} - -type pagerDutySpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - URL string `json:"url" mapstructure:"url"` - SecretID string `json:"secretId" mapstructure:"secretId"` - IntegrationType string `json:"integrationType" mapstructure:"integrationType"` - SendResolved bool `json:"sendResolved" mapstructure:"sendResolved"` -} - -type slackSpec struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - SecretID string `json:"secretId" mapstructure:"secretId"` - Channel string `json:"channel" mapstructure:"channel"` - SendResolved bool `json:"sendResolved" mapstructure:"sendResolved"` -} - -func (s integratedServiceSpec) Validate() error { - // Prometheus validation - if err := s.Prometheus.Validate(); err != nil { - return err - } - - // Grafana validation - if err := s.Grafana.Validate(); err != nil { - return err - } - - // Alertmanager validation - if err := s.Alertmanager.Validate(); err != nil { - return err - } - - if !s.Exporters.Enabled { - return canNotDisabledError{fieldName: "exporters"} - } - - if !s.Exporters.KubeStateMetrics.Enabled { - return canNotDisabledError{fieldName: "kubeStateMetrics"} - } - - if !s.Exporters.NodeExporter.Enabled { - return canNotDisabledError{fieldName: "nodeExporter"} - } - - return nil -} - -func (s prometheusSpec) Validate() error { - if !s.Enabled { - // Prometheus cannot be disabled - return canNotDisabledError{fieldName: "prometheus"} - } - - // ingress validation - if err := s.Ingress.Validate(ingressTypePrometheus); err != nil { - return errors.WrapIf(err, "error during validate Prometheus ingress") - } - - // storage validation - if err := s.Storage.Validate(); err != nil { - return err - } - - return nil -} - -func (s ingressSpecWithSecret) Validate(ingressType string) error { - return s.baseIngressSpec.Validate(ingressType) -} - -func (s baseIngressSpec) Validate(ingressType string) error { - if s.Enabled { - if s.Path == "" { - return requiredFieldError{fieldName: fmt.Sprintf("%s path", ingressType)} - } - - if s.Domain != "" { - err := dns.ValidateSubdomain(s.Domain) - if err != nil { - return errors.Append(err, invalidIngressHostError{hostType: ingressType}) - } - } - } - - return nil -} - -func (s storageSpec) Validate() error { - if s.Size < 0 { - return errors.New("storage size must be a non-negative number") - } - - if s.Retention == "" { - return requiredFieldError{fieldName: "retention"} - } - - match, err := regexp.MatchString("[0-9]+(ms|s|m|h|d|w|y)", s.Retention) - if err != nil { - return errors.WrapIf(err, "failed to check retention") - } - - if !match { - return errors.WrapIf(err, "invalid retention") - } - - return nil -} - -func (s grafanaSpec) Validate() error { - if s.Enabled { - if err := s.Ingress.Validate(ingressTypeGrafana); err != nil { - return errors.WrapIf(err, "error during validate Grafana ingress") - } - } - - return nil -} - -func (s alertmanagerSpec) Validate() error { - if s.Enabled { - // ingress validation - if err := s.Ingress.Validate(ingressTypeAlertmanager); err != nil { - return err - } - - // validate Slack notification provider - if slackProv, ok := s.Provider[alertmanagerProviderSlack]; ok { - var slack slackSpec - if err := mapstructure.Decode(slackProv, &slack); err != nil { - return errors.WrapIf(err, "failed to bind Slack config") - } - if err := slack.Validate(); err != nil { - return errors.WrapIf(err, "error during validating Slack") - } - } - - // validate PagerDuty notification provider - if pagerDutyProv, ok := s.Provider[alertmanagerProviderPagerDuty]; ok { - var pd pagerDutySpec - if err := mapstructure.Decode(pagerDutyProv, &pd); err != nil { - return errors.WrapIf(err, "failed to bind PagerDuty config") - } - if err := pd.Validate(); err != nil { - return errors.WrapIf(err, "error during validating PagerDuty") - } - } - } - - return nil -} - -func (s slackSpec) Validate() error { - if s.Enabled { - if s.SecretID == "" { - return requiredFieldError{fieldName: "secretId"} - } - - if s.Channel == "" { - return requiredFieldError{fieldName: "channel"} - } - } - - return nil -} - -func (s pagerDutySpec) Validate() error { - if s.Enabled { - if s.SecretID == "" { - return requiredFieldError{fieldName: "secretId"} - } - - if s.URL == "" { - return requiredFieldError{fieldName: "url"} - } - - if s.IntegrationType == "" { - return requiredFieldError{fieldName: "integrationType"} - } - - if s.IntegrationType != pagerDutyIntegrationEventApiV2 && s.IntegrationType != pagerDutyIntegrationPrometheus { - return errors.New(fmt.Sprintf("integration type should be only just: %s or %s", pagerDutyIntegrationEventApiV2, pagerDutyIntegrationPrometheus)) - } - } - - return nil -} - -func bindIntegratedServiceSpec(spec integratedservices.IntegratedServiceSpec) (integratedServiceSpec, error) { - var boundSpec integratedServiceSpec - if err := mapstructure.Decode(spec, &boundSpec); err != nil { - return boundSpec, integratedservices.InvalidIntegratedServiceSpecError{ - IntegratedServiceName: integratedServiceName, - Problem: errors.WrapIf(err, "failed to bind integrated service spec").Error(), - } - } - return boundSpec, nil -} diff --git a/internal/integratedservices/services/monitoring/values.go b/internal/integratedservices/services/monitoring/values.go deleted file mode 100644 index 55b0cfaead..0000000000 --- a/internal/integratedservices/services/monitoring/values.go +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright © 2019 Banzai Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package monitoring - -type prometheusOperatorValues struct { - PrometheusOperator operatorSpecValues `json:"prometheusOperator"` - Grafana *grafanaValues `json:"grafana"` - Alertmanager *alertmanagerValues `json:"alertmanager"` - Prometheus *prometheusValues `json:"prometheus"` - KubeStateMetrics kubeStateMetricsValues `json:"kubeStateMetrics"` - NodeExporter nodeExporterValues `json:"nodeExporter"` - KsmValues *ksmValues `json:"kube-state-metrics"` - NeValues *neValues `json:"prometheus-node-exporter"` -} - -type operatorSpecValues struct { - Image imageValues `json:"image"` - CleanupCustomResource bool `json:"cleanupCustomResource"` - CreateCustomResource bool `json:"createCustomResource"` -} - -type imageValues struct { - Repository string `json:"repository"` - Tag string `json:"tag"` -} - -type prometheusPushgatewayValues struct { - Image imageValues `json:"image"` - ServiceMonitor serviceMonitorValues `json:"serviceMonitor"` -} - -type serviceMonitorValues struct { - Enabled bool `json:"enabled"` - Namespace string `json:"namespace"` -} - -type baseValues struct { - Enabled bool `json:"enabled"` - Ingress ingressValues `json:"ingress"` -} - -type grafanaValues struct { - baseValues - - AdminUser string `json:"adminUser"` - AdminPassword string `json:"adminPassword"` - GrafanaIni grafanaIniValues `json:"grafana.ini"` - DefaultDashboardsEnabled bool `json:"defaultDashboardsEnabled"` - Image imageValues `json:"image"` - Persistence persistenceValues `json:"persistence"` - Sidecar sidecar `json:"sidecar"` -} - -type sidecar struct { - Datasources datasources `json:"datasources"` -} - -type datasources struct { - Enabled bool `json:"enabled"` - Label string `json:"label"` - SearchNamespace string `json:"searchNamespace"` -} - -type persistenceValues struct { - Enabled bool `json:"enabled"` -} - -type grafanaIniValues struct { - Server grafanaIniServerValues `json:"server"` -} - -type grafanaIniServerValues struct { - RootUrl string `json:"root_url"` - ServeFromSubPath bool `json:"serve_from_sub_path"` -} - -type alertmanagerValues struct { - baseValues - Spec baseSpecValues `json:"alertmanagerSpec"` - Config *configValues `json:"config"` -} - -type configValues struct { - Receivers []receiverItemValues `json:"receivers"` - Route routeValues `json:"route"` -} - -type routeValues struct { - Receiver string `json:"receiver"` - Routes []interface{} `json:"routes"` -} - -type receiverItemValues struct { - Name string `json:"name"` - SlackConfigs []slackConfigValues `json:"slack_configs,omitempty"` - PagerdutyConfigs []pagerdutyConfigValues `json:"pagerduty_config,omitempty"` -} - -type slackConfigValues struct { - ApiUrl string `json:"api_url"` - Channel string `json:"channel"` - SendResolved bool `json:"send_resolved"` -} - -type pagerdutyConfigValues struct { - RoutingKey string `json:"routing_key"` - ServiceKey string `json:"service_key"` - Url string `json:"url"` - SendResolved bool `json:"send_resolved"` -} - -type baseSpecValues struct { - RoutePrefix string `json:"routePrefix"` - Image imageValues `json:"image"` -} - -type PrometheusSpecValues struct { - baseSpecValues - RetentionSize string `json:"retentionSize"` - Retention string `json:"retention"` - StorageSpec map[string]interface{} `json:"storageSpec"` - ServiceMonitorSelectorNilUsesHelmValues bool `json:"serviceMonitorSelectorNilUsesHelmValues"` -} - -type prometheusValues struct { - baseValues - Spec PrometheusSpecValues `json:"prometheusSpec"` - Annotations map[string]interface{} `json:"annotations"` -} - -type kubeStateMetricsValues struct { - Enabled bool `json:"enabled"` -} - -type ksmValues struct { - Image imageValues `json:"image"` -} - -type neValues struct { - Image imageValues `json:"image"` -} - -type nodeExporterValues struct { - Enabled bool `json:"enabled"` -} - -type ingressValues struct { - Enabled bool `json:"enabled"` - Hosts []string `json:"hosts"` - Path string `json:"path,omitempty"` - Paths []string `json:"paths,omitempty"` - Annotations map[string]interface{} `json:"annotations,omitempty"` -} From 14e85835ac4fde21893c97744a209b930b08bbab Mon Sep 17 00:00:00 2001 From: Magyari Sandor Szilard Date: Wed, 8 Sep 2021 17:25:35 +0200 Subject: [PATCH 2/2] disable logging & montoring service on UI --- config/ui/feature-set.json.dist | 8 ++-- go.mod | 3 +- internal/cmd/config.go | 74 --------------------------------- 3 files changed, 5 insertions(+), 80 deletions(-) diff --git a/config/ui/feature-set.json.dist b/config/ui/feature-set.json.dist index 95bce3a1dc..59c0d68149 100644 --- a/config/ui/feature-set.json.dist +++ b/config/ui/feature-set.json.dist @@ -78,9 +78,9 @@ }, "features": { "monitoring": { - "enabled": true, + "enabled": false, "createClusterFlow": { - "enabled": true + "enabled": false } }, "securityScan": { @@ -93,9 +93,9 @@ "enabled": true }, "logging": { - "enabled": true, + "enabled": false, "createClusterFlow": { - "enabled": true + "enabled": false } }, "backup": { diff --git a/go.mod b/go.mod index 6951229b85..e405271b52 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,6 @@ require ( github.com/lestrrat-go/backoff v1.0.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/microcosm-cc/bluemonday v1.0.3 - github.com/mitchellh/copystructure v1.0.0 github.com/mitchellh/mapstructure v1.4.1 github.com/moogar0880/problems v0.1.1 github.com/oklog/run v1.1.0 @@ -93,7 +92,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 helm.sh/helm/v3 v3.5.3 k8s.io/api v0.20.9 - k8s.io/apiextensions-apiserver v0.20.9 + k8s.io/apiextensions-apiserver v0.20.9 // indirect k8s.io/apimachinery v0.20.9 k8s.io/cli-runtime v0.20.9 k8s.io/client-go v0.20.9 diff --git a/internal/cmd/config.go b/internal/cmd/config.go index 281e6f2318..54578629a4 100644 --- a/internal/cmd/config.go +++ b/internal/cmd/config.go @@ -537,80 +537,6 @@ func Configure(v *viper.Viper, p *pflag.FlagSet) { v.SetDefault("cluster::vault::charts::webhook::version", "1.10.1") v.SetDefault("cluster::vault::charts::webhook::values", map[string]interface{}{}) - v.SetDefault("cluster::monitoring::enabled", true) - v.SetDefault("cluster::monitoring::namespace", "") - v.SetDefault("cluster::monitoring::grafana::adminUser", "admin") - v.SetDefault("cluster::monitoring::charts::operator::chart", "prometheus-community/kube-prometheus-stack") - v.SetDefault("cluster::monitoring::charts::operator::version", "12.11.3") - v.SetDefault("cluster::monitoring::charts::operator::values", map[string]interface{}{ - "prometheus": map[string]interface{}{ - "ingress": map[string]interface{}{ - "annotations": map[string]interface{}{ - "traefik.frontend.rule.type": "PathPrefix", - "traefik.ingress.kubernetes.io/ssl-redirect": "true", - }, - }, - }, - "alertmanager": map[string]interface{}{ - "ingress": map[string]interface{}{ - "annotations": map[string]interface{}{ - "traefik.frontend.rule.type": "PathPrefix", - "traefik.ingress.kubernetes.io/ssl-redirect": "true", - }, - }, - }, - "grafana": map[string]interface{}{ - "ingress": map[string]interface{}{ - "annotations": map[string]interface{}{ - "traefik.frontend.rule.type": "PathPrefixStrip", - "traefik.ingress.kubernetes.io/redirect-permanent": "true", - "traefik.ingress.kubernetes.io/redirect-regex": "^http://(.*)", - "traefik.ingress.kubernetes.io/redirect-replacement": `https://$1\`, - }, - }, - "sidecar": map[string]interface{}{ - "datasources": map[string]interface{}{ - "enabled": "true", - }, - }, - }, - }) - v.SetDefault("cluster::monitoring::images::operator::repository", "quay.io/prometheus-operator/prometheus-operator") - v.SetDefault("cluster::monitoring::images::operator::tag", "v0.44.1") - v.SetDefault("cluster::monitoring::images::prometheus::repository", "quay.io/prometheus/prometheus") - v.SetDefault("cluster::monitoring::images::prometheus::tag", "v2.22.1") - v.SetDefault("cluster::monitoring::images::alertmanager::repository", "quay.io/prometheus/alertmanager") - v.SetDefault("cluster::monitoring::images::alertmanager::tag", "v0.21.0") - v.SetDefault("cluster::monitoring::images::grafana::repository", "grafana/grafana") - v.SetDefault("cluster::monitoring::images::grafana::tag", "7.3.5") - v.SetDefault("cluster::monitoring::images::kubestatemetrics::repository", "quay.io/coreos/kube-state-metrics") - v.SetDefault("cluster::monitoring::images::kubestatemetrics::tag", "v1.9.7") - v.SetDefault("cluster::monitoring::images::nodeexporter::repository", "quay.io/prometheus/node-exporter") - v.SetDefault("cluster::monitoring::images::nodeexporter::tag", "v1.0.1") - - v.SetDefault("cluster::monitoring::charts::pushgateway::chart", "prometheus-community/prometheus-pushgateway") - v.SetDefault("cluster::monitoring::charts::pushgateway::version", "1.5.1") - v.SetDefault("cluster::monitoring::charts::pushgateway::values", map[string]interface{}{}) - v.SetDefault("cluster::monitoring::images::pushgateway::repository", "prom/pushgateway") - v.SetDefault("cluster::monitoring::images::pushgateway::tag", "v1.0.1") - - v.SetDefault("cluster::logging::enabled", true) - v.SetDefault("cluster::logging::namespace", "") - v.SetDefault("cluster::logging::charts::operator::chart", "banzaicloud-stable/logging-operator") - v.SetDefault("cluster::logging::charts::operator::version", "3.2.2") - v.SetDefault("cluster::logging::charts::operator::values", map[string]interface{}{}) - v.SetDefault("cluster::logging::images::operator::repository", "banzaicloud/logging-operator") - v.SetDefault("cluster::logging::images::operator::tag", "3.2.0") - v.SetDefault("cluster::logging::charts::loki::chart", "banzaicloud-stable/loki") - v.SetDefault("cluster::logging::charts::loki::version", "0.17.4") - v.SetDefault("cluster::logging::charts::loki::values", map[string]interface{}{}) - v.SetDefault("cluster::logging::images::loki::repository", "grafana/loki") - v.SetDefault("cluster::logging::images::loki::tag", "v1.3.0") - v.SetDefault("cluster::logging::images::fluentbit::repository", "fluent/fluent-bit") - v.SetDefault("cluster::logging::images::fluentbit::tag", "1.4.4") - v.SetDefault("cluster::logging::images::fluentd::repository", "banzaicloud/fluentd") - v.SetDefault("cluster::logging::images::fluentd::tag", "v1.10.3-alpine-2") - v.SetDefault("cluster::dns::enabled", true) v.SetDefault("cluster::dns::namespace", "") v.SetDefault("cluster::dns::baseDomain", "")