From 173dd78ce8d1d2f0a56d43bac524bca83cdb788d Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Wed, 8 Nov 2023 15:56:36 -0500 Subject: [PATCH 01/23] THREESCALE-10388 Migrate apicast DeploymentConfigs to Deployments --- pkg/3scale/amp/component/apicast.go | 165 ++-- .../amp/operator/apicast_options_provider.go | 11 +- .../operator/apicast_options_provider_test.go | 17 +- pkg/3scale/amp/operator/apicast_reconciler.go | 136 +-- .../amp/operator/apicast_reconciler_test.go | 175 ++-- .../base_apimanager_logic_reconciler.go | 5 + pkg/helper/deployment.go | 18 + pkg/reconcilers/deployment.go | 290 ++++++ pkg/reconcilers/deployment_test.go | 916 ++++++++++++++++++ pkg/reconcilers/service.go | 46 + pkg/upgrade/migrate_deployment_config_2.15.go | 57 ++ 11 files changed, 1585 insertions(+), 251 deletions(-) create mode 100644 pkg/helper/deployment.go create mode 100644 pkg/reconcilers/deployment.go create mode 100644 pkg/reconcilers/deployment_test.go create mode 100644 pkg/upgrade/migrate_deployment_config_2.15.go diff --git a/pkg/3scale/amp/component/apicast.go b/pkg/3scale/amp/component/apicast.go index d6b6b5d9d..ae2d788ad 100644 --- a/pkg/3scale/amp/component/apicast.go +++ b/pkg/3scale/amp/component/apicast.go @@ -9,8 +9,9 @@ import ( "strings" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,8 +19,9 @@ import ( ) const ( - ApicastStagingName = "apicast-staging" - ApicastProductionName = "apicast-production" + ApicastStagingName = "apicast-staging" + ApicastProductionName = "apicast-production" + ApicastProductionInitContainerName = "system-master-svc" CustomPoliciesMountBasePath = "/opt/app-root/src/policies" CustomPoliciesAnnotationNameSegmentPrefix = "apicast-policy-volume" @@ -66,7 +68,7 @@ func (apicast *Apicast) StagingService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: apicast.stagingServicePorts(), - Selector: map[string]string{"deploymentConfig": ApicastStagingName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: ApicastStagingName}, }, } } @@ -83,59 +85,40 @@ func (apicast *Apicast) ProductionService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: apicast.productionServicePorts(), - Selector: map[string]string{"deploymentConfig": ApicastProductionName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: ApicastProductionName}, }, } } -func (apicast *Apicast) StagingDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps.openshift.io/v1", Kind: "DeploymentConfig"}, +func (apicast *Apicast) StagingDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: ApicastStagingName, Labels: apicast.Options.CommonStagingLabels, - Annotations: apicast.stagingDeploymentConfigAnnotations(), + Annotations: apicast.stagingDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Replicas: apicast.Options.StagingReplicas, - Selector: map[string]string{ - "deploymentConfig": ApicastStagingName, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &apicast.Options.StagingReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ApicastStagingName, + }, }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - IntervalSeconds: &[]int64{1}[0], + Strategy: k8sappsv1.DeploymentStrategy{ + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, - TimeoutSeconds: &[]int64{1800}[0], - UpdatePeriodSeconds: &[]int64{1}[0], }, - Type: appsv1.DeploymentStrategyTypeRolling, + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - ApicastStagingName, - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-apicast:%s", apicast.Options.ImageTag), - }, - }, - }, - }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: apicast.Options.StagingPodTemplateLabels, Annotations: apicast.stagingPodAnnotations(), @@ -146,7 +129,7 @@ func (apicast *Apicast) StagingDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", Volumes: apicast.stagingVolumes(), Containers: []v1.Container{ - v1.Container{ + { Ports: apicast.stagingContainerPorts(), Env: apicast.buildApicastStagingEnv(), Image: "amp-apicast:latest", @@ -182,55 +165,35 @@ func (apicast *Apicast) StagingDeploymentConfig() *appsv1.DeploymentConfig { } } -func (apicast *Apicast) ProductionDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps.openshift.io/v1", Kind: "DeploymentConfig"}, +func (apicast *Apicast) ProductionDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: ApicastProductionName, Labels: apicast.Options.CommonProductionLabels, - Annotations: apicast.productionDeploymentConfigAnnotations(), + Annotations: apicast.productionDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Replicas: apicast.Options.ProductionReplicas, - Selector: map[string]string{ - "deploymentConfig": ApicastProductionName, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &apicast.Options.ProductionReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ApicastProductionName, + }, }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - IntervalSeconds: &[]int64{1}[0], + Strategy: k8sappsv1.DeploymentStrategy{ + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, - TimeoutSeconds: &[]int64{1800}[0], - UpdatePeriodSeconds: &[]int64{1}[0], }, - Type: appsv1.DeploymentStrategyTypeRolling, + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-master-svc", - ApicastProductionName, - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-apicast:%s", apicast.Options.ImageTag), - }, - }, - }, - }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: apicast.Options.ProductionPodTemplateLabels, Annotations: apicast.productionPodAnnotations(), @@ -241,12 +204,12 @@ func (apicast *Apicast) ProductionDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", Volumes: apicast.productionVolumes(), InitContainers: []v1.Container{ - v1.Container{ - Name: "system-master-svc", + { + Name: ApicastProductionInitContainerName, Image: "amp-apicast:latest", Command: []string{"sh", "-c", "until $(curl --output /dev/null --silent --fail --head http://system-master:3000/status); do sleep $SLEEP_SECONDS; done"}, Env: []v1.EnvVar{ - v1.EnvVar{ + { Name: "SLEEP_SECONDS", Value: "1", }, @@ -254,7 +217,7 @@ func (apicast *Apicast) ProductionDeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ + { Ports: apicast.productionContainerPorts(), Env: apicast.buildApicastProductionEnv(), Image: "amp-apicast:latest", @@ -495,7 +458,7 @@ func (apicast *Apicast) StagingPodDisruptionBudget() *policyv1.PodDisruptionBudg }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": ApicastStagingName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: ApicastStagingName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -514,7 +477,7 @@ func (apicast *Apicast) ProductionPodDisruptionBudget() *policyv1.PodDisruptionB }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": ApicastProductionName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: ApicastProductionName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -763,9 +726,22 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { return volumes } -func (apicast *Apicast) productionDeploymentConfigAnnotations() map[string]string { +func (apicast *Apicast) productionDeploymentAnnotations() map[string]string { annotations := map[string]string{} + // Image trigger annotation should always be present + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: ApicastProductionInitContainerName, + Tag: fmt.Sprintf("amp-apicast:%v", apicast.Options.ImageTag), + }, + { + Name: ApicastProductionName, + Tag: fmt.Sprintf("amp-apicast:%v", apicast.Options.ImageTag), + }, + }) + annotations[reconcilers.DeploymentImageTriggerAnnotation] = imageTriggerString + for _, customPolicy := range apicast.Options.ProductionCustomPolicies { annotations[customPolicy.AnnotationKey()] = customPolicy.AnnotationValue() } @@ -779,17 +755,21 @@ func (apicast *Apicast) productionDeploymentConfigAnnotations() map[string]strin annotations[customEnvAnnotationKey(customEnvSecret)] = customEnvAnnotationValue(customEnvSecret) } - // keep backward compat - if len(annotations) == 0 { - return nil - } - return annotations } -func (apicast *Apicast) stagingDeploymentConfigAnnotations() map[string]string { +func (apicast *Apicast) stagingDeploymentAnnotations() map[string]string { annotations := map[string]string{} + // Image trigger annotation should always be present + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: ApicastStagingName, + Tag: fmt.Sprintf("amp-apicast:%v", apicast.Options.ImageTag), + }, + }) + annotations[reconcilers.DeploymentImageTriggerAnnotation] = imageTriggerString + for _, customPolicy := range apicast.Options.StagingCustomPolicies { annotations[customPolicy.AnnotationKey()] = customPolicy.AnnotationValue() } @@ -803,11 +783,6 @@ func (apicast *Apicast) stagingDeploymentConfigAnnotations() map[string]string { annotations[customEnvAnnotationKey(customEnvSecret)] = customEnvAnnotationValue(customEnvSecret) } - // keep backward compat - if len(annotations) == 0 { - return nil - } - return annotations } diff --git a/pkg/3scale/amp/operator/apicast_options_provider.go b/pkg/3scale/amp/operator/apicast_options_provider.go index 6ea40c043..c37c47fc7 100644 --- a/pkg/3scale/amp/operator/apicast_options_provider.go +++ b/pkg/3scale/amp/operator/apicast_options_provider.go @@ -19,6 +19,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) type ApicastOptionsProvider struct { @@ -146,7 +147,7 @@ func (a *ApicastOptionsProvider) setResourceRequirementsOptions() { a.apicastOptions.StagingResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if a.apimanager.Spec.Apicast.ProductionSpec.Resources != nil { @@ -208,7 +209,7 @@ func (a *ApicastOptionsProvider) stagingPodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "apicast-staging" + labels[reconcilers.DeploymentLabelSelector] = "apicast-staging" return labels } @@ -224,7 +225,7 @@ func (a *ApicastOptionsProvider) productionPodTemplateLabels() map[string]string labels[k] = v } - labels["deploymentConfig"] = "apicast-production" + labels[reconcilers.DeploymentLabelSelector] = "apicast-production" return labels } @@ -245,7 +246,7 @@ func (a *ApicastOptionsProvider) setCustomPolicies() error { Child("apicast"). Child("productionSpec"). Child("customPolicies").Index(idx) - fldErr = append(fldErr, field.Invalid(customPoliciesIdxFldPath, customPolicySpec, err.Error())) + fldErr = append(fldErr, field.Invalid(customPoliciesIdxFldPath, customPolicySpec, err.Error())) return fldErr.ToAggregate() } @@ -272,7 +273,7 @@ func (a *ApicastOptionsProvider) setCustomPolicies() error { Child("apicast"). Child("stagingSpec"). Child("customPolicies").Index(idx) - fldErr = append(fldErr, field.Invalid(customPoliciesIdxFldPath, customPolicySpec, err.Error())) + fldErr = append(fldErr, field.Invalid(customPoliciesIdxFldPath, customPolicySpec, err.Error())) return fldErr.ToAggregate() } diff --git a/pkg/3scale/amp/operator/apicast_options_provider_test.go b/pkg/3scale/amp/operator/apicast_options_provider_test.go index c875a3f34..04829d160 100644 --- a/pkg/3scale/amp/operator/apicast_options_provider_test.go +++ b/pkg/3scale/amp/operator/apicast_options_provider_test.go @@ -1,6 +1,7 @@ package operator import ( + "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "strconv" "testing" @@ -53,10 +54,10 @@ func testApicastProductionLabels() map[string]string { func testApicastStagingPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "apicast", - "threescale_component_element": "staging", - "deploymentConfig": "apicast-staging", + "app": appLabel, + "threescale_component": "apicast", + "threescale_component_element": "staging", + reconcilers.DeploymentLabelSelector: "apicast-staging", } addExpectedMeteringLabels(labels, "apicast-staging", helper.ApplicationType) @@ -65,10 +66,10 @@ func testApicastStagingPodLabels() map[string]string { func testApicastProductionPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "apicast", - "threescale_component_element": "production", - "deploymentConfig": "apicast-production", + "app": appLabel, + "threescale_component": "apicast", + "threescale_component_element": "production", + reconcilers.DeploymentLabelSelector: "apicast-production", } addExpectedMeteringLabels(labels, "apicast-production", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/apicast_reconciler.go b/pkg/3scale/amp/operator/apicast_reconciler.go index 7a4b97e6f..bab09caae 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler.go +++ b/pkg/3scale/amp/operator/apicast_reconciler.go @@ -6,7 +6,7 @@ import ( "reflect" "strings" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,6 +17,7 @@ import ( "github.com/3scale/3scale-operator/pkg/common" "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" ) func ApicastEnvCMMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { @@ -62,12 +63,12 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - stagingMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, + stagingMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentAnnotationsMutator, + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, apicastLogLevelEnvVarMutator, apicastTracingConfigEnvVarsMutator, apicastOpentelemetryConfigEnvVarsMutator, @@ -83,28 +84,37 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { apicastCustomEnvAnnotationsMutator, // Should be always after volume mutator portsMutator, apicastPodTemplateEnvConfigMapAnnotationsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, } if r.apiManager.Spec.Apicast.StagingSpec.Replicas != nil { - stagingMutators = append(stagingMutators, reconcilers.DeploymentConfigReplicasMutator) + stagingMutators = append(stagingMutators, reconcilers.DeploymentReplicasMutator) } - // Staging DC - err = r.ReconcileDeploymentConfig(apicast.StagingDeploymentConfig(), reconcilers.DeploymentConfigMutator(stagingMutators...)) + // Staging Deployment + err = r.ReconcileDeployment(apicast.StagingDeployment(), reconcilers.DeploymentMutator(stagingMutators...)) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ApicastStagingName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + // add apicast production env var mutator - productionMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, + productionMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentAnnotationsMutator, + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, apicastProductionWorkersEnvVarMutator, apicastLogLevelEnvVarMutator, apicastTracingConfigEnvVarsMutator, @@ -121,33 +131,43 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { apicastCustomEnvAnnotationsMutator, // Should be always after volume portsMutator, apicastPodTemplateEnvConfigMapAnnotationsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, } if r.apiManager.Spec.Apicast.ProductionSpec.Replicas != nil { - productionMutators = append(productionMutators, reconcilers.DeploymentConfigReplicasMutator) + productionMutators = append(productionMutators, reconcilers.DeploymentReplicasMutator) } - // Production DC - productionDCMutator := reconcilers.DeploymentConfigMutator( - productionMutators..., - ) + // Production Deployment + err = r.ReconcileDeployment(apicast.ProductionDeployment(), reconcilers.DeploymentMutator(productionMutators...)) + if err != nil { + return reconcile.Result{}, err + } - err = r.ReconcileDeploymentConfig(apicast.ProductionDeploymentConfig(), productionDCMutator) + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ApicastProductionName, r.apiManager.GetNamespace(), r.Client()) if err != nil { return reconcile.Result{}, err } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + getApiCastServiceMutator(r.apiManager.ObjectMeta.GetAnnotations()), + reconcilers.ServiceSelectorMutator, + } // Staging Service - err = r.ReconcileService(apicast.StagingService(), getApiCastServiceMutator(r.apiManager.ObjectMeta.GetAnnotations())) + err = r.ReconcileService(apicast.StagingService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } // Production Service - err = r.ReconcileService(apicast.ProductionService(), getApiCastServiceMutator(r.apiManager.ObjectMeta.GetAnnotations())) + err = r.ReconcileService(apicast.ProductionService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -226,44 +246,44 @@ func getApiCastServiceMutator(apiManagerAnnotations map[string]string) reconcile return reconcilers.ServicePortMutator } -func apicastProductionWorkersEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastProductionWorkersEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVar only for "APICAST_WORKERS" - return reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_WORKERS"), nil + return reconcilers.DeploymentEnvVarReconciler(desired, existing, "APICAST_WORKERS"), nil } -func apicastLogLevelEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastLogLevelEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVar only for "APICAST_LOG_LEVEL" - return reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_LOG_LEVEL"), nil + return reconcilers.DeploymentEnvVarReconciler(desired, existing, "APICAST_LOG_LEVEL"), nil } -func apicastTracingConfigEnvVarsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastTracingConfigEnvVarsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVars related to opentracing var changed bool - changed = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "OPENTRACING_TRACER") + changed = reconcilers.DeploymentEnvVarReconciler(desired, existing, "OPENTRACING_TRACER") - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "OPENTRACING_CONFIG") + tmpChanged := reconcilers.DeploymentEnvVarReconciler(desired, existing, "OPENTRACING_CONFIG") changed = changed || tmpChanged return changed, nil } -func apicastOpentelemetryConfigEnvVarsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastOpentelemetryConfigEnvVarsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVars related to opentracing var changed bool - changed = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "OPENTELEMETRY") + changed = reconcilers.DeploymentEnvVarReconciler(desired, existing, "OPENTELEMETRY") - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "OPENTELEMETRY_CONFIG") + tmpChanged := reconcilers.DeploymentEnvVarReconciler(desired, existing, "OPENTELEMETRY_CONFIG") changed = changed || tmpChanged return changed, nil } -func apicastEnvironmentEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastEnvironmentEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVar only for "APICAST_ENVIRONMENT" - return reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_ENVIRONMENT"), nil + return reconcilers.DeploymentEnvVarReconciler(desired, existing, "APICAST_ENVIRONMENT"), nil } -func apicastHTTPSEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastHTTPSEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVars related to opentracing var changed bool @@ -273,14 +293,14 @@ func apicastHTTPSEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool "APICAST_HTTPS_CERTIFICATE", "APICAST_HTTPS_CERTIFICATE_KEY", } { - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, envVar) + tmpChanged := reconcilers.DeploymentEnvVarReconciler(desired, existing, envVar) changed = changed || tmpChanged } return changed, nil } -func apicastProxyConfigurationsEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastProxyConfigurationsEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Reconcile EnvVars related to APIcast proxy-related configurations var changed bool @@ -290,18 +310,18 @@ func apicastProxyConfigurationsEnvVarMutator(desired, existing *appsv1.Deploymen "HTTPS_PROXY", "NO_PROXY", } { - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, envVar) + tmpChanged := reconcilers.DeploymentEnvVarReconciler(desired, existing, envVar) changed = changed || tmpChanged } return changed, nil } -func apicastServiceCacheSizeEnvVarMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - return reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_SERVICE_CACHE_SIZE"), nil +func apicastServiceCacheSizeEnvVarMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + return reconcilers.DeploymentEnvVarReconciler(desired, existing, "APICAST_SERVICE_CACHE_SIZE"), nil } -func portsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func portsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { changed := false if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[0].Ports, desired.Spec.Template.Spec.Containers[0].Ports) { @@ -312,11 +332,11 @@ func portsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { return changed, nil } -// volumeMountsMutator implements basic VolumeMount reconcilliation +// apicastVolumeMountsMutator implements basic VolumeMount reconciliation // Added when in desired and not in existing // Updated when in desired and in existing but not equal // Existing not in desired will NOT be removed. Allows manually added volumemounts -func apicastVolumeMountsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastVolumeMountsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { changed := false existingContainer := &existing.Spec.Template.Spec.Containers[0] desiredContainer := &desired.Spec.Template.Spec.Containers[0] @@ -405,11 +425,11 @@ func apicastVolumeMountsMutator(desired, existing *appsv1.DeploymentConfig) (boo return changed, nil } -// volumeMountsMutator implements basic VolumeMount reconcilliation +// apicastVolumesMutator implements basic VolumeMount reconciliation // Added when in desired and not in existing // Updated when in desired and in existing but not equal // Existing not in desired will NOT be removed. Allows manually added volumemounts -func apicastVolumesMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastVolumesMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { changed := false existingSpec := &existing.Spec.Template.Spec desiredSpec := &desired.Spec.Template.Spec @@ -498,7 +518,7 @@ func apicastVolumesMutator(desired, existing *appsv1.DeploymentConfig) (bool, er return changed, nil } -func apicastCustomPolicyAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastCustomPolicyAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // It is expected that APIManagerMutator has already added desired annotations to the existing annotations // find existing custom policy annotations not in desired and delete them updated := false @@ -522,7 +542,7 @@ func apicastCustomPolicyAnnotationsMutator(desired, existing *appsv1.DeploymentC return updated, nil } -func apicastTracingConfigAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastTracingConfigAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // It is expected that APIManagerMutator has already added desired annotations to the existing annotations // find existing tracing config volume annotations not in desired and delete them updated := false @@ -546,7 +566,7 @@ func apicastTracingConfigAnnotationsMutator(desired, existing *appsv1.Deployment return updated, nil } -func apicastOpentelemetryConfigAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastOpentelemetryConfigAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // It is expected that APIManagerMutator has already added desired annotations to the existing annotations // find existing tracing config volume annotations not in desired and delete them updated := false @@ -570,7 +590,7 @@ func apicastOpentelemetryConfigAnnotationsMutator(desired, existing *appsv1.Depl return updated, nil } -func apicastCustomEnvAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastCustomEnvAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // It is expected that APIManagerMutator has already added desired annotations to the existing annotations // find existing custom environments annotations not in desired and delete them updated := false @@ -594,7 +614,7 @@ func apicastCustomEnvAnnotationsMutator(desired, existing *appsv1.DeploymentConf return updated, nil } -func apicastPodTemplateEnvConfigMapAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func apicastPodTemplateEnvConfigMapAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Only reconcile the pod annotation regarding apicast-environment hash desiredVal, ok := desired.Spec.Template.Annotations[APIcastEnvironmentCMAnnotation] if !ok { diff --git a/pkg/3scale/amp/operator/apicast_reconciler_test.go b/pkg/3scale/amp/operator/apicast_reconciler_test.go index 892bb5daa..8c15b1666 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler_test.go +++ b/pkg/3scale/amp/operator/apicast_reconciler_test.go @@ -14,6 +14,7 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -120,8 +121,8 @@ func TestApicastReconciler(t *testing.T) { objName string obj k8sclient.Object }{ - {"stagingDeployment", "apicast-staging", &appsv1.DeploymentConfig{}}, - {"productionDeployment", "apicast-production", &appsv1.DeploymentConfig{}}, + {"stagingDeployment", "apicast-staging", &k8sappsv1.Deployment{}}, + {"productionDeployment", "apicast-production", &k8sappsv1.Deployment{}}, {"stagingService", "apicast-staging", &v1.Service{}}, {"productionService", "apicast-production", &v1.Service{}}, {"envConfigMap", "apicast-environment", &v1.ConfigMap{}}, @@ -218,10 +219,10 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { }, } - // Existing DC has 1 custom policy defined: P1 - // Desired DC has 1 custom policy defined: P2 - // P2 should be added to existing DC - // P1 should be deleted from existing DC + // Existing Deployment has 1 custom policy defined: P1 + // Desired Deployment has 1 custom policy defined: P2 + // P2 should be added to existing Deployment + // P1 should be deleted from existing Deployment apicastOptions := &component.ApicastOptions{ ProductionCustomPolicies: []component.CustomPolicy{p1CustomPolicy}, @@ -229,12 +230,12 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { ProductionTracingConfig: &component.APIcastTracingConfig{}, } apicast := component.NewApicast(apicastOptions) - existingProdDC := apicast.ProductionDeploymentConfig() - existingProdDC.Namespace = namespace + existingProdDeployment := apicast.ProductionDeployment() + existingProdDeployment.Namespace = namespace // - Policy annotation for P1 added p1Found := false - for key := range existingProdDC.Annotations { + for key := range existingProdDeployment.Annotations { if p1CustomPolicy.AnnotationKey() == key { p1Found = true } @@ -255,7 +256,7 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { } // Objects to track in the fake client. - objs := []runtime.Object{apimanager, existingProdDC, p2Secret} + objs := []runtime.Object{apimanager, existingProdDeployment, p2Secret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) err := appsv1.AddToScheme(s) @@ -299,14 +300,14 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { Name: "apicast-production", Namespace: namespace, } - existing := &appsv1.DeploymentConfig{} + existing := &k8sappsv1.Deployment{} err = cl.Get(context.TODO(), namespacedName, existing) // object must exist, that is all required to be tested if err != nil { t.Fatal(err) } - // Assert existing DC: + // Assert existing Deployment: // - Volume for P1 deleted for idx := range existing.Spec.Template.Spec.Volumes { if existing.Spec.Template.Spec.Volumes[idx].Name == p1CustomPolicy.VolumeName() { @@ -397,12 +398,12 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { ProductionTracingConfig: &existingTracingConfig1, } apicast := component.NewApicast(apicastOptions) - existingProdDC := apicast.ProductionDeploymentConfig() - existingProdDC.Namespace = namespace + existingProdDeployment := apicast.ProductionDeployment() + existingProdDeployment.Namespace = namespace - // - Tracing Configuration 1 added into the Production DC with the expected key + // - Tracing Configuration 1 added into the Production Deployment with the expected key existingTracingConfig1Found := false - for key := range existingProdDC.Annotations { + for key := range existingProdDeployment.Annotations { if existingTracingConfig1.AnnotationKey() == key { existingTracingConfig1Found = true } @@ -470,7 +471,7 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { } // Objects to track in the fake client. - objs := []runtime.Object{apimanager, existingProdDC, existingTc1Secret, desiredTc1Secret} + objs := []runtime.Object{apimanager, existingProdDeployment, existingTc1Secret, desiredTc1Secret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) err := appsv1.AddToScheme(s) @@ -515,14 +516,14 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { Name: "apicast-production", Namespace: namespace, } - existing := &appsv1.DeploymentConfig{} + existing := &k8sappsv1.Deployment{} err = cl.Get(context.TODO(), namespacedName, existing) // object must exist, that is all required to be tested if err != nil { t.Fatal(err) } - // // Assert existing DC: + // // Assert existing Deployment: // // - Volume for existingTracingConfig1 deleted for idx := range existing.Spec.Template.Spec.Volumes { if existing.Spec.Template.Spec.Volumes[idx].Name == existingTracingConfig1.VolumeName() { @@ -838,22 +839,22 @@ func TestReplicaApicastReconciler(t *testing.T) { t.Fatal(err) } - dc := &appsv1.DeploymentConfig{} + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: tc.objName, Namespace: namespace, } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - // bump the amount of replicas in the dc - dc.Spec.Replicas = twoValue - err = cl.Update(context.TODO(), dc) + // bump the amount of replicas in the deployment + deployment.Spec.Replicas = &twoValue + err = cl.Update(context.TODO(), deployment) if err != nil { - subT.Errorf("error updating dc of %s: %v", tc.objName, err) + subT.Errorf("error updating deployment of %s: %v", tc.objName, err) } // re-run the reconciler @@ -862,13 +863,13 @@ func TestReplicaApicastReconciler(t *testing.T) { t.Fatal(err) } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - if tc.expectedAmountOfReplicas != dc.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, dc.Spec.Replicas) + if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, deployment.Spec.Replicas) } }) } @@ -912,14 +913,14 @@ func testApicastAPIManagerCreator(stagingReplicas, productionReplicas *int64) *a func TestReplicaApicastTelemtryReconciler(t *testing.T) { var ( - trueValue = true - log = logf.Log.WithName("operator_test") - opentelemtryEnabled bool = true - apicastManagementAPI = "enabled" - openSSLVerify = &trueValue - includeResponseCodes = &trueValue - customKey = "my-custom-key.json" - configJsonKey = "config.json" + trueValue = true + log = logf.Log.WithName("operator_test") + opentelemtryEnabled = true + apicastManagementAPI = "enabled" + openSSLVerify = &trueValue + includeResponseCodes = &trueValue + customKey = "my-custom-key.json" + configJsonKey = "config.json" ) ctx := context.TODO() @@ -933,6 +934,10 @@ func TestReplicaApicastTelemtryReconciler(t *testing.T) { if err != nil { t.Fatal(err) } + err = k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } if err := configv1.AddToScheme(s); err != nil { t.Fatal(err) @@ -1181,7 +1186,7 @@ func TestReplicaApicastTelemtryReconciler(t *testing.T) { }, }, ), - opentelemetryEnvExistsWithCustomKeysOnBothDCs, + opentelemetryEnvExistsWithCustomKeysOnBothDeployments, false, false, multiKeyOtlpSecret(), @@ -1292,19 +1297,19 @@ func TestReplicaApicastTelemtryReconciler(t *testing.T) { } } -func validateOpentelemetryIsDisabled(dcType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { - dc := &appsv1.DeploymentConfig{} +func validateOpentelemetryIsDisabled(dType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ - Name: dcType, + Name: dType, Namespace: namespace, } - err := client.Get(context.TODO(), namespacedName, dc) + err := client.Get(context.TODO(), namespacedName, deployment) if err != nil { - return false, fmt.Errorf("error fetching object %s: %v", dcType, err) + return false, fmt.Errorf("error fetching object %s: %v", dType, err) } - envs := dc.Spec.Template.Spec.Containers[0].Env + envs := deployment.Spec.Template.Spec.Containers[0].Env var ( configEnvfound bool configEnvValueCorrect bool @@ -1330,22 +1335,22 @@ func validateOpentelemetryIsDisabled(dcType string, desiredConfigValue string, c // Check if required environment variables are present and have correct values if configEnvfound { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on deployment %s", dType) } if opentelemtryEnabledEnvFound { - return false, fmt.Errorf("OPENTELEMTRY env not found on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY env not found on deployment %s", dType) } if configEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on deployment %s", dType) } if opentelemtryEnabledEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY env value not correct on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY env value not correct on deployment %s", dType) } - volumeMounts := dc.Spec.Template.Spec.Containers[0].VolumeMounts + volumeMounts := deployment.Spec.Template.Spec.Containers[0].VolumeMounts var volumeMountFound bool - volumes := dc.Spec.Template.Spec.Volumes + volumes := deployment.Spec.Template.Spec.Volumes var volumeFound bool // Iterate over environment variables @@ -1362,16 +1367,16 @@ func validateOpentelemetryIsDisabled(dcType string, desiredConfigValue string, c } if volumeMountFound { - return false, fmt.Errorf("Opentelemetry volume mount found on dc %s", dcType) + return false, fmt.Errorf("opentelemetry volume mount found on deployment %s", dType) } if volumeFound { - return false, fmt.Errorf("Opentelemetry volume found on dc %s", dcType) + return false, fmt.Errorf("opentelemetry volume found on deployment %s", dType) } return true, nil } -func disableOpentelemtry(dc string, client k8sclient.WithWatch) (error, *appsv1alpha1.APIManager) { +func disableOpentelemtry(deployment string, client k8sclient.WithWatch) (error, *appsv1alpha1.APIManager) { apim := &appsv1alpha1.APIManager{} namespacedName := types.NamespacedName{ Name: "example-apimanager", @@ -1383,29 +1388,29 @@ func disableOpentelemtry(dc string, client k8sclient.WithWatch) (error, *appsv1a return fmt.Errorf("error fetching APIM %s", err), nil } - if dc == "apicast-staging" { + if deployment == "apicast-staging" { *apim.Spec.Apicast.StagingSpec.OpenTelemetry = appsv1alpha1.OpenTelemetrySpec{} } - if dc == "apicast-production" { + if deployment == "apicast-production" { *apim.Spec.Apicast.ProductionSpec.OpenTelemetry = appsv1alpha1.OpenTelemetrySpec{} } return nil, apim } -func opentelemetryEnvExistsWithDefaultValues(dcType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { - dc := &appsv1.DeploymentConfig{} +func opentelemetryEnvExistsWithDefaultValues(dType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ - Name: dcType, + Name: dType, Namespace: namespace, } - err := client.Get(context.TODO(), namespacedName, dc) + err := client.Get(context.TODO(), namespacedName, deployment) if err != nil { - return false, fmt.Errorf("error fetching object %s: %v", dcType, err) + return false, fmt.Errorf("error fetching object %s: %v", dType, err) } - envs := dc.Spec.Template.Spec.Containers[0].Env + envs := deployment.Spec.Template.Spec.Containers[0].Env var ( configEnvfound bool configEnvValueCorrect bool @@ -1431,22 +1436,22 @@ func opentelemetryEnvExistsWithDefaultValues(dcType string, desiredConfigValue s // Check if required environment variables are present and have correct values if !configEnvfound { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on deployment %s", dType) } if !opentelemtryEnabledEnvFound { - return false, fmt.Errorf("OPENTELEMTRY env not found on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY env not found on deployment %s", dType) } if !configEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on deployment %s", dType) } if !opentelemtryEnabledEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY env value not correct on dc %s", dcType) + return false, fmt.Errorf("OPENTELEMTRY env value not correct on deployment %s", dType) } - volumeMounts := dc.Spec.Template.Spec.Containers[0].VolumeMounts + volumeMounts := deployment.Spec.Template.Spec.Containers[0].VolumeMounts var volumeMountFound bool - volumes := dc.Spec.Template.Spec.Volumes + volumes := deployment.Spec.Template.Spec.Volumes var volumeFound bool // Iterate over environment variables @@ -1463,29 +1468,29 @@ func opentelemetryEnvExistsWithDefaultValues(dcType string, desiredConfigValue s } if !volumeMountFound { - return false, fmt.Errorf("Opentelemetry volume mount not found on dc %s", dcType) + return false, fmt.Errorf("opentelemetry volume mount not found on deployment %s", dType) } if !volumeFound { - return false, fmt.Errorf("Opentelemetry volume not found on dc %s", dcType) + return false, fmt.Errorf("opentelemetry volume not found on deployment %s", dType) } // All environment variables are correctly set return true, nil } -func opentelemetryEnvExistsWithCustomKeysOnBothDCs(dcType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { - dc := &appsv1.DeploymentConfig{} +func opentelemetryEnvExistsWithCustomKeysOnBothDeployments(dType string, desiredConfigValue string, client k8sclient.WithWatch) (bool, error) { + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: "apicast-staging", Namespace: namespace, } - err := client.Get(context.TODO(), namespacedName, dc) + err := client.Get(context.TODO(), namespacedName, deployment) if err != nil { - return false, fmt.Errorf("error fetching object %s: %v", dcType, err) + return false, fmt.Errorf("error fetching object %s: %v", dType, err) } - stageEnvs := dc.Spec.Template.Spec.Containers[0].Env + stageEnvs := deployment.Spec.Template.Spec.Containers[0].Env var ( stageConfigEnvfound bool stageConfigEnvValueCorrect bool @@ -1511,30 +1516,30 @@ func opentelemetryEnvExistsWithCustomKeysOnBothDCs(dcType string, desiredConfigV // Check if required environment variables are present and have correct values if !stageConfigEnvfound { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on dc stage %s", err) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on deployment stage %s", err) } if !stageOpentelemtryEnabledEnvFound { - return false, fmt.Errorf("OPENTELEMTRY env not found on dc stage %s", err) + return false, fmt.Errorf("OPENTELEMTRY env not found on deployment stage %s", err) } if !stageConfigEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on dc stage %s", err) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on deployment stage %s", err) } if !stageOpentelemtryEnabledEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY env value not correct on dc stage %s", err) + return false, fmt.Errorf("OPENTELEMTRY env value not correct on deployment stage %s", err) } - dc = &appsv1.DeploymentConfig{} + deployment = &k8sappsv1.Deployment{} namespacedName = types.NamespacedName{ Name: "apicast-production", Namespace: namespace, } - err = client.Get(context.TODO(), namespacedName, dc) + err = client.Get(context.TODO(), namespacedName, deployment) if err != nil { - return false, fmt.Errorf("error fetching object %s: %v", dcType, err) + return false, fmt.Errorf("error fetching object %s: %v", dType, err) } - prodEnvs := dc.Spec.Template.Spec.Containers[0].Env + prodEnvs := deployment.Spec.Template.Spec.Containers[0].Env var ( prodConfigEnvfound bool prodConfigEnvValueCorrect bool @@ -1560,16 +1565,16 @@ func opentelemetryEnvExistsWithCustomKeysOnBothDCs(dcType string, desiredConfigV // Check if required environment variables are present and have correct values if !prodConfigEnvfound { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on dc production %s", err) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG not found on deployment production %s", err) } if !prodOpentelemtryEnabledEnvFound { - return false, fmt.Errorf("OPENTELEMTRY env not found on dc production %s", err) + return false, fmt.Errorf("OPENTELEMTRY env not found on deployment production %s", err) } if !prodConfigEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on dc production %s", err) + return false, fmt.Errorf("OPENTELEMTRY_CONFIG env value not correct on deployment production %s", err) } if !prodOpentelemtryEnabledEnvValueCorrect { - return false, fmt.Errorf("OPENTELEMTRY env value not correct on dc production %s", err) + return false, fmt.Errorf("OPENTELEMTRY env value not correct on deployment production %s", err) } // All environment variables are correctly set diff --git a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go index 732d54a05..3d10f05f0 100644 --- a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go +++ b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go @@ -14,6 +14,7 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -63,6 +64,10 @@ func (r *BaseAPIManagerLogicReconciler) ReconcileDeploymentConfig(desired *appsv return r.ReconcileResource(&appsv1.DeploymentConfig{}, desired, mutatefn) } +func (r *BaseAPIManagerLogicReconciler) ReconcileDeployment(desired *k8sappsv1.Deployment, mutatefn reconcilers.MutateFn) error { + return r.ReconcileResource(&k8sappsv1.Deployment{}, desired, mutatefn) +} + func (r *BaseAPIManagerLogicReconciler) ReconcileService(desired *v1.Service, mutateFn reconcilers.MutateFn) error { return r.ReconcileResource(&v1.Service{}, desired, mutateFn) } diff --git a/pkg/helper/deployment.go b/pkg/helper/deployment.go new file mode 100644 index 000000000..c759918af --- /dev/null +++ b/pkg/helper/deployment.go @@ -0,0 +1,18 @@ +package helper + +import ( + k8sappsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// IsDeploymentAvailable returns true when the provided Deployment +// has the "Available" condition set to true +func IsDeploymentAvailable(d *k8sappsv1.Deployment) bool { + dConditions := d.Status.Conditions + for _, dCondition := range dConditions { + if dCondition.Type == k8sappsv1.DeploymentAvailable && dCondition.Status == corev1.ConditionTrue { + return true + } + } + return false +} diff --git a/pkg/reconcilers/deployment.go b/pkg/reconcilers/deployment.go new file mode 100644 index 000000000..9a2395333 --- /dev/null +++ b/pkg/reconcilers/deployment.go @@ -0,0 +1,290 @@ +package reconcilers + +import ( + "encoding/json" + "fmt" + "reflect" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + k8sappsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/3scale/3scale-operator/pkg/common" + "github.com/3scale/3scale-operator/pkg/helper" +) + +const ( + DeploymentKind = "Deployment" + DeploymentAPIVersion = "apps/v1" + DeploymentLabelSelector = "deployment" + DeploymentImageTriggerAnnotation = "image.openshift.io/triggers" +) + +type ContainerImage struct { + Name string + Tag string +} + +type ImageTriggerFrom struct { + Kind string `json:"kind"` + Name string `json:"name"` + // +optional + Namespace *string `json:"namespace,omitempty"` +} + +type ImageTrigger struct { + From ImageTriggerFrom `json:"from"` + FieldPath string `json:"fieldPath"` + // +optional + Paused bool `json:"paused,omitempty"` +} + +// DMutateFn is a function which mutates the existing Deployment into it's desired state. +type DMutateFn func(desired, existing *k8sappsv1.Deployment) (bool, error) + +func DeploymentMutator(opts ...DMutateFn) MutateFn { + return func(existingObj, desiredObj common.KubernetesObject) (bool, error) { + existing, ok := existingObj.(*k8sappsv1.Deployment) + if !ok { + return false, fmt.Errorf("%T is not a *k8sappsv1.Deployment", existingObj) + } + desired, ok := desiredObj.(*k8sappsv1.Deployment) + if !ok { + return false, fmt.Errorf("%T is not a *k8sappsv1.Deployment", desiredObj) + } + + update := false + + // Loop through each option + for _, opt := range opts { + tmpUpdate, err := opt(desired, existing) + if err != nil { + return false, err + } + update = update || tmpUpdate + } + + return update, nil + } +} + +// GenericBackendDeploymentMutators returns the generic mutators for backend +func GenericBackendDeploymentMutators() []DMutateFn { + return []DMutateFn{ + DeploymentAnnotationsMutator, + DeploymentContainerResourcesMutator, + DeploymentAffinityMutator, + DeploymentTolerationsMutator, + DeploymentPodTemplateLabelsMutator, + DeploymentPriorityClassMutator, + DeploymentTopologySpreadConstraintsMutator, + DeploymentPodTemplateAnnotationsMutator, + } +} + +// DeploymentAnnotationsMutator ensures Deployment Annotations are reconciled +func DeploymentAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + helper.MergeMapStringString(&updated, &existing.ObjectMeta.Annotations, desired.ObjectMeta.Annotations) + + return updated, nil +} + +func DeploymentReplicasMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + update := false + + if desired.Spec.Replicas != existing.Spec.Replicas { + existing.Spec.Replicas = desired.Spec.Replicas + update = true + } + + return update, nil +} + +func DeploymentAffinityMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + if !reflect.DeepEqual(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity) { + diff := cmp.Diff(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity) + log.Info(fmt.Sprintf("%s spec.template.spec.Affinity has changed: %s", common.ObjectInfo(desired), diff)) + existing.Spec.Template.Spec.Affinity = desired.Spec.Template.Spec.Affinity + updated = true + } + + return updated, nil +} + +func DeploymentTolerationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + if !reflect.DeepEqual(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations) { + diff := cmp.Diff(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations) + log.Info(fmt.Sprintf("%s spec.template.spec.Tolerations has changed: %s", common.ObjectInfo(desired), diff)) + existing.Spec.Template.Spec.Tolerations = desired.Spec.Template.Spec.Tolerations + updated = true + } + + return updated, nil +} + +func DeploymentContainerResourcesMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + desiredName := common.ObjectInfo(desired) + update := false + + if len(desired.Spec.Template.Spec.Containers) != 1 { + return false, fmt.Errorf("%s desired spec.template.spec.containers length changed to '%d', should be 1", desiredName, len(desired.Spec.Template.Spec.Containers)) + } + + if len(existing.Spec.Template.Spec.Containers) != 1 { + log.Info(fmt.Sprintf("%s spec.template.spec.containers length changed to '%d', recreating dc", desiredName, len(existing.Spec.Template.Spec.Containers))) + existing.Spec.Template.Spec.Containers = desired.Spec.Template.Spec.Containers + update = true + } + + if !helper.CmpResources(&existing.Spec.Template.Spec.Containers[0].Resources, &desired.Spec.Template.Spec.Containers[0].Resources) { + diff := cmp.Diff(existing.Spec.Template.Spec.Containers[0].Resources, desired.Spec.Template.Spec.Containers[0].Resources, cmpopts.IgnoreUnexported(resource.Quantity{})) + log.Info(fmt.Sprintf("%s spec.template.spec.containers[0].resources have changed: %s", desiredName, diff)) + existing.Spec.Template.Spec.Containers[0].Resources = desired.Spec.Template.Spec.Containers[0].Resources + update = true + } + + return update, nil +} + +// DeploymentEnvVarReconciler implements basic env var reconciliation deployments. +// Existing and desired DC must have same number of containers +// Added when in desired and not in existing +// Updated when in desired and in existing but not equal +// Removed when not in desired and exists in existing DC +func DeploymentEnvVarReconciler(desired, existing *k8sappsv1.Deployment, envVar string) bool { + updated := false + + if len(desired.Spec.Template.Spec.Containers) != len(existing.Spec.Template.Spec.Containers) { + log.Info("[WARNING] not reconciling deployment", + "name", client.ObjectKeyFromObject(desired), + "reason", "existing and desired do not have same number of containers") + return false + } + + if len(desired.Spec.Template.Spec.InitContainers) != len(existing.Spec.Template.Spec.InitContainers) { + log.Info("[WARNING] not reconciling deployment", + "name", client.ObjectKeyFromObject(desired), + "reason", "existing and desired do not have same number of init containers") + return false + } + + // Init Containers + for idx := range existing.Spec.Template.Spec.InitContainers { + tmpChanged := helper.EnvVarReconciler( + desired.Spec.Template.Spec.InitContainers[idx].Env, + &existing.Spec.Template.Spec.InitContainers[idx].Env, + envVar) + updated = updated || tmpChanged + } + + // Containers + for idx := range existing.Spec.Template.Spec.Containers { + tmpChanged := helper.EnvVarReconciler( + desired.Spec.Template.Spec.Containers[idx].Env, + &existing.Spec.Template.Spec.Containers[idx].Env, + envVar) + updated = updated || tmpChanged + } + + return updated +} + +// DeploymentPodTemplateLabelsMutator ensures pod template labels are reconciled +func DeploymentPodTemplateLabelsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + helper.MergeMapStringString(&updated, &existing.Spec.Template.Labels, desired.Spec.Template.Labels) + + return updated, nil +} + +// DeploymentRemoveDuplicateEnvVarMutator ensures pod env vars are not duplicated +func DeploymentRemoveDuplicateEnvVarMutator(_, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + for idx := range existing.Spec.Template.Spec.Containers { + prunedEnvs := helper.RemoveDuplicateEnvVars(existing.Spec.Template.Spec.Containers[idx].Env) + if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[idx].Env, prunedEnvs) { + existing.Spec.Template.Spec.Containers[idx].Env = prunedEnvs + updated = true + } + } + + return updated, nil +} + +// DeploymentPriorityClassMutator ensures priorityclass is reconciled +func DeploymentPriorityClassMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + if existing.Spec.Template.Spec.PriorityClassName != desired.Spec.Template.Spec.PriorityClassName { + existing.Spec.Template.Spec.PriorityClassName = desired.Spec.Template.Spec.PriorityClassName + updated = true + } + + return updated, nil +} + +// DeploymentStrategyMutator ensures desired strategy +func DeploymentStrategyMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + if !reflect.DeepEqual(existing.Spec.Strategy, desired.Spec.Strategy) { + existing.Spec.Strategy = desired.Spec.Strategy + updated = true + } + + return updated, nil +} + +// DeploymentTopologySpreadConstraintsMutator ensures TopologySpreadConstraints is reconciled +func DeploymentTopologySpreadConstraintsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + if !reflect.DeepEqual(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints) { + diff := cmp.Diff(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints) + log.Info(fmt.Sprintf("%s spec.template.spec.TopologySpreadConstraints has changed: %s", common.ObjectInfo(desired), diff)) + existing.Spec.Template.Spec.TopologySpreadConstraints = desired.Spec.Template.Spec.TopologySpreadConstraints + updated = true + } + + return updated, nil +} + +// DeploymentPodTemplateAnnotationsMutator ensures Pod Template Annotations is reconciled +func DeploymentPodTemplateAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + helper.MergeMapStringString(&updated, &existing.Spec.Template.Annotations, desired.Spec.Template.Annotations) + + return updated, nil +} + +// CreateImageTriggerAnnotationString creates the annotation for Deployments to leverage existing ImageStreamTags +func CreateImageTriggerAnnotationString(containers []ContainerImage) string { + var triggers []ImageTrigger + + for _, container := range containers { + triggers = append(triggers, ImageTrigger{ + From: ImageTriggerFrom{ + Kind: "ImageStreamTag", + Name: container.Tag, + }, + FieldPath: fmt.Sprintf(`spec.template.spec.containers[?(@.name=="%v")].image`, container.Name), + }) + } + + if len(triggers) > 0 { + jsonData, _ := json.Marshal(triggers) + return string(jsonData) + } + + return "" +} diff --git a/pkg/reconcilers/deployment_test.go b/pkg/reconcilers/deployment_test.go new file mode 100644 index 000000000..be54892be --- /dev/null +++ b/pkg/reconcilers/deployment_test.go @@ -0,0 +1,916 @@ +package reconcilers + +import ( + "reflect" + "testing" + + "github.com/3scale/3scale-operator/pkg/helper" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + k8sappsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestDeploymentReplicasMutator(t *testing.T) { + numReplicas := int32(3) + dFactory := func() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + }, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &numReplicas, + }, + } + } + + cases := []struct { + testName string + desired func() *k8sappsv1.Deployment + expectedResult bool + }{ + {"NothingToReconcile", func() *k8sappsv1.Deployment { return dFactory() }, false}, + {"ReplicasReconcile", + func() *k8sappsv1.Deployment { + desired := dFactory() + newNumReplicas := *desired.Spec.Replicas + int32(1000) + desired.Spec.Replicas = &newNumReplicas + return desired + }, true, + }, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory() + update, err := DeploymentReplicasMutator(tc.desired(), existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if *existing.Spec.Replicas != *tc.desired().Spec.Replicas { + subT.Fatalf("replica reconciliation failed, existing: %d, desired: %d", existing.Spec.Replicas, tc.desired().Spec.Replicas) + } + + }) + } +} + +func TestDeploymentContainerResourcesMutator(t *testing.T) { + emptyResourceRequirements := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{}, + Requests: corev1.ResourceList{}, + } + notEmptyResources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("110Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("220Mi"), + }, + } + dFactory := func(resources corev1.ResourceRequirements) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + }, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container1", + Resources: resources, + }, + }, + }, + }, + }, + } + } + + cases := []struct { + testName string + existingResources corev1.ResourceRequirements + desiredResources corev1.ResourceRequirements + expectedResult bool + }{ + {"NothingToReconcile", emptyResourceRequirements, emptyResourceRequirements, false}, + {"NothingToReconcileWithResources", notEmptyResources, notEmptyResources, false}, + {"AddResources", emptyResourceRequirements, notEmptyResources, true}, + {"RemoveResources", notEmptyResources, emptyResourceRequirements, true}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingResources) + desired := dFactory(tc.desiredResources) + update, err := DeploymentContainerResourcesMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !helper.CmpResources(&existing.Spec.Template.Spec.Containers[0].Resources, &desired.Spec.Template.Spec.Containers[0].Resources) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Containers[0].Resources, desired.Spec.Template.Spec.Containers[0].Resources, cmpopts.IgnoreUnexported(resource.Quantity{}))) + } + }) + } +} + +func TestDeploymentAffinityMutator(t *testing.T) { + testAffinity1 := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchFields: []corev1.NodeSelectorRequirement{ + { + Key: "key1", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"val1"}, + }, + }, + }, + }, + }, + }, + } + testAffinity2 := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchFields: []corev1.NodeSelectorRequirement{ + { + Key: "key2", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"val2"}, + }, + }, + }, + }, + }, + }, + } + dFactory := func(affinity *corev1.Affinity) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + }, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Affinity: affinity, + }, + }, + }, + } + } + + cases := []struct { + testName string + existingAffinity *corev1.Affinity + desiredAffinity *corev1.Affinity + expectedResult bool + }{ + {"NothingToReconcile", nil, nil, false}, + {"EqualAffinities", testAffinity1, testAffinity1, false}, + {"DifferentAffinities", testAffinity1, testAffinity2, true}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingAffinity) + desired := dFactory(tc.desiredAffinity) + update, err := DeploymentAffinityMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity)) + } + }) + } +} + +func TestDeploymentTolerationsMutator(t *testing.T) { + testTolerations1 := []corev1.Toleration{ + { + Key: "key1", + Effect: corev1.TaintEffectNoExecute, + Operator: corev1.TolerationOpEqual, + Value: "val1", + }, + { + Key: "key2", + Effect: corev1.TaintEffectNoExecute, + Operator: corev1.TolerationOpEqual, + Value: "val2", + }, + } + testTolerations2 := []corev1.Toleration{ + { + Key: "key3", + Effect: corev1.TaintEffectNoExecute, + Operator: corev1.TolerationOpEqual, + Value: "val3", + }, + { + Key: "key4", + Effect: corev1.TaintEffectNoExecute, + Operator: corev1.TolerationOpEqual, + Value: "val4", + }, + } + dFactory := func(toleration []corev1.Toleration) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + }, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Tolerations: toleration, + }, + }, + }, + } + } + + cases := []struct { + testName string + existingTolerations []corev1.Toleration + desiredTolerations []corev1.Toleration + expectedResult bool + }{ + {"NothingToReconcile", nil, nil, false}, + {"EqualAffinities", testTolerations1, testTolerations1, false}, + {"DifferentAffinities", testTolerations1, testTolerations2, true}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingTolerations) + desired := dFactory(tc.desiredTolerations) + update, err := DeploymentTolerationsMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations)) + } + }) + } + +} + +func TestDeploymentEnvVarReconciler(t *testing.T) { + t.Run("DifferentNumberOfContainers", func(subT *testing.T) { + desired := &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container1", + }, + }, + }, + }, + }, + } + existing := &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container1", + }, + { + Name: "container2", + }, + }, + }, + }, + }, + } + + update := DeploymentEnvVarReconciler(desired, existing, "A") + if update { + subT.Fatal("expected not to be updated") + } + }) + + t.Run("DifferentNumberOfInitContainers", func(subT *testing.T) { + desired := &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container1", + }, + }, + InitContainers: []corev1.Container{ + { + Name: "initcontainer1", + }, + }, + }, + }, + }, + } + existing := &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container1", + }, + }, + InitContainers: []corev1.Container{ + { + Name: "initcontainer1", + }, + { + Name: "initcontainer2", + }, + }, + }, + }, + }, + } + + update := DeploymentEnvVarReconciler(desired, existing, "A") + if update { + subT.Fatal("expected not to be updated") + } + }) + + t.Run("ContainersEnvVarReconciled", func(subT *testing.T) { + desired := &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container1", + Env: []corev1.EnvVar{ + {Name: "A", Value: "valueA"}, + }, + }, + { + Name: "container2", + Env: []corev1.EnvVar{}, + }, + }, + }, + }, + }, + } + existing := &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container1", + Env: []corev1.EnvVar{}, + }, + { + Name: "container2", + Env: []corev1.EnvVar{ + {Name: "A", Value: "valueA"}, + }, + }, + }, + }, + }, + }, + } + + update := DeploymentEnvVarReconciler(desired, existing, "A") + if !update { + subT.Fatal("expected not be updated") + } + + for i := range []int{0, 1} { + if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[i].Env, desired.Spec.Template.Spec.Containers[i].Env) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Containers[i].Env, desired.Spec.Template.Spec.Containers[i].Env)) + } + } + + }) + + t.Run("InitContainersEnvVarReconciled", func(subT *testing.T) { + desired := &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "intcontainer1", + Env: []corev1.EnvVar{ + {Name: "A", Value: "valueA"}, + }, + }, + { + Name: "intcontainer2", + Env: []corev1.EnvVar{}, + }, + }, + }, + }, + }, + } + existing := &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: "myDeployment", Namespace: "myNS"}, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "initcontainer1", + Env: []corev1.EnvVar{}, + }, + { + Name: "initcontainer2", + Env: []corev1.EnvVar{ + {Name: "A", Value: "valueA"}, + }, + }, + }, + }, + }, + }, + } + + update := DeploymentEnvVarReconciler(desired, existing, "A") + if !update { + subT.Fatal("expected not be updated") + } + + for i := range []int{0, 1} { + if !reflect.DeepEqual(existing.Spec.Template.Spec.InitContainers[i].Env, desired.Spec.Template.Spec.InitContainers[i].Env) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.InitContainers[i].Env, desired.Spec.Template.Spec.InitContainers[i].Env)) + } + } + }) +} + +func TestDeploymentPodTemplateLabelsMutator(t *testing.T) { + dFactory := func(labels map[string]string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + }, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + } + } + + mapCopy := func(originalMap map[string]string) map[string]string { + // Create the target map + targetMap := make(map[string]string) + + // Copy from the original map to the target map + for key, value := range originalMap { + targetMap[key] = value + } + + return targetMap + } + + labelsA := map[string]string{"a": "1", "a2": "2"} + labelsB := map[string]string{"a": "other", "b": "1"} + + cases := []struct { + testName string + existingLabels map[string]string + desiredLabels map[string]string + expectedResult bool + expectedNewLabels map[string]string + }{ + {"NothingToReconcile", mapCopy(labelsA), mapCopy(labelsA), false, mapCopy(labelsA)}, + {"LabelsReconciled", mapCopy(labelsB), mapCopy(labelsA), true, map[string]string{ + "a": "1", "a2": "2", "b": "1", + }}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingLabels) + desired := dFactory(tc.desiredLabels) + update, err := DeploymentPodTemplateLabelsMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.Spec.Template.Labels, tc.expectedNewLabels) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Labels, tc.expectedNewLabels)) + } + }) + } +} + +func TestDeploymentRemoveDuplicateEnvVarMutator(t *testing.T) { + dFactory := func(envs []corev1.EnvVar) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + }, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container1", + Env: envs, + }, + { + Name: "container2", + Env: envs, + }, + }, + }, + }, + }, + } + } + + envsA := []corev1.EnvVar{{Name: "a", Value: "1"}, {Name: "b", Value: "2"}} + envsB := []corev1.EnvVar{{Name: "a", Value: "1"}, {Name: "a", Value: "1"}, {Name: "b", Value: "2"}} + + cases := []struct { + testName string + existingEnvs []corev1.EnvVar + expectedResult bool + expectedNewEnvs []corev1.EnvVar + }{ + {"NothingToReconcile", envsA, false, envsA}, + {"EnvsReconciled", envsB, true, envsA}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingEnvs) + update, err := DeploymentRemoveDuplicateEnvVarMutator(nil, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + for idx := range existing.Spec.Template.Spec.Containers { + if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[idx].Env, tc.expectedNewEnvs) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Containers[idx].Env, tc.expectedNewEnvs)) + } + } + }) + } +} + +func TestDeploymentTopologySpreadConstraintsMutator(t *testing.T) { + testTopologySpreadConstraint1 := []corev1.TopologySpreadConstraint{ + { + TopologyKey: "topologyKey1", + WhenUnsatisfiable: "DoNotSchedule", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "3scale-api-management"}, + }, + }, + } + testTopologySpreadConstraint2 := []corev1.TopologySpreadConstraint{ + { + TopologyKey: "topologyKey2", + WhenUnsatisfiable: "ScheduleAnyway", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "3scale-api-management", "threescale_component": "system"}, + }, + }, + } + + dFactory := func(topologySpreadConstraint []corev1.TopologySpreadConstraint) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + }, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + TopologySpreadConstraints: topologySpreadConstraint, + }, + }, + }, + } + } + + cases := []struct { + testName string + existingTopologySpreadConstraints []corev1.TopologySpreadConstraint + desiredTopologySpreadConstraints []corev1.TopologySpreadConstraint + expectedResult bool + }{ + {"NothingToReconcile", nil, nil, false}, + {"EqualTopologies", testTopologySpreadConstraint1, testTopologySpreadConstraint1, false}, + {"DifferentTopologie", testTopologySpreadConstraint1, testTopologySpreadConstraint2, true}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingTopologySpreadConstraints) + desired := dFactory(tc.desiredTopologySpreadConstraints) + update, err := DeploymentTopologySpreadConstraintsMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints)) + } + }) + } + +} + +func TestDeploymentPodTemplateAnnotationsMutator(t *testing.T) { + dFactory := func(annotations map[string]string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + }, + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + }, + }, + } + } + + mapCopy := func(originalMap map[string]string) map[string]string { + // Create the target map + targetMap := make(map[string]string) + + // Copy from the original map to the target map + for key, value := range originalMap { + targetMap[key] = value + } + + return targetMap + } + + annotationsA := map[string]string{"a": "1", "a2": "2"} + annotationsB := map[string]string{"a": "other", "b": "1"} + + cases := []struct { + testName string + existingAnnotations map[string]string + desiredAnnotations map[string]string + expectedResult bool + expectedNewAnnotations map[string]string + }{ + {"NothingToReconcile", mapCopy(annotationsA), mapCopy(annotationsA), false, mapCopy(annotationsA)}, + {"AnnotationsReconciled", mapCopy(annotationsB), mapCopy(annotationsA), true, map[string]string{ + "a": "1", "a2": "2", "b": "1", + }}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingAnnotations) + desired := dFactory(tc.desiredAnnotations) + update, err := DeploymentPodTemplateAnnotationsMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.Spec.Template.Annotations, tc.expectedNewAnnotations) { + subT.Fatal(cmp.Diff(existing.Spec.Template.Annotations, tc.expectedNewAnnotations)) + } + }) + } +} + +func TestDeploymentAnnotationsMutator(t *testing.T) { + dFactory := func(annotations map[string]string) *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps.openshift.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "myDeployment", + Namespace: "myNS", + Annotations: annotations, + }, + } + } + + mapCopy := func(originalMap map[string]string) map[string]string { + // Create the target map + targetMap := make(map[string]string) + + // Copy from the original map to the target map + for key, value := range originalMap { + targetMap[key] = value + } + + return targetMap + } + + annotationsA := map[string]string{"a": "1", "a2": "2"} + annotationsB := map[string]string{"a": "other", "b": "1"} + + cases := []struct { + testName string + existingAnnotations map[string]string + desiredAnnotations map[string]string + expectedResult bool + expectedNewAnnotations map[string]string + }{ + {"NothingToReconcile", mapCopy(annotationsA), mapCopy(annotationsA), false, mapCopy(annotationsA)}, + {"AnnotationsReconciled", mapCopy(annotationsB), mapCopy(annotationsA), true, map[string]string{ + "a": "1", "a2": "2", "b": "1", + }}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := dFactory(tc.existingAnnotations) + desired := dFactory(tc.desiredAnnotations) + update, err := DeploymentAnnotationsMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.ObjectMeta.Annotations, tc.expectedNewAnnotations) { + subT.Fatal(cmp.Diff(existing.ObjectMeta.Annotations, tc.expectedNewAnnotations)) + } + }) + } +} + +func TestCreateImageTriggerAnnotationString(t *testing.T) { + type args struct { + containers []ContainerImage + } + tests := []struct { + name string + args args + want string + }{ + { + name: "NoContainers", + args: args{ + containers: []ContainerImage{}, + }, + want: "", + }, + { + name: "OneContainer", + args: args{ + containers: []ContainerImage{ + { + Name: "container1", + Tag: "test-image:1.1", + }, + }, + }, + want: `[{"from":{"kind":"ImageStreamTag","name":"test-image:1.1"},"fieldPath":"spec.template.spec.containers[?(@.name==\"container1\")].image"}]`, + }, + { + name: "TwoContainers", + args: args{ + containers: []ContainerImage{ + { + Name: "container1", + Tag: "test-image:1.1", + }, + { + Name: "container2", + Tag: "test-image:2.2", + }, + }, + }, + want: `[{"from":{"kind":"ImageStreamTag","name":"test-image:1.1"},"fieldPath":"spec.template.spec.containers[?(@.name==\"container1\")].image"},{"from":{"kind":"ImageStreamTag","name":"test-image:2.2"},"fieldPath":"spec.template.spec.containers[?(@.name==\"container2\")].image"}]`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CreateImageTriggerAnnotationString(tt.args.containers); got != tt.want { + t.Errorf("CreateImageTriggerAnnotationString() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/reconcilers/service.go b/pkg/reconcilers/service.go index 9cb12b8d9..f3bbcc320 100644 --- a/pkg/reconcilers/service.go +++ b/pkg/reconcilers/service.go @@ -8,6 +8,32 @@ import ( v1 "k8s.io/api/core/v1" ) +func ServiceMutator(opts ...MutateFn) MutateFn { + return func(existingObj, desiredObj common.KubernetesObject) (bool, error) { + existing, ok := existingObj.(*v1.Service) + if !ok { + return false, fmt.Errorf("%T is not a *v1.Service", existingObj) + } + desired, ok := desiredObj.(*v1.Service) + if !ok { + return false, fmt.Errorf("%T is not a *v1.Service", desiredObj) + } + + update := false + + // Loop through each option + for _, opt := range opts { + tmpUpdate, err := opt(existing, desired) + if err != nil { + return false, err + } + update = update || tmpUpdate + } + + return update, nil + } +} + func ServicePortMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { existing, ok := existingObj.(*v1.Service) if !ok { @@ -27,3 +53,23 @@ func ServicePortMutator(existingObj, desiredObj common.KubernetesObject) (bool, return updated, nil } + +func ServiceSelectorMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { + existing, ok := existingObj.(*v1.Service) + if !ok { + return false, fmt.Errorf("%T is not a *v1.Service", existingObj) + } + desired, ok := desiredObj.(*v1.Service) + if !ok { + return false, fmt.Errorf("%T is not a *v1.Service", desiredObj) + } + + updated := false + + if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) { + updated = true + existing.Spec.Selector = desired.Spec.Selector + } + + return updated, nil +} diff --git a/pkg/upgrade/migrate_deployment_config_2.15.go b/pkg/upgrade/migrate_deployment_config_2.15.go new file mode 100644 index 000000000..f29bdecb1 --- /dev/null +++ b/pkg/upgrade/migrate_deployment_config_2.15.go @@ -0,0 +1,57 @@ +package upgrade + +import ( + "context" + "fmt" + "github.com/3scale/3scale-operator/pkg/helper" + appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MigrateDeploymentConfigToDeployment verifies the Deployment is healthy and then deletes the corresponding DeploymentConfig +// 3scale 2.14 -> 2.15 +func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, client k8sclient.Client) (bool, error) { + deploymentConfig := &appsv1.DeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: dName, + Namespace: dNamespace, + }, + } + err := client.Get(context.TODO(), k8sclient.ObjectKey{Name: deploymentConfig.Name, Namespace: deploymentConfig.Namespace}, deploymentConfig) + + // Breakout if the DeploymentConfig has already been deleted + if k8serr.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, fmt.Errorf("error getting deploymentconfig %s: %v", deploymentConfig.Name, err) + } + + // Verify that the Deployment is healthy + deployment := &k8sappsv1.Deployment{} + err = client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: dNamespace, + Name: dName, + }, deployment) + if err != nil { + return false, fmt.Errorf("error getting deployment %s: %v", deployment.Name, err) + } + if !helper.IsDeploymentAvailable(deployment) { + log.V(1).Info(fmt.Sprintf("deployment %s is not yet available", deployment.Name)) + return false, nil + } + + // Delete the DeploymentConfig because the Deployment replacing it is healthy + err = client.Delete(context.TODO(), deploymentConfig) + if err != nil { + if !k8serr.IsNotFound(err) { + return false, fmt.Errorf("error deleting deploymentconfig %s: %v", deploymentConfig.Name, err) + } + } + + log.Info(fmt.Sprintf("%s Deployment has replaced its corresponding DeploymentConfig", deployment.Name)) + return true, nil +} From 34a44ca9cd4d5b900d46845f8d72be165041e923 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Thu, 16 Nov 2023 11:18:36 -0500 Subject: [PATCH 02/23] THREESCALE-10388 Updated APIManager to set status from Deployments --- controllers/apps/apimanager_controller.go | 4 +-- .../apps/apimanager_status_reconciler.go | 32 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/controllers/apps/apimanager_controller.go b/controllers/apps/apimanager_controller.go index 46485f9fc..caeb00a14 100644 --- a/controllers/apps/apimanager_controller.go +++ b/controllers/apps/apimanager_controller.go @@ -19,8 +19,8 @@ package controllers import ( "context" "fmt" + k8sappsv1 "k8s.io/api/apps/v1" - appsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" v1 "k8s.io/api/core/v1" @@ -154,7 +154,7 @@ func (r *APIManagerReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(secretToApimanagerEventMapper.Map), builder.WithPredicates(labelSelectorPredicate), ). - Owns(&appsv1.DeploymentConfig{}). + Owns(&k8sappsv1.Deployment{}). Watches(&source.Kind{Type: &routev1.Route{}}, handler.EnqueueRequestsFromMapFunc(handlers.Map)). Complete(r) } diff --git a/controllers/apps/apimanager_status_reconciler.go b/controllers/apps/apimanager_status_reconciler.go index 26f18742d..e89c69538 100644 --- a/controllers/apps/apimanager_status_reconciler.go +++ b/controllers/apps/apimanager_status_reconciler.go @@ -3,6 +3,7 @@ package controllers import ( "context" "fmt" + k8sappsv1 "k8s.io/api/apps/v1" "sort" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -12,7 +13,6 @@ import ( "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/RHsyseng/operator-utils/pkg/olm" "github.com/go-logr/logr" - appsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -67,7 +67,7 @@ func (s *APIManagerStatusReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{Requeue: true}, nil } - return reconcile.Result{}, fmt.Errorf("Failed to update status: %w", updateErr) + return reconcile.Result{}, fmt.Errorf("failed to update status: %w", updateErr) } return reconcile.Result{}, nil } @@ -88,7 +88,7 @@ func (s *APIManagerStatusReconciler) calculateStatus() (*appsv1alpha1.APIManager } newStatus.Conditions.SetCondition(availableCondition) - deploymentStatus := olm.GetDeploymentConfigStatus(deployments) + deploymentStatus := olm.GetDeploymentStatus(deployments) newStatus.Deployments = deploymentStatus return newStatus, nil @@ -124,7 +124,7 @@ func (s *APIManagerStatusReconciler) expectedDeploymentNames(instance *appsv1alp return deploymentLister.DeploymentNames() } -func (s *APIManagerStatusReconciler) deploymentsAvailable(existingDeployments []appsv1.DeploymentConfig) bool { +func (s *APIManagerStatusReconciler) deploymentsAvailable(existingDeployments []k8sappsv1.Deployment) bool { expectedDeploymentNames := s.expectedDeploymentNames(s.apimanagerResource) for _, deploymentName := range expectedDeploymentNames { foundExistingDCIdx := -1 @@ -134,7 +134,7 @@ func (s *APIManagerStatusReconciler) deploymentsAvailable(existingDeployments [] break } } - if foundExistingDCIdx == -1 || !helper.IsDeploymentConfigAvailable(&existingDeployments[foundExistingDCIdx]) { + if foundExistingDCIdx == -1 || !helper.IsDeploymentAvailable(&existingDeployments[foundExistingDCIdx]) { return false } } @@ -142,13 +142,13 @@ func (s *APIManagerStatusReconciler) deploymentsAvailable(existingDeployments [] return true } -func (s *APIManagerStatusReconciler) existingDeployments() ([]appsv1.DeploymentConfig, error) { +func (s *APIManagerStatusReconciler) existingDeployments() ([]k8sappsv1.Deployment, error) { expectedDeploymentNames := s.expectedDeploymentNames(s.apimanagerResource) - var dcs []appsv1.DeploymentConfig - for _, dcName := range expectedDeploymentNames { - existingDeploymentConfig := &appsv1.DeploymentConfig{} - err := s.Client().Get(context.Background(), types.NamespacedName{Namespace: s.apimanagerResource.Namespace, Name: dcName}, existingDeploymentConfig) + var deployments []k8sappsv1.Deployment + for _, dName := range expectedDeploymentNames { + existingDeployment := &k8sappsv1.Deployment{} + err := s.Client().Get(context.Background(), types.NamespacedName{Namespace: s.apimanagerResource.Namespace, Name: dName}, existingDeployment) if err != nil && !errors.IsNotFound(err) { return nil, err } @@ -156,19 +156,19 @@ func (s *APIManagerStatusReconciler) existingDeployments() ([]appsv1.DeploymentC continue } - for _, ownerRef := range existingDeploymentConfig.GetOwnerReferences() { + for _, ownerRef := range existingDeployment.GetOwnerReferences() { if ownerRef.UID == s.apimanagerResource.UID { - dcs = append(dcs, *existingDeploymentConfig) + deployments = append(deployments, *existingDeployment) break } } } - sort.Slice(dcs, func(i, j int) bool { return dcs[i].Name < dcs[j].Name }) + sort.Slice(deployments, func(i, j int) bool { return deployments[i].Name < deployments[j].Name }) - return dcs, nil + return deployments, nil } -func (s *APIManagerStatusReconciler) apimanagerAvailableCondition(existingDeployments []appsv1.DeploymentConfig) (common.Condition, error) { +func (s *APIManagerStatusReconciler) apimanagerAvailableCondition(existingDeployments []k8sappsv1.Deployment) (common.Condition, error) { deploymentsAvailable := s.deploymentsAvailable(existingDeployments) defaultRoutesReady, err := s.defaultRoutesReady() @@ -207,7 +207,7 @@ func (s *APIManagerStatusReconciler) defaultRoutesReady() (bool, error) { routeList := &routev1.RouteList{} err := s.Client().List(context.TODO(), routeList, listOps...) if err != nil { - return false, fmt.Errorf("Failed to list routes: %w", err) + return false, fmt.Errorf("failed to list routes: %w", err) } routes := append([]routev1.Route(nil), routeList.Items...) From 688d1934eb4040acb2923eb780507aa896edae42 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Wed, 22 Nov 2023 14:57:12 -0500 Subject: [PATCH 03/23] THREESCALE-10389 Migrate backend DeploymentConfigs to Deployments --- pkg/3scale/amp/component/backend.go | 257 +++++++-------- .../amp/operator/backend_options_provider.go | 7 +- .../operator/backend_options_provider_test.go | 25 +- pkg/3scale/amp/operator/backend_reconciler.go | 59 +++- .../amp/operator/backend_reconciler_test.go | 60 ++-- pkg/reconcilers/deployment.go | 39 +++ pkg/reconcilers/deployment_test.go | 293 ++++++++++++++++++ pkg/reconcilers/deploymentconfig.go | 2 +- pkg/reconcilers/deploymentconfig_test.go | 250 +++++++-------- pkg/upgrade/migrate_deployment_config_2.15.go | 10 +- 10 files changed, 702 insertions(+), 300 deletions(-) diff --git a/pkg/3scale/amp/component/backend.go b/pkg/3scale/amp/component/backend.go index 07fcc6114..ce94bf952 100644 --- a/pkg/3scale/amp/component/backend.go +++ b/pkg/3scale/amp/component/backend.go @@ -5,9 +5,10 @@ import ( "strconv" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" - appsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,9 +16,10 @@ import ( ) const ( - BackendListenerName = "backend-listener" - BackendWorkerName = "backend-worker" - BackendCronName = "backend-cron" + BackendListenerName = "backend-listener" + BackendWorkerName = "backend-worker" + BackendCronName = "backend-cron" + BackendInitContainerName = "backend-redis-svc" ) const ( @@ -60,47 +62,36 @@ func NewBackend(options *BackendOptions) *Backend { return &Backend{Options: options} } -func (backend *Backend) WorkerDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (backend *Backend) WorkerDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: BackendWorkerName, - Labels: backend.Options.CommonWorkerLabels, - }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{1200}[0], + Name: BackendWorkerName, + Labels: backend.Options.CommonWorkerLabels, + Annotations: backend.backendWorkerDeploymentAnnotations(), + }, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Type: intstr.String, + StrVal: "25%", + }, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{"backend-redis-svc", BackendWorkerName}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-backend:%s", backend.Options.ImageTag)}}}, + Replicas: &backend.Options.WorkerReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: BackendWorkerName, + }, }, - Replicas: backend.Options.WorkerReplicas, - Selector: map[string]string{"deploymentConfig": BackendWorkerName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: backend.Options.WorkerPodTemplateLabels, Annotations: backend.Options.WorkerPodTemplateAnnotations, @@ -109,7 +100,7 @@ func (backend *Backend) WorkerDeploymentConfig() *appsv1.DeploymentConfig { Affinity: backend.Options.WorkerAffinity, Tolerations: backend.Options.WorkerTolerations, InitContainers: []v1.Container{ - v1.Container{ + { Name: "backend-redis-svc", Image: "amp-backend:latest", Command: []string{ @@ -117,11 +108,12 @@ func (backend *Backend) WorkerDeploymentConfig() *appsv1.DeploymentConfig { "sh", "-c", "until rake connectivity:redis_storage_queue_check; do sleep $SLEEP_SECONDS; done", - }, Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + }, + Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), }, }, Containers: []v1.Container{ - v1.Container{ + { Name: BackendWorkerName, Image: "amp-backend:latest", Args: []string{"bin/3scale_backend_worker", "run"}, @@ -147,52 +139,43 @@ func (backend *Backend) WorkerDeploymentConfig() *appsv1.DeploymentConfig { }, ServiceAccountName: "amp", PriorityClassName: backend.Options.PriorityClassNameWorker, - TopologySpreadConstraints: backend.Options.TopologySpreadConstraintsWorker}}, + TopologySpreadConstraints: backend.Options.TopologySpreadConstraintsWorker, + }, + }, }, } } -func (backend *Backend) CronDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (backend *Backend) CronDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: BackendCronName, - Labels: backend.Options.CommonCronLabels, - }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{1200}[0], + Name: BackendCronName, + Labels: backend.Options.CommonCronLabels, + Annotations: backend.backendCronDeploymentAnnotations(), + }, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Type: intstr.String, + StrVal: "25%", + }, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{"backend-redis-svc", "backend-cron"}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-backend:%s", backend.Options.ImageTag)}}}, + Replicas: &backend.Options.CronReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: BackendCronName, + }, }, - Replicas: backend.Options.CronReplicas, - Selector: map[string]string{"deploymentConfig": BackendCronName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: backend.Options.CronPodTemplateLabels, Annotations: backend.Options.CronPodTemplateAnnotations, @@ -201,7 +184,7 @@ func (backend *Backend) CronDeploymentConfig() *appsv1.DeploymentConfig { Affinity: backend.Options.CronAffinity, Tolerations: backend.Options.CronTolerations, InitContainers: []v1.Container{ - v1.Container{ + { Name: "backend-redis-svc", Image: "amp-backend:latest", Command: []string{ @@ -209,11 +192,12 @@ func (backend *Backend) CronDeploymentConfig() *appsv1.DeploymentConfig { "sh", "-c", "until rake connectivity:redis_storage_queue_check; do sleep $SLEEP_SECONDS; done", - }, Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + }, + Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), }, }, Containers: []v1.Container{ - v1.Container{ + { Name: "backend-cron", Image: "amp-backend:latest", Args: []string{"touch /tmp/healthy && backend-cron"}, @@ -234,52 +218,42 @@ func (backend *Backend) CronDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", PriorityClassName: backend.Options.PriorityClassNameCron, TopologySpreadConstraints: backend.Options.TopologySpreadConstraintsCron, - }}, + }, + }, }, } } -func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (backend *Backend) ListenerDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: BackendListenerName, - Labels: backend.Options.CommonListenerLabels, - }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{600}[0], + Name: BackendListenerName, + Labels: backend.Options.CommonListenerLabels, + Annotations: backend.backendListenerDeploymentAnnotations(), + }, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Type: intstr.String, + StrVal: "25%", + }, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{BackendListenerName}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-backend:%s", backend.Options.ImageTag)}}}, + Replicas: &backend.Options.ListenerReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: BackendListenerName, + }, }, - Replicas: backend.Options.ListenerReplicas, - Selector: map[string]string{"deploymentConfig": BackendListenerName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: backend.Options.ListenerPodTemplateLabels, Annotations: backend.Options.ListenerPodTemplateAnnotations, @@ -288,7 +262,7 @@ func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { Affinity: backend.Options.ListenerAffinity, Tolerations: backend.Options.ListenerTolerations, Containers: []v1.Container{ - v1.Container{ + { Name: BackendListenerName, Image: "amp-backend:latest", Args: []string{"bin/3scale_backend", "start", "-e", "production", "-p", "3000", "-x", "/dev/stdout"}, @@ -298,7 +272,7 @@ func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), + Type: intstr.Int, IntVal: 3000}}, }, InitialDelaySeconds: 30, @@ -311,7 +285,7 @@ func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ Path: "/status", Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), + Type: intstr.Int, IntVal: 3000}}, }, InitialDelaySeconds: 30, @@ -326,7 +300,8 @@ func (backend *Backend) ListenerDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", PriorityClassName: backend.Options.PriorityClassNameListener, TopologySpreadConstraints: backend.Options.TopologySpreadConstraintsListener, - }}, + }, + }, }, } } @@ -343,17 +318,17 @@ func (backend *Backend) ListenerService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "http", Protocol: v1.ProtocolTCP, Port: 3000, TargetPort: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), + Type: intstr.Int, IntVal: 3000, }, }, }, - Selector: map[string]string{"deploymentConfig": BackendListenerName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: BackendListenerName}, }, } } @@ -502,7 +477,7 @@ func (backend *Backend) WorkerPodDisruptionBudget() *policyv1.PodDisruptionBudge }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": BackendWorkerName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: BackendWorkerName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -521,7 +496,7 @@ func (backend *Backend) CronPodDisruptionBudget() *policyv1.PodDisruptionBudget }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": "backend-cron"}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: "backend-cron"}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -540,7 +515,7 @@ func (backend *Backend) ListenerPodDisruptionBudget() *policyv1.PodDisruptionBud }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": BackendListenerName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: BackendListenerName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -549,7 +524,7 @@ func (backend *Backend) ListenerPodDisruptionBudget() *policyv1.PodDisruptionBud func (backend *Backend) listenerPorts() []v1.ContainerPort { ports := []v1.ContainerPort{ - v1.ContainerPort{HostPort: 0, ContainerPort: 3000, Protocol: v1.ProtocolTCP}, + {HostPort: 0, ContainerPort: 3000, Protocol: v1.ProtocolTCP}, } if backend.Options.ListenerMetrics { @@ -568,3 +543,41 @@ func (backend *Backend) workerPorts() []v1.ContainerPort { return ports } + +func (backend *Backend) backendWorkerDeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: BackendInitContainerName, + Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), + }, + { + Name: BackendWorkerName, + Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} + +func (backend *Backend) backendCronDeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: BackendInitContainerName, + Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), + }, + { + Name: BackendCronName, + Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} + +func (backend *Backend) backendListenerDeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: BackendListenerName, + Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} diff --git a/pkg/3scale/amp/operator/backend_options_provider.go b/pkg/3scale/amp/operator/backend_options_provider.go index 742050cd6..0e75e8cfb 100644 --- a/pkg/3scale/amp/operator/backend_options_provider.go +++ b/pkg/3scale/amp/operator/backend_options_provider.go @@ -7,6 +7,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -197,7 +198,7 @@ func (o *OperatorBackendOptionsProvider) listenerPodTemplateLabels() map[string] labels[k] = v } - labels["deploymentConfig"] = "backend-listener" + labels[reconcilers.DeploymentLabelSelector] = "backend-listener" return labels } @@ -213,7 +214,7 @@ func (o *OperatorBackendOptionsProvider) workerPodTemplateLabels() map[string]st labels[k] = v } - labels["deploymentConfig"] = "backend-worker" + labels[reconcilers.DeploymentLabelSelector] = "backend-worker" return labels } @@ -229,7 +230,7 @@ func (o *OperatorBackendOptionsProvider) cronPodTemplateLabels() map[string]stri labels[k] = v } - labels["deploymentConfig"] = "backend-cron" + labels[reconcilers.DeploymentLabelSelector] = "backend-cron" return labels } diff --git a/pkg/3scale/amp/operator/backend_options_provider_test.go b/pkg/3scale/amp/operator/backend_options_provider_test.go index a350c1e71..e19207e50 100644 --- a/pkg/3scale/amp/operator/backend_options_provider_test.go +++ b/pkg/3scale/amp/operator/backend_options_provider_test.go @@ -9,6 +9,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" @@ -56,10 +57,10 @@ func testBackendCommonCronLabels() map[string]string { func testBackendListenerPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "backend", - "threescale_component_element": "listener", - "deploymentConfig": "backend-listener", + "app": appLabel, + "threescale_component": "backend", + "threescale_component_element": "listener", + reconcilers.DeploymentLabelSelector: "backend-listener", } addExpectedMeteringLabels(labels, "backend-listener", helper.ApplicationType) @@ -68,10 +69,10 @@ func testBackendListenerPodLabels() map[string]string { func testBackendWorkerPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "backend", - "threescale_component_element": "worker", - "deploymentConfig": "backend-worker", + "app": appLabel, + "threescale_component": "backend", + "threescale_component_element": "worker", + reconcilers.DeploymentLabelSelector: "backend-worker", } addExpectedMeteringLabels(labels, "backend-worker", helper.ApplicationType) @@ -80,10 +81,10 @@ func testBackendWorkerPodLabels() map[string]string { func testBackendCronPodLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "backend", - "threescale_component_element": "cron", - "deploymentConfig": "backend-cron", + "app": appLabel, + "threescale_component": "backend", + "threescale_component_element": "cron", + reconcilers.DeploymentLabelSelector: "backend-cron", } addExpectedMeteringLabels(labels, "backend-cron", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/backend_reconciler.go b/pkg/3scale/amp/operator/backend_reconciler.go index e962aabc2..cf9ea7658 100644 --- a/pkg/3scale/amp/operator/backend_reconciler.go +++ b/pkg/3scale/amp/operator/backend_reconciler.go @@ -5,6 +5,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -26,30 +27,53 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // Cron DC - cronConfigMutator := reconcilers.GenericBackendMutators() + // Cron Deployment + cronDeploymentMutator := reconcilers.GenericBackendDeploymentMutators() if r.apiManager.Spec.Backend.CronSpec.Replicas != nil { - cronConfigMutator = append(cronConfigMutator, reconcilers.DeploymentConfigReplicasMutator) + cronDeploymentMutator = append(cronDeploymentMutator, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(backend.CronDeploymentConfig(), reconcilers.DeploymentConfigMutator(cronConfigMutator...)) + err = r.ReconcileDeployment(backend.CronDeployment(), reconcilers.DeploymentMutator(cronDeploymentMutator...)) if err != nil { return reconcile.Result{}, err } - // Listener DC - listenerConfigMutator := reconcilers.GenericBackendMutators() + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.BackendCronName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + // Listener Deployment + listenerDeploymentMutator := reconcilers.GenericBackendDeploymentMutators() if r.apiManager.Spec.Backend.ListenerSpec.Replicas != nil { - listenerConfigMutator = append(listenerConfigMutator, reconcilers.DeploymentConfigReplicasMutator) + listenerDeploymentMutator = append(listenerDeploymentMutator, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(backend.ListenerDeploymentConfig(), reconcilers.DeploymentConfigMutator(listenerConfigMutator...)) + err = r.ReconcileDeployment(backend.ListenerDeployment(), reconcilers.DeploymentMutator(listenerDeploymentMutator...)) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendListenerName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + // Listener Service - err = r.ReconcileService(backend.ListenerService(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(backend.ListenerService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -60,16 +84,25 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // Worker DC - workerConfigMutator := reconcilers.GenericBackendMutators() + // Worker Deployment + workerDeploymentMutator := reconcilers.GenericBackendDeploymentMutators() if r.apiManager.Spec.Backend.WorkerSpec.Replicas != nil { - workerConfigMutator = append(workerConfigMutator, reconcilers.DeploymentConfigReplicasMutator) + workerDeploymentMutator = append(workerDeploymentMutator, reconcilers.DeploymentReplicasMutator) + } + + err = r.ReconcileDeployment(backend.WorkerDeployment(), reconcilers.DeploymentMutator(workerDeploymentMutator...)) + if err != nil { + return reconcile.Result{}, err } - err = r.ReconcileDeploymentConfig(backend.WorkerDeploymentConfig(), reconcilers.DeploymentConfigMutator(workerConfigMutator...)) + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendWorkerName, r.apiManager.GetNamespace(), r.Client()) if err != nil { return reconcile.Result{}, err } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } // Environment ConfigMap err = r.ReconcileConfigMap(backend.EnvironmentConfigMap(), reconcilers.CreateOnlyMutator) diff --git a/pkg/3scale/amp/operator/backend_reconciler_test.go b/pkg/3scale/amp/operator/backend_reconciler_test.go index 6bd8f4355..874c67900 100644 --- a/pkg/3scale/amp/operator/backend_reconciler_test.go +++ b/pkg/3scale/amp/operator/backend_reconciler_test.go @@ -2,6 +2,7 @@ package operator import ( "context" + k8sappsv1 "k8s.io/api/apps/v1" "testing" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -60,19 +61,31 @@ func TestNewBackendReconciler(t *testing.T) { PodDisruptionBudget: &appsv1alpha1.PodDisruptionBudgetSpec{Enabled: true}, }, } + + backendRedisSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backend-redis", + Namespace: namespace, + }, + } + // Objects to track in the fake client. - objs := []runtime.Object{apimanager} + objs := []runtime.Object{apimanager, backendRedisSecret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + err = imagev1.Install(s) if err != nil { t.Fatal(err) } - err = imagev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = configv1.Install(s) if err != nil { t.Fatal(err) } @@ -97,11 +110,11 @@ func TestNewBackendReconciler(t *testing.T) { objName string obj k8sclient.Object }{ - {"cronDC", "backend-cron", &appsv1.DeploymentConfig{}}, - {"listenerDC", "backend-listener", &appsv1.DeploymentConfig{}}, + {"cronDeployment", "backend-cron", &k8sappsv1.Deployment{}}, + {"listenerDeployment", "backend-listener", &k8sappsv1.Deployment{}}, {"listenerService", "backend-listener", &v1.Service{}}, {"listenerRoute", "backend", &routev1.Route{}}, - {"workerDC", "backend-worker", &appsv1.DeploymentConfig{}}, + {"workerDeployment", "backend-worker", &k8sappsv1.Deployment{}}, {"environmentCM", "backend-environment", &v1.ConfigMap{}}, {"internalAPISecret", component.BackendSecretInternalApiSecretName, &v1.Secret{}}, {"listenerSecret", component.BackendSecretBackendListenerSecretName, &v1.Secret{}}, @@ -141,18 +154,25 @@ func TestReplicaBackendReconciler(t *testing.T) { if err != nil { t.Fatal(err) } - err = appsv1.AddToScheme(s) + err = appsv1.Install(s) if err != nil { t.Fatal(err) } - if err := configv1.AddToScheme(s); err != nil { + if err := configv1.Install(s); err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } + backendRedisSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backend-redis", + Namespace: namespace, + }, + } + cases := []struct { testName string objName string @@ -171,7 +191,7 @@ func TestReplicaBackendReconciler(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - objs := []runtime.Object{tc.apimanager} + objs := []runtime.Object{tc.apimanager, backendRedisSecret} cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) clientset := fakeclientset.NewSimpleClientset() @@ -185,21 +205,21 @@ func TestReplicaBackendReconciler(t *testing.T) { t.Fatal(err) } - dc := &appsv1.DeploymentConfig{} + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: tc.objName, Namespace: namespace, } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - // bump the amount of replicas in the dc - dc.Spec.Replicas = twoValue - err = cl.Update(context.TODO(), dc) + // bump the amount of replicas in the deployment + deployment.Spec.Replicas = &twoValue + err = cl.Update(context.TODO(), deployment) if err != nil { - subT.Errorf("error updating dc of %s: %v", tc.objName, err) + subT.Errorf("error updating deployment of %s: %v", tc.objName, err) } // re-run the reconciler @@ -208,13 +228,13 @@ func TestReplicaBackendReconciler(t *testing.T) { t.Fatal(err) } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - if tc.expectedAmountOfReplicas != dc.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, dc.Spec.Replicas) + if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, deployment.Spec.Replicas) } }) } diff --git a/pkg/reconcilers/deployment.go b/pkg/reconcilers/deployment.go index 9a2395333..33046eac3 100644 --- a/pkg/reconcilers/deployment.go +++ b/pkg/reconcilers/deployment.go @@ -81,6 +81,8 @@ func GenericBackendDeploymentMutators() []DMutateFn { DeploymentPriorityClassMutator, DeploymentTopologySpreadConstraintsMutator, DeploymentPodTemplateAnnotationsMutator, + DeploymentArgsMutator, + DeploymentProbesMutator, } } @@ -267,6 +269,43 @@ func DeploymentPodTemplateAnnotationsMutator(desired, existing *k8sappsv1.Deploy return updated, nil } +// DeploymentArgsMutator ensures deployment's containers' args are reconciled +func DeploymentArgsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + for i, desiredContainer := range desired.Spec.Template.Spec.Containers { + existingContainer := &existing.Spec.Template.Spec.Containers[i] + + if !reflect.DeepEqual(existingContainer.Args, desiredContainer.Args) { + existingContainer.Args = desiredContainer.Args + updated = true + } + } + + return updated, nil +} + +// DeploymentProbesMutator ensures probes are reconciled +func DeploymentProbesMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + for i, desiredContainer := range desired.Spec.Template.Spec.Containers { + existingContainer := &existing.Spec.Template.Spec.Containers[i] + + if !reflect.DeepEqual(existingContainer.LivenessProbe, desiredContainer.LivenessProbe) { + existingContainer.LivenessProbe = desiredContainer.LivenessProbe + updated = true + } + + if !reflect.DeepEqual(existingContainer.ReadinessProbe, desiredContainer.ReadinessProbe) { + existingContainer.ReadinessProbe = desiredContainer.ReadinessProbe + updated = true + } + } + + return updated, nil +} + // CreateImageTriggerAnnotationString creates the annotation for Deployments to leverage existing ImageStreamTags func CreateImageTriggerAnnotationString(containers []ContainerImage) string { var triggers []ImageTrigger diff --git a/pkg/reconcilers/deployment_test.go b/pkg/reconcilers/deployment_test.go index be54892be..d1986c764 100644 --- a/pkg/reconcilers/deployment_test.go +++ b/pkg/reconcilers/deployment_test.go @@ -12,6 +12,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) func TestDeploymentReplicasMutator(t *testing.T) { @@ -800,6 +801,298 @@ func TestDeploymentPodTemplateAnnotationsMutator(t *testing.T) { } } +func TestDeploymentArgsMutator(t *testing.T) { + type args struct { + desired *k8sappsv1.Deployment + existing *k8sappsv1.Deployment + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "No Args Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Args: []string{"testArg"}}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Args: []string{"testArg"}}, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, + { + name: "Args Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Args: []string{"testArg1", "testArg2"}}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Args: []string{"testArg1"}}, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeploymentArgsMutator(tt.args.desired, tt.args.existing) + if (err != nil) != tt.wantErr { + t.Errorf("DeploymentConfigArgsMutator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DeploymentConfigArgsMutator() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDeploymentProbesMutator(t *testing.T) { + type args struct { + desired *k8sappsv1.Deployment + existing *k8sappsv1.Deployment + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "Liveness Probe Updated", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(9306), + }, + }, + InitialDelaySeconds: 60, + PeriodSeconds: 10, + }, + }, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + LivenessProbe: nil, + }, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }, + { + name: "Liveness Probe Not Updated", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(9306), + }, + }, + InitialDelaySeconds: 60, + PeriodSeconds: 10, + }, + }, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(9306), + }, + }, + InitialDelaySeconds: 60, + PeriodSeconds: 10, + }, + }, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, + { + name: "Readiness Probe Updated", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/status", + Port: intstr.IntOrString{ + Type: intstr.Type(intstr.Int), + IntVal: 3000}}, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + }, + }, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: nil, + }, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }, + { + name: "Readiness Probe Not Updated", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/status", + Port: intstr.IntOrString{ + Type: intstr.Type(intstr.Int), + IntVal: 3000}}, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + }, + }, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/status", + Port: intstr.IntOrString{ + Type: intstr.Type(intstr.Int), + IntVal: 3000}}, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + }, + }, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeploymentProbesMutator(tt.args.desired, tt.args.existing) + if (err != nil) != tt.wantErr { + t.Errorf("DeploymentConfigProbesMutator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DeploymentConfigProbesMutator() = %v, want %v", got, tt.want) + } + }) + } +} + func TestDeploymentAnnotationsMutator(t *testing.T) { dFactory := func(annotations map[string]string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ diff --git a/pkg/reconcilers/deploymentconfig.go b/pkg/reconcilers/deploymentconfig.go index 4808f5ec8..26caca55e 100644 --- a/pkg/reconcilers/deploymentconfig.go +++ b/pkg/reconcilers/deploymentconfig.go @@ -321,4 +321,4 @@ func DeploymentConfigProbesMutator(desired, existing *appsv1.DeploymentConfig) ( } return updated, nil -} \ No newline at end of file +} diff --git a/pkg/reconcilers/deploymentconfig_test.go b/pkg/reconcilers/deploymentconfig_test.go index ceed447f8..50428b7cf 100644 --- a/pkg/reconcilers/deploymentconfig_test.go +++ b/pkg/reconcilers/deploymentconfig_test.go @@ -1078,15 +1078,15 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { wantErr bool }{ { - name: "Liveness Probe Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: &v1.Probe{ + name: "Liveness Probe Updated", + args: args{ + desired: &appsv1.DeploymentConfig{ + Spec: appsv1.DeploymentConfigSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.FromInt(9306), @@ -1095,39 +1095,39 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { InitialDelaySeconds: 60, PeriodSeconds: 10, }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: nil, - }, - }, - }, - }, - }, - }, - }, - want: true, - wantErr: false, - }, + }, + }, + }, + }, + }, + }, + existing: &appsv1.DeploymentConfig{ + Spec: appsv1.DeploymentConfigSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + LivenessProbe: nil, + }, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }, { - name: "Liveness Probe Not Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: &v1.Probe{ + name: "Liveness Probe Not Updated", + args: args{ + desired: &appsv1.DeploymentConfig{ + Spec: appsv1.DeploymentConfigSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.FromInt(9306), @@ -1136,18 +1136,18 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { InitialDelaySeconds: 60, PeriodSeconds: 10, }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { + }, + }, + }, + }, + }, + }, + existing: &appsv1.DeploymentConfig{ + Spec: appsv1.DeploymentConfigSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -1156,26 +1156,26 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { }, InitialDelaySeconds: 60, PeriodSeconds: 10, - }, + }, }, - }, - }, - }, - }, - }, - }, - want: false, - wantErr: false, - }, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, { - name: "Readiness Probe Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { + name: "Readiness Probe Updated", + args: args{ + desired: &appsv1.DeploymentConfig{ + Spec: appsv1.DeploymentConfigSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ Path: "/status", @@ -1186,39 +1186,39 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { InitialDelaySeconds: 30, TimeoutSeconds: 5, }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: nil, - }, - }, - }, - }, - }, - }, - }, - want: true, - wantErr: false, - }, + }, + }, + }, + }, + }, + }, + existing: &appsv1.DeploymentConfig{ + Spec: appsv1.DeploymentConfigSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + ReadinessProbe: nil, + }, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }, { - name: "Readiness Probe Not Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: &v1.Probe{ + name: "Readiness Probe Not Updated", + args: args{ + desired: &appsv1.DeploymentConfig{ + Spec: appsv1.DeploymentConfigSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ Path: "/status", Port: intstr.IntOrString{ @@ -1228,18 +1228,18 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { InitialDelaySeconds: 30, TimeoutSeconds: 5, }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { + }, + }, + }, + }, + }, + }, + existing: &appsv1.DeploymentConfig{ + Spec: appsv1.DeploymentConfigSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ Path: "/status", @@ -1249,17 +1249,17 @@ func TestDeploymentConfigProbesMutator(t *testing.T) { }, InitialDelaySeconds: 30, TimeoutSeconds: 5, - }, + }, }, - }, - }, - }, - }, - }, - }, - want: false, - wantErr: false, - }, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/upgrade/migrate_deployment_config_2.15.go b/pkg/upgrade/migrate_deployment_config_2.15.go index f29bdecb1..27339349f 100644 --- a/pkg/upgrade/migrate_deployment_config_2.15.go +++ b/pkg/upgrade/migrate_deployment_config_2.15.go @@ -3,12 +3,14 @@ package upgrade import ( "context" "fmt" - "github.com/3scale/3scale-operator/pkg/helper" + appsv1 "github.com/openshift/api/apps/v1" k8sappsv1 "k8s.io/api/apps/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/3scale/3scale-operator/pkg/helper" ) // MigrateDeploymentConfigToDeployment verifies the Deployment is healthy and then deletes the corresponding DeploymentConfig @@ -36,10 +38,10 @@ func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, client Namespace: dNamespace, Name: dName, }, deployment) - if err != nil { - return false, fmt.Errorf("error getting deployment %s: %v", deployment.Name, err) + if err != nil && !k8serr.IsNotFound(err) { + return false, fmt.Errorf("error getting deployment %s: %w", deployment.Name, err) } - if !helper.IsDeploymentAvailable(deployment) { + if k8serr.IsNotFound(err) || !helper.IsDeploymentAvailable(deployment) { log.V(1).Info(fmt.Sprintf("deployment %s is not yet available", deployment.Name)) return false, nil } From 05486121c2e6c4264d43b3e45efe4f8b234cf060 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Tue, 28 Nov 2023 17:55:31 -0500 Subject: [PATCH 04/23] THREESCALE-10390 Migrate memcache DeploymentConfig to Deployment --- pkg/3scale/amp/component/memcached.go | 99 +++++++++---------- .../operator/memcached_options_provider.go | 3 +- .../memcached_options_provider_test.go | 9 +- .../amp/operator/memcached_reconciler.go | 31 +++--- .../amp/operator/memcached_reconciler_test.go | 8 +- 5 files changed, 78 insertions(+), 72 deletions(-) diff --git a/pkg/3scale/amp/component/memcached.go b/pkg/3scale/amp/component/memcached.go index 7070da9fa..316564175 100644 --- a/pkg/3scale/amp/component/memcached.go +++ b/pkg/3scale/amp/component/memcached.go @@ -3,7 +3,9 @@ package component import ( "fmt" - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,52 +23,28 @@ func NewMemcached(options *MemcachedOptions) *Memcached { return &Memcached{Options: options} } -func (m *Memcached) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (m *Memcached) Deployment() *k8sappsv1.Deployment { + var memcachedReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: SystemMemcachedDeploymentName, - Labels: m.Options.DeploymentLabels, + Name: SystemMemcachedDeploymentName, + Labels: m.Options.DeploymentLabels, + Annotations: m.memcacheDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{600}[0], - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%", - }, - MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange}, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "memcache", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-memcached:%s", m.Options.ImageTag), - }, - }, + Replicas: &memcachedReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemMemcachedDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemMemcachedDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: m.Options.PodTemplateLabels, Annotations: m.Options.PodTemplateAnnotations, @@ -76,21 +54,24 @@ func (m *Memcached) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: m.Options.Tolerations, ServiceAccountName: "amp", //TODO make this configurable via flag Containers: []v1.Container{ - v1.Container{ + { Name: "memcache", Image: "system-memcached:latest", Command: []string{"memcached", "-m", "64"}, Ports: []v1.ContainerPort{ - v1.ContainerPort{HostPort: 0, + {HostPort: 0, ContainerPort: 11211, Protocol: v1.ProtocolTCP}, }, Resources: m.Options.ResourceRequirements, LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{TCPSocket: &v1.TCPSocketAction{ - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 11211}}, + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 11211, + }, + }, }, InitialDelaySeconds: 10, TimeoutSeconds: 0, @@ -99,10 +80,13 @@ func (m *Memcached) DeploymentConfig() *appsv1.DeploymentConfig { FailureThreshold: 0, }, ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{TCPSocket: &v1.TCPSocketAction{ - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 11211}}, + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 11211, + }, + }, }, InitialDelaySeconds: 10, TimeoutSeconds: 5, @@ -115,7 +99,18 @@ func (m *Memcached) DeploymentConfig() *appsv1.DeploymentConfig { }, PriorityClassName: m.Options.PriorityClassName, TopologySpreadConstraints: m.Options.TopologySpreadConstraints, - }}, + }, + }, }, } } + +func (m *Memcached) memcacheDeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: "memcache", + Tag: fmt.Sprintf("system-memcached:%v", m.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} diff --git a/pkg/3scale/amp/operator/memcached_options_provider.go b/pkg/3scale/amp/operator/memcached_options_provider.go index ea74e4a72..2829f9928 100644 --- a/pkg/3scale/amp/operator/memcached_options_provider.go +++ b/pkg/3scale/amp/operator/memcached_options_provider.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" @@ -80,7 +81,7 @@ func (m *MemcachedOptionsProvider) podTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "system-memcache" + labels[reconcilers.DeploymentLabelSelector] = "system-memcache" return labels } diff --git a/pkg/3scale/amp/operator/memcached_options_provider_test.go b/pkg/3scale/amp/operator/memcached_options_provider_test.go index bac6eb982..9f44ed6ad 100644 --- a/pkg/3scale/amp/operator/memcached_options_provider_test.go +++ b/pkg/3scale/amp/operator/memcached_options_provider_test.go @@ -1,6 +1,7 @@ package operator import ( + "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -24,10 +25,10 @@ func testMemcachedDeploymentLabels() map[string]string { func testPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "memcache", - "deploymentConfig": "system-memcache", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "memcache", + reconcilers.DeploymentLabelSelector: "system-memcache", } addExpectedMeteringLabels(labels, "system-memcache", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/memcached_reconciler.go b/pkg/3scale/amp/operator/memcached_reconciler.go index 08ee2c6f7..974b71e0b 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler.go +++ b/pkg/3scale/amp/operator/memcached_reconciler.go @@ -4,6 +4,7 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -23,22 +24,30 @@ func (r *MemcachedReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // DC - mutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // Deployment + mutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, ) - err = r.ReconcileDeploymentConfig(memcached.DeploymentConfig(), mutator) + err = r.ReconcileDeployment(memcached.Deployment(), mutator) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemMemcachedDeploymentName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + return reconcile.Result{}, nil } diff --git a/pkg/3scale/amp/operator/memcached_reconciler_test.go b/pkg/3scale/amp/operator/memcached_reconciler_test.go index 4465fb7c3..f15a14543 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler_test.go +++ b/pkg/3scale/amp/operator/memcached_reconciler_test.go @@ -7,7 +7,7 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/reconcilers" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" fakeclientset "k8s.io/client-go/kubernetes/fake" @@ -18,13 +18,13 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -func TestMemcachedDCReconciler(t *testing.T) { +func TestMemcachedDeploymentReconciler(t *testing.T) { log := logf.Log.WithName("operator_test") ctx := context.TODO() apimanager := basicApimanager() s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } @@ -51,7 +51,7 @@ func TestMemcachedDCReconciler(t *testing.T) { objName string obj client.Object }{ - {"memcachedDC", "system-memcache", &appsv1.DeploymentConfig{}}, + {"memcachedDeployment", "system-memcache", &k8sappsv1.Deployment{}}, } for _, tc := range cases { From db605234485e94bd4600dcb9f383adae360ef7af Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Mon, 4 Dec 2023 13:17:35 -0500 Subject: [PATCH 05/23] THREESCALE-10392 Migrate zync DeploymentConfigs to Deployments --- pkg/3scale/amp/component/zync.go | 256 +++++++++--------- .../amp/operator/backend_reconciler_test.go | 6 + .../amp/operator/memcached_reconciler_test.go | 7 + .../amp/operator/zync_options_provider.go | 9 +- .../operator/zync_options_provider_test.go | 25 +- pkg/3scale/amp/operator/zync_reconciler.go | 106 +++++--- .../amp/operator/zync_reconciler_test.go | 69 +++-- pkg/reconcilers/role.go | 29 ++ pkg/reconcilers/role_test.go | 82 ++++++ 9 files changed, 383 insertions(+), 206 deletions(-) create mode 100644 pkg/reconcilers/role.go create mode 100644 pkg/reconcilers/role_test.go diff --git a/pkg/3scale/amp/component/zync.go b/pkg/3scale/amp/component/zync.go index 5ac724b0c..b5e730ba4 100644 --- a/pkg/3scale/amp/component/zync.go +++ b/pkg/3scale/amp/component/zync.go @@ -2,11 +2,12 @@ package component import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" + k8sappsv1 "k8s.io/api/apps/v1" policyv1 "k8s.io/api/policy/v1" "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,6 +18,7 @@ const ( ZyncName = "zync" ZyncQueDeploymentName = "zync-que" ZyncDatabaseDeploymentName = "zync-database" + ZyncInitContainerName = "zync-db-svc" ) const ( @@ -83,7 +85,7 @@ func (zync *Zync) QueRoleBinding() *rbacv1.RoleBinding { Name: "zync-que-rolebinding", }, Subjects: []rbacv1.Subject{ - rbacv1.Subject{ + { Kind: "ServiceAccount", Name: "zync-que-sa", }, @@ -106,17 +108,17 @@ func (zync *Zync) QueRole() *rbacv1.Role { Name: "zync-que-role", }, Rules: []rbacv1.PolicyRule{ - rbacv1.PolicyRule{ - APIGroups: []string{"apps.openshift.io"}, + { + APIGroups: []string{"apps"}, Resources: []string{ - "deploymentconfigs", + "deployment", }, Verbs: []string{ "get", "list", }, }, - rbacv1.PolicyRule{ + { APIGroups: []string{""}, Resources: []string{ "pods", @@ -127,7 +129,7 @@ func (zync *Zync) QueRole() *rbacv1.Role { "list", }, }, - rbacv1.PolicyRule{ + { APIGroups: []string{"route.openshift.io"}, Resources: []string{ "routes", @@ -141,7 +143,7 @@ func (zync *Zync) QueRole() *rbacv1.Role { "update", }, }, - rbacv1.PolicyRule{ + { APIGroups: []string{"route.openshift.io"}, Resources: []string{ "routes/status", @@ -150,7 +152,7 @@ func (zync *Zync) QueRole() *rbacv1.Role { "get", }, }, - rbacv1.PolicyRule{ + { APIGroups: []string{"route.openshift.io"}, Resources: []string{ "routes/custom-host", @@ -163,39 +165,22 @@ func (zync *Zync) QueRole() *rbacv1.Role { } } -func (zync *Zync) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (zync *Zync) Deployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: ZyncName, - Labels: zync.Options.CommonZyncLabels, + Name: ZyncName, + Labels: zync.Options.CommonZyncLabels, + Annotations: zync.deploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "zync-db-svc", - ZyncName, - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-zync:%s", zync.Options.ImageTag), - }, - }, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &zync.Options.ZyncReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ZyncName, }, }, - Replicas: zync.Options.ZyncReplicas, - Selector: map[string]string{"deploymentConfig": ZyncName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: zync.Options.ZyncPodTemplateLabels, Annotations: zync.Options.ZyncPodTemplateAnnotations, @@ -205,19 +190,20 @@ func (zync *Zync) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: zync.Options.ZyncTolerations, ServiceAccountName: "amp", InitContainers: []v1.Container{ - v1.Container{ - Name: "zync-db-svc", + { + Name: ZyncInitContainerName, Image: "amp-zync:latest", Command: []string{ "bash", "-c", "bundle exec sh -c \"until rake boot:db; do sleep $SLEEP_SECONDS; done\"", - }, Env: []v1.EnvVar{ - v1.EnvVar{ + }, + Env: []v1.EnvVar{ + { Name: "SLEEP_SECONDS", Value: "1", }, - v1.EnvVar{ + { Name: "DATABASE_URL", ValueFrom: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ @@ -232,7 +218,7 @@ func (zync *Zync) DeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ + { Name: ZyncName, Image: "amp-zync:latest", Ports: zync.zyncPorts(), @@ -283,7 +269,7 @@ func (zync *Zync) commonZyncEnvVars() []v1.EnvVar { helper.EnvVarFromSecret("DATABASE_URL", "zync", "DATABASE_URL"), helper.EnvVarFromSecret("SECRET_KEY_BASE", "zync", "SECRET_KEY_BASE"), helper.EnvVarFromSecret("ZYNC_AUTHENTICATION_TOKEN", "zync", "ZYNC_AUTHENTICATION_TOKEN"), - v1.EnvVar{ + { Name: "POD_NAME", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ @@ -292,7 +278,7 @@ func (zync *Zync) commonZyncEnvVars() []v1.EnvVar { }, }, }, - v1.EnvVar{ + { Name: "POD_NAMESPACE", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ @@ -303,54 +289,35 @@ func (zync *Zync) commonZyncEnvVars() []v1.EnvVar { }, } } -func (zync *Zync) QueDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (zync *Zync) QueDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: ZyncQueDeploymentName, - Labels: zync.Options.CommonZyncQueLabels, + Name: ZyncQueDeploymentName, + Labels: zync.Options.CommonZyncQueLabels, + Annotations: zync.queDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Replicas: zync.Options.ZyncQueReplicas, - Selector: map[string]string{"deploymentConfig": ZyncQueDeploymentName}, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{600}[0], + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &zync.Options.ZyncQueReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ZyncQueDeploymentName, + }, + }, + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, }, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "que", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-zync:%s", zync.Options.ImageTag), - }, - }, - }, - }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: zync.Options.ZyncQuePodTemplateLabels, Annotations: zync.Options.ZyncQuePodTemplateAnnotations, @@ -362,7 +329,7 @@ func (zync *Zync) QueDeploymentConfig() *appsv1.DeploymentConfig { RestartPolicy: v1.RestartPolicyAlways, TerminationGracePeriodSeconds: &[]int64{30}[0], Containers: []v1.Container{ - v1.Container{ + { Name: "que", Command: []string{"/usr/bin/bash"}, Args: []string{"-c", "bundle exec rake 'que[--worker-count 10]'"}, @@ -383,7 +350,11 @@ func (zync *Zync) QueDeploymentConfig() *appsv1.DeploymentConfig { }, }, Ports: []v1.ContainerPort{ - v1.ContainerPort{Name: "metrics", ContainerPort: ZyncQueMetricsPort, Protocol: v1.ProtocolTCP}, + { + Name: "metrics", + ContainerPort: ZyncQueMetricsPort, + Protocol: v1.ProtocolTCP, + }, }, Resources: zync.Options.QueContainerResourceRequirements, Env: zync.commonZyncEnvVars(), @@ -397,41 +368,27 @@ func (zync *Zync) QueDeploymentConfig() *appsv1.DeploymentConfig { } } -func (zync *Zync) DatabaseDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (zync *Zync) DatabaseDeployment() *k8sappsv1.Deployment { + var zyncDatabaseReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: ZyncDatabaseDeploymentName, - Labels: zync.Options.CommonZyncDatabaseLabels, + Name: ZyncDatabaseDeploymentName, + Labels: zync.Options.CommonZyncDatabaseLabels, + Annotations: zync.databaseDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "postgresql", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("zync-database-postgresql:%s", zync.Options.DatabaseImageTag), - }, - }, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &zyncDatabaseReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: ZyncDatabaseDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": ZyncDatabaseDeploymentName}, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: zync.Options.ZyncDatabasePodTemplateLabels, Annotations: zync.Options.ZyncDatabasePodTemplateAnnotations, @@ -442,26 +399,28 @@ func (zync *Zync) DatabaseDeploymentConfig() *appsv1.DeploymentConfig { RestartPolicy: v1.RestartPolicyAlways, ServiceAccountName: "amp", Containers: []v1.Container{ - v1.Container{ + { Name: "postgresql", Image: " ", Ports: []v1.ContainerPort{ - v1.ContainerPort{ + { ContainerPort: 5432, - Protocol: v1.ProtocolTCP}, + Protocol: v1.ProtocolTCP, + }, }, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "zync-database-data", MountPath: "/var/lib/pgsql/data", }, }, ImagePullPolicy: v1.PullIfNotPresent, Env: []v1.EnvVar{ - v1.EnvVar{ + { Name: "POSTGRESQL_USER", Value: "zync", - }, v1.EnvVar{ + }, + { Name: "POSTGRESQL_PASSWORD", ValueFrom: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ @@ -471,7 +430,8 @@ func (zync *Zync) DatabaseDeploymentConfig() *appsv1.DeploymentConfig { Key: "ZYNC_DATABASE_PASSWORD", }, }, - }, v1.EnvVar{ + }, + { Name: "POSTGRESQL_DATABASE", Value: "zync_production", }, @@ -500,7 +460,7 @@ func (zync *Zync) DatabaseDeploymentConfig() *appsv1.DeploymentConfig { PriorityClassName: zync.Options.ZyncDatabasePriorityClassName, TopologySpreadConstraints: zync.Options.ZyncDatabaseTopologySpreadConstraints, Volumes: []v1.Volume{ - v1.Volume{ + { Name: "zync-database-data", VolumeSource: v1.VolumeSource{ EmptyDir: &v1.EmptyDirVolumeSource{ @@ -527,14 +487,14 @@ func (zync *Zync) Service() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "8080-tcp", Protocol: v1.ProtocolTCP, Port: 8080, TargetPort: intstr.FromInt(8080), }, }, - Selector: map[string]string{"deploymentConfig": ZyncName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: ZyncName}, }, } } @@ -551,14 +511,14 @@ func (zync *Zync) DatabaseService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "postgresql", Protocol: v1.ProtocolTCP, Port: 5432, TargetPort: intstr.FromInt(5432), }, }, - Selector: map[string]string{"deploymentConfig": "zync-database"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "zync-database"}, }, } } @@ -575,7 +535,7 @@ func (zync *Zync) ZyncPodDisruptionBudget() *policyv1.PodDisruptionBudget { }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": ZyncName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: ZyncName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -594,7 +554,7 @@ func (zync *Zync) QuePodDisruptionBudget() *policyv1.PodDisruptionBudget { }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": "zync-que"}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: "zync-que"}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -603,7 +563,7 @@ func (zync *Zync) QuePodDisruptionBudget() *policyv1.PodDisruptionBudget { func (zync *Zync) zyncPorts() []v1.ContainerPort { ports := []v1.ContainerPort{ - v1.ContainerPort{ContainerPort: 8080, Protocol: v1.ProtocolTCP}, + {ContainerPort: 8080, Protocol: v1.ProtocolTCP}, } if zync.Options.ZyncMetrics { @@ -612,3 +572,37 @@ func (zync *Zync) zyncPorts() []v1.ContainerPort { return ports } + +func (zync *Zync) deploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: ZyncInitContainerName, + Tag: fmt.Sprintf("amp-zync:%v", zync.Options.ImageTag), + }, + { + Name: ZyncName, + Tag: fmt.Sprintf("amp-zync:%v", zync.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} + +func (zync *Zync) queDeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: "que", + Tag: fmt.Sprintf("amp-zync:%v", zync.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} + +func (zync *Zync) databaseDeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: "postgresql", + Tag: fmt.Sprintf("zync-database-postgresql:%v", zync.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} diff --git a/pkg/3scale/amp/operator/backend_reconciler_test.go b/pkg/3scale/amp/operator/backend_reconciler_test.go index 874c67900..246fdb0d8 100644 --- a/pkg/3scale/amp/operator/backend_reconciler_test.go +++ b/pkg/3scale/amp/operator/backend_reconciler_test.go @@ -90,6 +90,12 @@ func TestNewBackendReconciler(t *testing.T) { t.Fatal(err) } + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + // Create a fake client to mock API calls. cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) diff --git a/pkg/3scale/amp/operator/memcached_reconciler_test.go b/pkg/3scale/amp/operator/memcached_reconciler_test.go index f15a14543..d0e2f3903 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler_test.go +++ b/pkg/3scale/amp/operator/memcached_reconciler_test.go @@ -2,6 +2,7 @@ package operator import ( "context" + appsv1 "github.com/openshift/api/apps/v1" "testing" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -29,6 +30,12 @@ func TestMemcachedDeploymentReconciler(t *testing.T) { t.Fatal(err) } + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + // Objects to track in the fake client. objs := []runtime.Object{} diff --git a/pkg/3scale/amp/operator/zync_options_provider.go b/pkg/3scale/amp/operator/zync_options_provider.go index 035c384a3..edc0d33b3 100644 --- a/pkg/3scale/amp/operator/zync_options_provider.go +++ b/pkg/3scale/amp/operator/zync_options_provider.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" "net/url" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -178,7 +179,7 @@ func (z *ZyncOptionsProvider) setResourceRequirementsOptions() { z.zyncOptions.DatabaseContainerResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if z.apimanager.Spec.Zync.AppSpec.Resources != nil { @@ -249,7 +250,7 @@ func (z *ZyncOptionsProvider) zyncPodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "zync" + labels[reconcilers.DeploymentLabelSelector] = "zync" return labels } @@ -265,7 +266,7 @@ func (z *ZyncOptionsProvider) zyncQuePodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "zync-que" + labels[reconcilers.DeploymentLabelSelector] = "zync-que" return labels } @@ -281,7 +282,7 @@ func (z *ZyncOptionsProvider) zyncDatabasePodTemplateLabels() map[string]string labels[k] = v } - labels["deploymentConfig"] = "zync-database" + labels[reconcilers.DeploymentLabelSelector] = "zync-database" return labels } diff --git a/pkg/3scale/amp/operator/zync_options_provider_test.go b/pkg/3scale/amp/operator/zync_options_provider_test.go index 3d45b7cd6..57607f7b0 100644 --- a/pkg/3scale/amp/operator/zync_options_provider_test.go +++ b/pkg/3scale/amp/operator/zync_options_provider_test.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -61,10 +62,10 @@ func testZyncDatabaseCommonLabels() map[string]string { func testZyncPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "zync", - "threescale_component_element": "zync", - "deploymentConfig": "zync", + "app": appLabel, + "threescale_component": "zync", + "threescale_component_element": "zync", + reconcilers.DeploymentLabelSelector: "zync", } addExpectedMeteringLabels(labels, "zync", helper.ApplicationType) @@ -73,10 +74,10 @@ func testZyncPodTemplateLabels() map[string]string { func testZyncQuePodTemplateCommonLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "zync", - "threescale_component_element": "zync-que", - "deploymentConfig": "zync-que", + "app": appLabel, + "threescale_component": "zync", + "threescale_component_element": "zync-que", + reconcilers.DeploymentLabelSelector: "zync-que", } addExpectedMeteringLabels(labels, "zync-que", helper.ApplicationType) @@ -85,10 +86,10 @@ func testZyncQuePodTemplateCommonLabels() map[string]string { func testZyncDatabasePodTemplateCommonLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "zync", - "threescale_component_element": "database", - "deploymentConfig": "zync-database", + "app": appLabel, + "threescale_component": "zync", + "threescale_component_element": "database", + reconcilers.DeploymentLabelSelector: "zync-database", } addExpectedMeteringLabels(labels, "zync-database", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/zync_reconciler.go b/pkg/3scale/amp/operator/zync_reconciler.go index c650c2aa5..0b45cca5e 100644 --- a/pkg/3scale/amp/operator/zync_reconciler.go +++ b/pkg/3scale/amp/operator/zync_reconciler.go @@ -5,6 +5,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -27,7 +28,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { } // Zync Que Role - err = r.ReconcileRole(zync.QueRole(), reconcilers.CreateOnlyMutator) + err = r.ReconcileRole(zync.QueRole(), reconcilers.RoleRuleMutator) if err != nil { return reconcile.Result{}, err } @@ -44,69 +45,98 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // Zync DC - zyncMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // Zync Deployment + zyncMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, } if r.apiManager.Spec.Zync.AppSpec.Replicas != nil { - zyncMutators = append(zyncMutators, reconcilers.DeploymentConfigReplicasMutator) + zyncMutators = append(zyncMutators, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(zync.DeploymentConfig(), reconcilers.DeploymentConfigMutator(zyncMutators...)) + err = r.ReconcileDeployment(zync.Deployment(), reconcilers.DeploymentMutator(zyncMutators...)) if err != nil { return reconcile.Result{}, err } - // Zync Que DC - zyncQueMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ZyncName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + // Zync Que Deployment + zyncQueMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, } if r.apiManager.Spec.Zync.QueSpec.Replicas != nil { - zyncQueMutators = append(zyncQueMutators, reconcilers.DeploymentConfigReplicasMutator) + zyncQueMutators = append(zyncQueMutators, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeploymentConfig(zync.QueDeploymentConfig(), reconcilers.DeploymentConfigMutator(zyncQueMutators...)) + err = r.ReconcileDeployment(zync.QueDeployment(), reconcilers.DeploymentMutator(zyncQueMutators...)) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncQueDeploymentName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + // Zync Service - err = r.ReconcileService(zync.Service(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(zync.Service(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } if !r.apiManager.IsExternal(appsv1alpha1.ZyncDatabase) { - // Zync DB DC - zyncDBDCMutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // Zync DB Deployment + zyncDBDMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, ) - err = r.ReconcileDeploymentConfig(zync.DatabaseDeploymentConfig(), zyncDBDCMutator) + err = r.ReconcileDeployment(zync.DatabaseDeployment(), zyncDBDMutator) + if err != nil { + return reconcile.Result{}, err + } + + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncDatabaseDeploymentName, r.apiManager.GetNamespace(), r.Client()) if err != nil { return reconcile.Result{}, err } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } // Zync DB Service - err = r.ReconcileService(zync.DatabaseService(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(zync.DatabaseService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/zync_reconciler_test.go b/pkg/3scale/amp/operator/zync_reconciler_test.go index ef8259f72..3169110de 100644 --- a/pkg/3scale/amp/operator/zync_reconciler_test.go +++ b/pkg/3scale/amp/operator/zync_reconciler_test.go @@ -2,6 +2,7 @@ package operator import ( "context" + k8sappsv1 "k8s.io/api/apps/v1" "testing" policyv1 "k8s.io/api/policy/v1" @@ -68,15 +69,19 @@ func TestNewZyncReconciler(t *testing.T) { objs := []runtime.Object{apimanager} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } - err = imagev1.AddToScheme(s) + err = configv1.Install(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = imagev1.Install(s) + if err != nil { + t.Fatal(err) + } + err = routev1.Install(s) if err != nil { t.Fatal(err) } @@ -87,6 +92,12 @@ func TestNewZyncReconciler(t *testing.T) { t.Fatal(err) } + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + // Create a fake client to mock API calls. cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) @@ -110,9 +121,9 @@ func TestNewZyncReconciler(t *testing.T) { {"queRole", "zync-que-role", &rbacv1.Role{}}, {"queServiceAccount", "zync-que-sa", &v1.ServiceAccount{}}, {"queRoleBinding", "zync-que-rolebinding", &rbacv1.RoleBinding{}}, - {"zyncDC", "zync", &appsv1.DeploymentConfig{}}, - {"zyncQueDC", "zync-que", &appsv1.DeploymentConfig{}}, - {"zyncDatabaseDC", "zync-database", &appsv1.DeploymentConfig{}}, + {"zyncDeployment", "zync", &k8sappsv1.Deployment{}}, + {"zyncQueDeployment", "zync-que", &k8sappsv1.Deployment{}}, + {"zyncDatabaseDeployment", "zync-database", &k8sappsv1.Deployment{}}, {"zyncService", "zync", &v1.Service{}}, {"zyncDatabaseService", "zync-database", &v1.Service{}}, {"zyncSecret", component.ZyncSecretName, &v1.Secret{}}, @@ -182,15 +193,19 @@ func TestNewZyncReconcilerWithAllExternalDatabases(t *testing.T) { objs := []runtime.Object{apimanager, zyncExternalDatabaseSecret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } + err = configv1.Install(s) if err != nil { t.Fatal(err) } - err = imagev1.AddToScheme(s) + err = imagev1.Install(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } @@ -201,6 +216,12 @@ func TestNewZyncReconcilerWithAllExternalDatabases(t *testing.T) { t.Fatal(err) } + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + // Create a fake client to mock API calls. cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) @@ -225,9 +246,9 @@ func TestNewZyncReconcilerWithAllExternalDatabases(t *testing.T) { {"queRole", "zync-que-role", &rbacv1.Role{}, true}, {"queServiceAccount", "zync-que-sa", &v1.ServiceAccount{}, true}, {"queRoleBinding", "zync-que-rolebinding", &rbacv1.RoleBinding{}, true}, - {"zyncDC", "zync", &appsv1.DeploymentConfig{}, true}, - {"zyncQueDC", "zync-que", &appsv1.DeploymentConfig{}, true}, - {"zyncDatabaseDC", "zync-database", &appsv1.DeploymentConfig{}, false}, + {"zyncDeployment", "zync", &k8sappsv1.Deployment{}, true}, + {"zyncQueDeployment", "zync-que", &k8sappsv1.Deployment{}, true}, + {"zyncDatabaseDeployment", "zync-database", &k8sappsv1.Deployment{}, false}, {"zyncService", "zync", &v1.Service{}, true}, {"zyncDatabaseService", "zync-database", &v1.Service{}, false}, {"zyncSecret", component.ZyncSecretName, &v1.Secret{}, true}, @@ -271,12 +292,18 @@ func TestReplicaZyncReconciler(t *testing.T) { if err != nil { t.Fatal(err) } - err = appsv1.AddToScheme(s) + err = k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } - if err := configv1.AddToScheme(s); err != nil { + if err := configv1.Install(s); err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { t.Fatal(err) } @@ -310,20 +337,20 @@ func TestReplicaZyncReconciler(t *testing.T) { t.Fatal(err) } - dc := &appsv1.DeploymentConfig{} + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: tc.objName, Namespace: namespace, } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } // bump the amount of replicas in the dc - dc.Spec.Replicas = twoValue - err = cl.Update(context.TODO(), dc) + deployment.Spec.Replicas = &twoValue + err = cl.Update(context.TODO(), deployment) if err != nil { subT.Errorf("error updating dc of %s: %v", tc.objName, err) } @@ -334,13 +361,13 @@ func TestReplicaZyncReconciler(t *testing.T) { t.Fatal(err) } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - if tc.expectedAmountOfReplicas != dc.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, dc.Spec.Replicas) + if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, deployment.Spec.Replicas) } }) } diff --git a/pkg/reconcilers/role.go b/pkg/reconcilers/role.go new file mode 100644 index 000000000..6982d4cdb --- /dev/null +++ b/pkg/reconcilers/role.go @@ -0,0 +1,29 @@ +package reconcilers + +import ( + "fmt" + "github.com/3scale/3scale-operator/pkg/common" + "reflect" + + rbacv1 "k8s.io/api/rbac/v1" +) + +func RoleRuleMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { + existing, ok := existingObj.(*rbacv1.Role) + if !ok { + return false, fmt.Errorf("%T is not a *rbacv1.Role", existingObj) + } + desired, ok := desiredObj.(*rbacv1.Role) + if !ok { + return false, fmt.Errorf("%T is not a *rbacv1.Role", desiredObj) + } + + updated := false + + if !reflect.DeepEqual(existing.Rules, desired.Rules) { + existing.Rules = desired.Rules + updated = true + } + + return updated, nil +} diff --git a/pkg/reconcilers/role_test.go b/pkg/reconcilers/role_test.go new file mode 100644 index 000000000..483c02456 --- /dev/null +++ b/pkg/reconcilers/role_test.go @@ -0,0 +1,82 @@ +package reconcilers + +import ( + "github.com/google/go-cmp/cmp" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "reflect" + "testing" +) + +func TestRoleRuleMutator(t *testing.T) { + + testRules1 := []rbacv1.PolicyRule{ + { + APIGroups: []string{"apps"}, + Resources: []string{ + "deployment", + }, + Verbs: []string{ + "get", + "list", + }, + }, + } + + testRules2 := []rbacv1.PolicyRule{ + { + APIGroups: []string{"apps"}, + Resources: []string{ + "deployment", + }, + Verbs: []string{ + "get", + "list", + "create", + "delete", + }, + }, + } + + roleFactory := func(rules []rbacv1.PolicyRule) *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + Kind: "Role", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testRole", + Namespace: "testNamespace", + }, + Rules: rules, + } + } + + cases := []struct { + testName string + existingRules []rbacv1.PolicyRule + desiredRules []rbacv1.PolicyRule + expectedResult bool + }{ + {"NothingToReconcile", nil, nil, false}, + {"EqualRules", testRules1, testRules1, false}, + {"DifferentRules", testRules1, testRules2, true}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + existing := roleFactory(tc.existingRules) + desired := roleFactory(tc.desiredRules) + update, err := RoleRuleMutator(desired, existing) + if err != nil { + subT.Fatal(err) + } + if update != tc.expectedResult { + subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) + } + if !reflect.DeepEqual(existing.Rules, desired.Rules) { + subT.Fatal(cmp.Diff(existing.Rules, desired.Rules)) + } + }) + } +} From 1026b7b07c0f2f5db73c421ed12044e02a80eeb2 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Fri, 15 Dec 2023 11:36:48 -0500 Subject: [PATCH 06/23] THREESCALE-10391 Migrate system DCs to Deployments --- pkg/3scale/amp/component/system.go | 328 ++++++++++++------ pkg/3scale/amp/component/system_searchd.go | 72 ++-- pkg/3scale/amp/operator/apicast_reconciler.go | 4 +- pkg/3scale/amp/operator/backend_reconciler.go | 6 +- .../base_apimanager_logic_reconciler.go | 5 + .../amp/operator/memcached_reconciler.go | 2 +- .../amp/operator/system_options_provider.go | 5 +- .../operator/system_options_provider_test.go | 17 +- pkg/3scale/amp/operator/system_reconciler.go | 192 ++++++---- .../amp/operator/system_reconciler_test.go | 76 +++- .../system_searchd_options_provider.go | 3 +- .../system_searchd_options_provider_test.go | 9 +- .../amp/operator/system_searchd_reconciler.go | 71 ++-- .../system_searchd_reconciler_test.go | 139 +------- pkg/3scale/amp/operator/zync_reconciler.go | 6 +- pkg/helper/job.go | 28 ++ pkg/upgrade/migrate_deployment_config_2.15.go | 20 +- pkg/upgrade/upgrade_sphinx_reference_2.14.go | 19 - .../upgrade_system_backend_urls_2.14.go | 30 -- 19 files changed, 540 insertions(+), 492 deletions(-) delete mode 100644 pkg/upgrade/upgrade_sphinx_reference_2.14.go delete mode 100644 pkg/upgrade/upgrade_system_backend_urls_2.14.go diff --git a/pkg/3scale/amp/component/system.go b/pkg/3scale/amp/component/system.go index b8327c8bd..90021dee1 100644 --- a/pkg/3scale/amp/component/system.go +++ b/pkg/3scale/amp/component/system.go @@ -2,10 +2,12 @@ package component import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" + k8sappsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" "sort" "strconv" - appsv1 "github.com/openshift/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -91,8 +93,11 @@ const ( ) const ( - SystemSidekiqName = "system-sidekiq" - SystemAppDeploymentName = "system-app" + SystemSidekiqName = "system-sidekiq" + SystemSideKiqInitContainerName = "check-svc" + SystemAppDeploymentName = "system-app" + SystemAppPreHookJobName = "system-app-pre" + SystemAppPostHookJobName = "system-app-post" SystemAppMasterContainerName = "system-master" SystemAppProviderContainerName = "system-provider" @@ -309,7 +314,9 @@ func (system *System) buildSystemAppPreHookEnv() []v1.EnvVar { } func (system *System) buildSystemAppPostHookEnv() []v1.EnvVar { - var result []v1.EnvVar + result := []v1.EnvVar{} + baseEnv := system.buildSystemBaseEnv() + result = append(result, baseEnv...) if system.Options.IncludeOracleOptionalSettings { result = append(result, helper.EnvVarFromSecretOptional("ORACLE_SYSTEM_PASSWORD", SystemSecretSystemDatabaseSecretName, "ORACLE_SYSTEM_PASSWORD")) } @@ -527,75 +534,51 @@ func (system *System) appPodVolumes() []v1.Volume { return res } -func (system *System) volumeNamesForSystemAppPreHookPod() []string { - res := []string{} +func (system *System) volumesForSystemAppLifecycleHookPods() []v1.VolumeMount { + res := []v1.VolumeMount{} if system.Options.PvcFileStorageOptions != nil { - res = append(res, SystemFileStoragePVCName) + res = append(res, v1.VolumeMount{ + Name: SystemFileStoragePVCName, + }) } if system.Options.S3FileStorageOptions != nil && system.Options.S3FileStorageOptions.STSEnabled { - res = append(res, S3StsCredentialsSecretName) + res = append(res, v1.VolumeMount{ + Name: S3StsCredentialsSecretName, + }) } return res } -func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (system *System) AppDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: SystemAppDeploymentName, - Labels: system.Options.CommonAppLabels, + Name: SystemAppDeploymentName, + Labels: system.Options.CommonAppLabels, + Annotations: system.appDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{1200}[0], + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, - Pre: &appsv1.LifecycleHook{ - FailurePolicy: appsv1.LifecycleHookFailurePolicyRetry, - ExecNewPod: &appsv1.ExecNewPodHook{ - // TODO the MASTER_ACCESS_TOKEN reference should be probably set as an envvar that gathers its value from the system-seed secret - // but changing that probably has some implications during an upgrade process of the product - Command: []string{"bash", "-c", "bundle exec rake boot openshift:deploy"}, - Env: system.buildSystemAppPreHookEnv(), - ContainerName: SystemAppMasterContainerName, - Volumes: system.volumeNamesForSystemAppPreHookPod()}, - }, - Post: &appsv1.LifecycleHook{ - FailurePolicy: appsv1.LifecycleHookFailurePolicyAbort, - ExecNewPod: &appsv1.ExecNewPodHook{ - Command: []string{"bash", "-c", "bundle exec rake boot openshift:post_deploy"}, - Env: system.buildSystemAppPostHookEnv(), - ContainerName: SystemAppMasterContainerName}}}, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{SystemAppProviderContainerName, SystemAppDeveloperContainerName, SystemAppMasterContainerName}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-system:%s", system.Options.ImageTag)}}}, + Replicas: &system.Options.AppReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemAppDeploymentName, + }, }, - Replicas: system.Options.AppReplicas, - Selector: map[string]string{"deploymentConfig": SystemAppDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: system.Options.AppPodTemplateLabels, Annotations: system.Options.AppPodTemplateAnnotations, @@ -605,7 +588,7 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { Tolerations: system.Options.AppTolerations, Volumes: system.appPodVolumes(), Containers: []v1.Container{ - v1.Container{ + { Name: SystemAppMasterContainerName, Image: "amp-system:latest", Args: []string{"env", "TENANT_MODE=master", "PORT=3002", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, @@ -617,8 +600,10 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "master"}}, + Type: intstr.String, + StrVal: "master", + }, + }, }, InitialDelaySeconds: 40, TimeoutSeconds: 10, @@ -631,14 +616,17 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { HTTPGet: &v1.HTTPGetAction{ Path: "/check.txt", Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "master", }, Scheme: v1.URISchemeHTTP, HTTPHeaders: []v1.HTTPHeader{ - v1.HTTPHeader{ + { Name: "X-Forwarded-Proto", - Value: "https"}}}, + Value: "https", + }, + }, + }, }, InitialDelaySeconds: 60, TimeoutSeconds: 10, @@ -650,7 +638,8 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { Stdin: false, StdinOnce: false, TTY: false, - }, v1.Container{ + }, + { Name: SystemAppProviderContainerName, Image: "amp-system:latest", Args: []string{"env", "TENANT_MODE=provider", "PORT=3000", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, @@ -662,8 +651,10 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "provider"}}, + Type: intstr.String, + StrVal: "provider", + }, + }, }, InitialDelaySeconds: 40, TimeoutSeconds: 10, @@ -676,14 +667,17 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { HTTPGet: &v1.HTTPGetAction{ Path: "/check.txt", Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "provider", }, Scheme: v1.URISchemeHTTP, HTTPHeaders: []v1.HTTPHeader{ - v1.HTTPHeader{ + { Name: "X-Forwarded-Proto", - Value: "https"}}}, + Value: "https", + }, + }, + }, }, InitialDelaySeconds: 60, TimeoutSeconds: 10, @@ -695,7 +689,8 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { Stdin: false, StdinOnce: false, TTY: false, - }, v1.Container{ + }, + { Name: SystemAppDeveloperContainerName, Image: "amp-system:latest", Args: []string{"env", "PORT=3001", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, @@ -707,8 +702,10 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "developer"}}, + Type: intstr.String, + StrVal: "developer", + }, + }, }, InitialDelaySeconds: 40, TimeoutSeconds: 10, @@ -721,14 +718,17 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { HTTPGet: &v1.HTTPGetAction{ Path: "/check.txt", Port: intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "developer", }, Scheme: v1.URISchemeHTTP, HTTPHeaders: []v1.HTTPHeader{ - v1.HTTPHeader{ + { Name: "X-Forwarded-Proto", - Value: "https"}}}, + Value: "https", + }, + }, + }, }, InitialDelaySeconds: 60, TimeoutSeconds: 10, @@ -742,7 +742,82 @@ func (system *System) AppDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", PriorityClassName: system.Options.AppPriorityClassName, TopologySpreadConstraints: system.Options.AppTopologySpreadConstraints, - }}, + }, + }, + }, + } +} + +func (system *System) AppPreHookJob(containerImage string) *batchv1.Job { + var completions int32 = 1 + + return &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "batch/v1", + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: SystemAppPreHookJobName, + Labels: system.Options.CommonAppLabels, + }, + Spec: batchv1.JobSpec{ + Completions: &completions, + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Volumes: system.appPodVolumes(), + Containers: []v1.Container{ + { + Name: SystemAppPreHookJobName, + Image: containerImage, + Args: []string{"bash", "-c", "bundle exec rake boot openshift:deploy"}, + Env: system.buildSystemAppPreHookEnv(), + Resources: *system.Options.AppMasterContainerResourceRequirements, + VolumeMounts: system.volumesForSystemAppLifecycleHookPods(), + ImagePullPolicy: v1.PullIfNotPresent, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: "amp", + PriorityClassName: system.Options.AppPriorityClassName, + }, + }, + }, + } +} + +func (system *System) AppPostHookJob(containerImage string) *batchv1.Job { + var completions int32 = 1 + + return &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "batch/v1", + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: SystemAppPostHookJobName, + Labels: system.Options.CommonAppLabels, + }, + Spec: batchv1.JobSpec{ + Completions: &completions, + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Volumes: system.appPodVolumes(), + Containers: []v1.Container{ + { + Name: SystemAppPostHookJobName, + Image: containerImage, + Args: []string{"bash", "-c", "bundle exec rake boot openshift:post_deploy"}, + Env: system.buildSystemAppPostHookEnv(), + Resources: *system.Options.AppMasterContainerResourceRequirements, + VolumeMounts: system.volumesForSystemAppLifecycleHookPods(), + ImagePullPolicy: v1.PullIfNotPresent, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: "amp", + PriorityClassName: system.Options.AppPriorityClassName, + }, + }, }, } } @@ -819,47 +894,36 @@ func (system *System) SidekiqPodVolumes() []v1.Volume { return res } -func (system *System) SidekiqDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (system *System) SidekiqDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: SystemSidekiqName, - Labels: system.Options.CommonSidekiqLabels, + Name: SystemSidekiqName, + Labels: system.Options.CommonSidekiqLabels, + Annotations: system.sidekiqDeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRolling, - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - UpdatePeriodSeconds: &[]int64{1}[0], - IntervalSeconds: &[]int64{1}[0], - TimeoutSeconds: &[]int64{1200}[0], + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &k8sappsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), + Type: intstr.String, StrVal: "25%", }, MaxSurge: &intstr.IntOrString{ - Type: intstr.Type(intstr.String), - StrVal: "25%"}}, + Type: intstr.String, + StrVal: "25%", + }, + }, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{"check-svc", SystemSidekiqName}, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("amp-system:%s", system.Options.ImageTag)}}}, + Replicas: &system.Options.SidekiqReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemSidekiqName, + }, }, - Replicas: system.Options.SidekiqReplicas, - Selector: map[string]string{"deploymentConfig": SystemSidekiqName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: system.Options.SidekiqPodTemplateLabels, Annotations: system.Options.SideKiqPodTemplateAnnotations, @@ -869,7 +933,7 @@ func (system *System) SidekiqDeploymentConfig() *appsv1.DeploymentConfig { Tolerations: system.Options.SidekiqTolerations, Volumes: system.SidekiqPodVolumes(), InitContainers: []v1.Container{ - v1.Container{ + { Name: "check-svc", Image: "amp-system:latest", Command: []string{ @@ -881,7 +945,7 @@ func (system *System) SidekiqDeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ + { Name: SystemSidekiqName, Image: "amp-system:latest", Args: []string{"rake", "sidekiq:worker", "RAILS_MAX_THREADS=25"}, @@ -907,7 +971,8 @@ func (system *System) SidekiqDeploymentConfig() *appsv1.DeploymentConfig { ServiceAccountName: "amp", PriorityClassName: system.Options.SideKiqPriorityClassName, TopologySpreadConstraints: system.Options.SideKiqTopologySpreadConstraints, - }}, + }, + }, }, } } @@ -1034,7 +1099,7 @@ func (system *System) ProviderService() *v1.Service { TargetPort: intstr.FromString("provider"), }, }, - Selector: map[string]string{"deploymentConfig": SystemAppDeploymentName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: SystemAppDeploymentName}, }, } } @@ -1058,7 +1123,7 @@ func (system *System) MasterService() *v1.Service { TargetPort: intstr.FromString("master"), }, }, - Selector: map[string]string{"deploymentConfig": SystemAppDeploymentName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: SystemAppDeploymentName}, }, } } @@ -1082,7 +1147,7 @@ func (system *System) DeveloperService() *v1.Service { TargetPort: intstr.FromString("developer"), }, }, - Selector: map[string]string{"deploymentConfig": SystemAppDeploymentName}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: SystemAppDeploymentName}, }, } } @@ -1106,7 +1171,7 @@ func (system *System) MemcachedService() *v1.Service { TargetPort: intstr.FromInt(11211), }, }, - Selector: map[string]string{"deploymentConfig": "system-memcache"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-memcache"}, }, } } @@ -1204,7 +1269,7 @@ func (system *System) AppPodDisruptionBudget() *policyv1.PodDisruptionBudget { }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": SystemAppDeploymentName}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: SystemAppDeploymentName}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -1223,7 +1288,7 @@ func (system *System) SidekiqPodDisruptionBudget() *policyv1.PodDisruptionBudget }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"deploymentConfig": "system-sidekiq"}, + MatchLabels: map[string]string{reconcilers.DeploymentLabelSelector: "system-sidekiq"}, }, MaxUnavailable: &intstr.IntOrString{IntVal: PDB_MAX_UNAVAILABLE_POD_NUMBER}, }, @@ -1275,3 +1340,36 @@ func (system *System) appDeveloperPorts() []v1.ContainerPort { return ports } + +func (system *System) sidekiqDeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: SystemSideKiqInitContainerName, + Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), + }, + { + Name: SystemSidekiqName, + Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} + +func (system *System) appDeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: SystemAppProviderContainerName, + Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), + }, + { + Name: SystemAppDeveloperContainerName, + Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), + }, + { + Name: SystemAppMasterContainerName, + Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), + }, + }) + + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} +} diff --git a/pkg/3scale/amp/component/system_searchd.go b/pkg/3scale/amp/component/system_searchd.go index bfc902996..512355edb 100644 --- a/pkg/3scale/amp/component/system_searchd.go +++ b/pkg/3scale/amp/component/system_searchd.go @@ -2,8 +2,9 @@ package component import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" + k8sappsv1 "k8s.io/api/apps/v1" - appsv1 "github.com/openshift/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -36,53 +37,39 @@ func (s *SystemSearchd) Service() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "searchd", Protocol: v1.ProtocolTCP, Port: 9306, TargetPort: intstr.FromInt(9306), }, }, - Selector: map[string]string{"deploymentConfig": "system-searchd"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-searchd"}, }, } } -func (s *SystemSearchd) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (s *SystemSearchd) Deployment() *k8sappsv1.Deployment { + var searchdReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: SystemSearchdDeploymentName, - Labels: s.Options.Labels, + Name: SystemSearchdDeploymentName, + Labels: s.Options.Labels, + Annotations: s.DeploymentAnnotations(), }, - Spec: appsv1.DeploymentConfigSpec{ - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-searchd", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-searchd:%s", s.Options.ImageTag), - }, - }, + Spec: k8sappsv1.DeploymentSpec{ + Replicas: &searchdReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemSearchdDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemSearchdDeploymentName}, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: s.Options.PodTemplateLabels, Annotations: s.Options.PodTemplateAnnotations, @@ -92,7 +79,7 @@ func (s *SystemSearchd) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: s.Options.Tolerations, ServiceAccountName: "amp", Volumes: []v1.Volume{ - v1.Volume{ + { Name: SystemSearchdDBVolumeName, VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ @@ -103,12 +90,12 @@ func (s *SystemSearchd) DeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ - Name: "system-searchd", + { + Name: SystemSearchdDeploymentName, Image: "system-searchd:latest", ImagePullPolicy: v1.PullIfNotPresent, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { ReadOnly: false, Name: SystemSearchdDBVolumeName, MountPath: "/var/lib/searchd", @@ -170,3 +157,14 @@ func (s *SystemSearchd) PVC() *v1.PersistentVolumeClaim { }, } } + +func (s *SystemSearchd) DeploymentAnnotations() map[string]string { + imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ + { + Name: SystemSearchdDeploymentName, + Tag: fmt.Sprintf("system-searchd:%v", s.Options.ImageTag), + }, + }) + return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} + +} diff --git a/pkg/3scale/amp/operator/apicast_reconciler.go b/pkg/3scale/amp/operator/apicast_reconciler.go index bab09caae..a5687fa9c 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler.go +++ b/pkg/3scale/amp/operator/apicast_reconciler.go @@ -100,7 +100,7 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ApicastStagingName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ApicastStagingName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } @@ -147,7 +147,7 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ApicastProductionName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ApicastProductionName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/backend_reconciler.go b/pkg/3scale/amp/operator/backend_reconciler.go index cf9ea7658..e8ba14292 100644 --- a/pkg/3scale/amp/operator/backend_reconciler.go +++ b/pkg/3scale/amp/operator/backend_reconciler.go @@ -39,7 +39,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.BackendCronName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.BackendCronName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } @@ -59,7 +59,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendListenerName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendListenerName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } @@ -96,7 +96,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendWorkerName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendWorkerName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go index 3d10f05f0..b6822fb6a 100644 --- a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go +++ b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go @@ -15,6 +15,7 @@ import ( routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" k8sappsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -100,6 +101,10 @@ func (r *BaseAPIManagerLogicReconciler) ReconcileRoleBinding(desired *rbacv1.Rol return r.ReconcileResource(&rbacv1.RoleBinding{}, desired, mutateFn) } +func (r *BaseAPIManagerLogicReconciler) ReconcileJob(desired *batchv1.Job, mutateFn reconcilers.MutateFn) error { + return r.ReconcileResource(&batchv1.Job{}, desired, mutateFn) +} + func (r *BaseAPIManagerLogicReconciler) ReconcileGrafanaDashboard(desired *grafanav1alpha1.GrafanaDashboard, mutateFn reconcilers.MutateFn) error { kindExists, err := r.HasGrafanaDashboards() if err != nil { diff --git a/pkg/3scale/amp/operator/memcached_reconciler.go b/pkg/3scale/amp/operator/memcached_reconciler.go index 974b71e0b..b9fa7a6b1 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler.go +++ b/pkg/3scale/amp/operator/memcached_reconciler.go @@ -40,7 +40,7 @@ func (r *MemcachedReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemMemcachedDeploymentName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemMemcachedDeploymentName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/system_options_provider.go b/pkg/3scale/amp/operator/system_options_provider.go index 54bf71481..c7ed93688 100644 --- a/pkg/3scale/amp/operator/system_options_provider.go +++ b/pkg/3scale/amp/operator/system_options_provider.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" "path/filepath" v1 "k8s.io/api/core/v1" @@ -546,7 +547,7 @@ func (s *SystemOptionsProvider) appPodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "system-app" + labels[reconcilers.DeploymentLabelSelector] = "system-app" return labels } @@ -568,7 +569,7 @@ func (s *SystemOptionsProvider) sidekiqPodTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "system-sidekiq" + labels[reconcilers.DeploymentLabelSelector] = "system-sidekiq" return labels } diff --git a/pkg/3scale/amp/operator/system_options_provider_test.go b/pkg/3scale/amp/operator/system_options_provider_test.go index 5f3ca537d..bc102f32e 100644 --- a/pkg/3scale/amp/operator/system_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_options_provider_test.go @@ -1,6 +1,7 @@ package operator import ( + "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -39,10 +40,10 @@ func testSystemCommonAppLabels() map[string]string { func testSystemAppPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "app", - "deploymentConfig": "system-app", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "app", + reconcilers.DeploymentLabelSelector: "system-app", } addExpectedMeteringLabels(labels, "system-app", helper.ApplicationType) @@ -59,10 +60,10 @@ func testSystemCommonSidekiqLabels() map[string]string { func testSystemSidekiqPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "sidekiq", - "deploymentConfig": "system-sidekiq", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "sidekiq", + reconcilers.DeploymentLabelSelector: "system-sidekiq", } addExpectedMeteringLabels(labels, "system-sidekiq", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index ab5583164..5a6d21022 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -1,7 +1,15 @@ package operator import ( + "context" "fmt" + "time" + + k8sappsv1 "k8s.io/api/apps/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" @@ -9,13 +17,8 @@ import ( "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/pkg/upgrade" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - appsv1 "github.com/openshift/api/apps/v1" - "k8s.io/apimachinery/pkg/api/resource" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) type SystemReconciler struct { @@ -42,88 +45,46 @@ func (r *SystemReconciler) reconcileFileStorage(system *component.System) error } func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { - system, err := System(r.apiManager, r.Client()) + ampImages, err := AmpImages(r.apiManager) if err != nil { return reconcile.Result{}, err } - err = r.reconcileFileStorage(system) + system, err := System(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } - // Provider Service - err = r.ReconcileService(system.ProviderService(), reconcilers.CreateOnlyMutator) + err = r.reconcileFileStorage(system) if err != nil { return reconcile.Result{}, err } - // Master Service - err = r.ReconcileService(system.MasterService(), reconcilers.CreateOnlyMutator) - if err != nil { - return reconcile.Result{}, err + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, } - // Developer Service - err = r.ReconcileService(system.DeveloperService(), reconcilers.CreateOnlyMutator) + // Provider Service + err = r.ReconcileService(system.ProviderService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } - // Memcached Service - err = r.ReconcileService(system.MemcachedService(), reconcilers.CreateOnlyMutator) + // Master Service + err = r.ReconcileService(system.MasterService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } - // SystemApp DC - systemAppDCMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, - r.systemAppDCResourceMutator, - reconcilers.DeploymentConfigRemoveDuplicateEnvVarMutator, - // 3scale 2.13 -> 2.14 - upgrade.SphinxAddressReference, - // 3scale 2.13 -> 2.14 - upgrade.SystemBackendUrls, - } - - if r.apiManager.Spec.System.AppSpec.Replicas != nil { - systemAppDCMutators = append(systemAppDCMutators, reconcilers.DeploymentConfigReplicasMutator) - } - - err = r.ReconcileDeploymentConfig(system.AppDeploymentConfig(), reconcilers.DeploymentConfigMutator(systemAppDCMutators...)) + // Developer Service + err = r.ReconcileService(system.DeveloperService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } - // Sidekiq DC - sidekiqDCMutators := []reconcilers.DCMutateFn{ - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigRemoveDuplicateEnvVarMutator, - reconcilers.DeploymentConfigArgsMutator, - reconcilers.DeploymentConfigProbesMutator, - // 3scale 2.13 -> 2.14 - upgrade.SphinxAddressReference, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, - } - - if r.apiManager.Spec.System.SidekiqSpec.Replicas != nil { - sidekiqDCMutators = append(sidekiqDCMutators, reconcilers.DeploymentConfigReplicasMutator) - } - - err = r.ReconcileDeploymentConfig(system.SidekiqDeploymentConfig(), reconcilers.DeploymentConfigMutator(sidekiqDCMutators...)) + // Memcached Service + err = r.ReconcileService(system.MemcachedService(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -182,6 +143,103 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } + // Used to synchronize rollout of system Deployments + systemComponentNotReady := false + + // SystemApp PreHook Job + preHookJob := system.AppPreHookJob(ampImages.Options.SystemImage) + err = r.ReconcileJob(preHookJob, reconcilers.CreateOnlyMutator) + if err != nil { + return reconcile.Result{}, err + } + + // Block reconciling system-app Deployment until PreHook Job has completed + if !helper.HasJobCompleted(preHookJob.Name, preHookJob.Namespace, r.Client()) { + systemComponentNotReady = true + } + + // SystemApp Deployment + systemAppDeploymentMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + r.systemAppDeploymentResourceMutator, + reconcilers.DeploymentRemoveDuplicateEnvVarMutator, + } + if r.apiManager.Spec.System.AppSpec.Replicas != nil { + systemAppDeploymentMutators = append(systemAppDeploymentMutators, reconcilers.DeploymentReplicasMutator) + } + if !systemComponentNotReady { + err = r.ReconcileDeployment(system.AppDeployment(), reconcilers.DeploymentMutator(systemAppDeploymentMutators...)) + if err != nil { + return reconcile.Result{}, err + } + } + + // Block reconciling PostHook Job unless system-app Deployment is ready + deployment := &k8sappsv1.Deployment{} + err = r.Client().Get(context.TODO(), client.ObjectKey{ + Namespace: r.apiManager.GetNamespace(), + Name: component.SystemAppDeploymentName, + }, deployment) + if err != nil && !k8serr.IsNotFound(err) { + return reconcile.Result{}, err + } + if k8serr.IsNotFound(err) || !helper.IsDeploymentAvailable(deployment) { + systemComponentNotReady = true + } + + // SystemApp PostHook Job + if !systemComponentNotReady { + err = r.ReconcileJob(system.AppPostHookJob(ampImages.Options.SystemImage), reconcilers.CreateOnlyMutator) + if err != nil { + return reconcile.Result{}, err + } + } + + // 3scale 2.14 -> 2.15 + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemAppDeploymentName, r.apiManager.GetNamespace(), false, r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + systemComponentNotReady = true + } + + // Sidekiq Deployment + sidekiqDeploymentMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentRemoveDuplicateEnvVarMutator, + reconcilers.DeploymentArgsMutator, + reconcilers.DeploymentProbesMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + } + if r.apiManager.Spec.System.SidekiqSpec.Replicas != nil { + sidekiqDeploymentMutators = append(sidekiqDeploymentMutators, reconcilers.DeploymentReplicasMutator) + } + + err = r.ReconcileDeployment(system.SidekiqDeployment(), reconcilers.DeploymentMutator(sidekiqDeploymentMutators...)) + if err != nil { + return reconcile.Result{}, err + } + + // 3scale 2.14 -> 2.15 + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.SystemSidekiqName, r.apiManager.GetNamespace(), false, r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + systemComponentNotReady = true + } + // SystemApp PDB err = r.ReconcilePodDisruptionBudget(system.AppPodDisruptionBudget(), reconcilers.GenericPDBMutator) if err != nil { @@ -224,16 +282,19 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } + // Requeue if any of the system-app Deployment's components aren't ready + if systemComponentNotReady { + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + } + return reconcile.Result{}, nil } -func (r *SystemReconciler) systemAppDCResourceMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { +func (r *SystemReconciler) systemAppDeploymentResourceMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { desiredName := common.ObjectInfo(desired) update := false - // // Check containers - // if len(desired.Spec.Template.Spec.Containers) != 3 { return false, fmt.Errorf(fmt.Sprintf("%s desired spec.template.spec.containers length changed to '%d', should be 3", desiredName, len(desired.Spec.Template.Spec.Containers))) } @@ -244,10 +305,7 @@ func (r *SystemReconciler) systemAppDCResourceMutator(desired, existing *appsv1. update = true } - // // Check containers resource requirements - // - for idx := 0; idx < 3; idx++ { if !helper.CmpResources(&existing.Spec.Template.Spec.Containers[idx].Resources, &desired.Spec.Template.Spec.Containers[idx].Resources) { diff := cmp.Diff(existing.Spec.Template.Spec.Containers[idx].Resources, desired.Spec.Template.Spec.Containers[idx].Resources, cmpopts.IgnoreUnexported(resource.Quantity{})) diff --git a/pkg/3scale/amp/operator/system_reconciler_test.go b/pkg/3scale/amp/operator/system_reconciler_test.go index 3c6d515a6..8bd3127e2 100644 --- a/pkg/3scale/amp/operator/system_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_reconciler_test.go @@ -2,6 +2,7 @@ package operator import ( "context" + k8sappsv1 "k8s.io/api/apps/v1" "testing" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -14,6 +15,7 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,19 +37,31 @@ func TestSystemReconcilerCreate(t *testing.T) { ctx := context.TODO() apimanager := basicApimanagerSpecTestSystemOptions() + appPreHookJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: component.SystemAppPreHookJobName, Namespace: apimanager.Namespace}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: v1.ConditionTrue, + }, + }, + }, + } + // Objects to track in the fake client. - objs := []runtime.Object{apimanager} + objs := []runtime.Object{apimanager, appPreHookJob} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } - err = imagev1.AddToScheme(s) + err = imagev1.Install(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } @@ -57,7 +71,13 @@ func TestSystemReconcilerCreate(t *testing.T) { if err := grafanav1alpha1.AddToScheme(s); err != nil { t.Fatal(err) } - if err := configv1.AddToScheme(s); err != nil { + if err := configv1.Install(s); err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { t.Fatal(err) } @@ -86,8 +106,8 @@ func TestSystemReconcilerCreate(t *testing.T) { {"systemMasterService", "system-master", &v1.Service{}}, {"systemDeveloperService", "system-developer", &v1.Service{}}, {"systemMemcacheService", "system-memcache", &v1.Service{}}, - {"systemAppDC", "system-app", &appsv1.DeploymentConfig{}}, - {"systemSideKiqDC", "system-sidekiq", &appsv1.DeploymentConfig{}}, + {"systemAppDeployment", "system-app", &k8sappsv1.Deployment{}}, + {"systemSideKiqDeployment", "system-sidekiq", &k8sappsv1.Deployment{}}, {"systemCM", "system", &v1.ConfigMap{}}, {"systemEnvironmentCM", "system-environment", &v1.ConfigMap{}}, {"systemSMTPSecret", "system-smtp", &v1.Secret{}}, @@ -126,6 +146,19 @@ func TestReplicaSystemReconciler(t *testing.T) { oneValue64 int64 = 1 twoValue int32 = 2 ) + + appPreHookJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: component.SystemAppPreHookJobName, Namespace: namespace}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: v1.ConditionTrue, + }, + }, + }, + } + ctx := context.TODO() s := scheme.Scheme @@ -133,12 +166,17 @@ func TestReplicaSystemReconciler(t *testing.T) { if err != nil { t.Fatal(err) } - err = appsv1.AddToScheme(s) + err = k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } + if err := configv1.Install(s); err != nil { + t.Fatal(err) + } - if err := configv1.AddToScheme(s); err != nil { + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { t.Fatal(err) } @@ -157,7 +195,7 @@ func TestReplicaSystemReconciler(t *testing.T) { for _, tc := range cases { t.Run(tc.testName, func(subT *testing.T) { - objs := []runtime.Object{tc.apimanager} + objs := []runtime.Object{tc.apimanager, appPreHookJob} // Create a fake client to mock API calls. cl := fake.NewFakeClient(objs...) clientAPIReader := fake.NewFakeClient(objs...) @@ -172,22 +210,22 @@ func TestReplicaSystemReconciler(t *testing.T) { t.Fatal(err) } - dc := &appsv1.DeploymentConfig{} + deployment := &k8sappsv1.Deployment{} namespacedName := types.NamespacedName{ Name: tc.objName, Namespace: namespace, } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - // bump the amount of replicas in the dc - dc.Spec.Replicas = twoValue - err = cl.Update(context.TODO(), dc) + // bump the amount of replicas in the deployment + deployment.Spec.Replicas = &twoValue + err = cl.Update(context.TODO(), deployment) if err != nil { - subT.Errorf("error updating dc of %s: %v", tc.objName, err) + subT.Errorf("error updating deployment of %s: %v", tc.objName, err) } // re-run the reconciler @@ -196,13 +234,13 @@ func TestReplicaSystemReconciler(t *testing.T) { t.Fatal(err) } - err = cl.Get(context.TODO(), namespacedName, dc) + err = cl.Get(context.TODO(), namespacedName, deployment) if err != nil { subT.Errorf("error fetching object %s: %v", tc.objName, err) } - if tc.expectedAmountOfReplicas != dc.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, dc.Spec.Replicas) + if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, deployment.Spec.Replicas) } }) } diff --git a/pkg/3scale/amp/operator/system_searchd_options_provider.go b/pkg/3scale/amp/operator/system_searchd_options_provider.go index 0f5407f33..235d47886 100644 --- a/pkg/3scale/amp/operator/system_searchd_options_provider.go +++ b/pkg/3scale/amp/operator/system_searchd_options_provider.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -85,7 +86,7 @@ func (s *SystemSearchdOptionsProvider) podTemplateLabels() map[string]string { labels[k] = v } - labels["deploymentConfig"] = "system-searchd" + labels[reconcilers.DeploymentLabelSelector] = "system-searchd" return labels } diff --git a/pkg/3scale/amp/operator/system_searchd_options_provider_test.go b/pkg/3scale/amp/operator/system_searchd_options_provider_test.go index 31802c929..56ba91011 100644 --- a/pkg/3scale/amp/operator/system_searchd_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_searchd_options_provider_test.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -49,10 +50,10 @@ func testSystemSearchdLabels() map[string]string { func testSystemSearchdPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "searchd", - "deploymentConfig": "system-searchd", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "searchd", + reconcilers.DeploymentLabelSelector: "system-searchd", } addExpectedMeteringLabels(labels, "system-searchd", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler.go b/pkg/3scale/amp/operator/system_searchd_reconciler.go index 4b41439cd..bd98282b2 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler.go @@ -1,14 +1,11 @@ package operator import ( - appsv1 "github.com/openshift/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" - "github.com/3scale/3scale-operator/pkg/common" "github.com/3scale/3scale-operator/pkg/reconcilers" ) @@ -28,14 +25,13 @@ func (r *SystemSearchdReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // 3scale 2.13 -> 2.14 - err = r.upgradeFromSphinx() - if err != nil { - return reconcile.Result{}, err + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, } // Service - err = r.ReconcileService(searchd.Service(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(searchd.Service(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -46,52 +42,35 @@ func (r *SystemSearchdReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // DC - searchdDCmutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigStrategyMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, - reconcilers.DeploymentConfigProbesMutator, - reconcilers.DeploymentConfigArgsMutator, + // Deployment + searchdDeploymentMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentStrategyMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentProbesMutator, + reconcilers.DeploymentArgsMutator, ) - err = r.ReconcileDeploymentConfig(searchd.DeploymentConfig(), searchdDCmutator) + err = r.ReconcileDeployment(searchd.Deployment(), searchdDeploymentMutator) if err != nil { return reconcile.Result{}, err } - return reconcile.Result{}, nil -} - -func (r *SystemSearchdReconciler) upgradeFromSphinx() error { - // The upgrade procedure is simply based on: - // * Delete "old" DC called system-sphinx (if found) - // * The regular reconciling logic will create a new DC called system-searchd - - oldService := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "system-sphinx", Namespace: r.apiManager.Namespace}, - } - common.TagObjectToDelete(oldService) - err := r.ReconcileResource(&v1.Service{}, oldService, reconcilers.CreateOnlyMutator) + // 3scale 2.14 -> 2.15 + // Overriding the Deployment health check because the system-searchd PVC is ReadWriteOnce and so it can't be assigned across multiple nodes (pods) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemSearchdDeploymentName, r.apiManager.GetNamespace(), true, r.Client()) if err != nil { - return err - } - - oldDC := &appsv1.DeploymentConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "system-sphinx", Namespace: r.apiManager.Namespace}, + return reconcile.Result{}, err } - common.TagObjectToDelete(oldDC) - err = r.ReconcileResource(&appsv1.DeploymentConfig{}, oldDC, reconcilers.CreateOnlyMutator) - if err != nil { - return err + if !isMigrated { + return reconcile.Result{Requeue: true}, nil } - return nil + return reconcile.Result{}, nil } func SystemSearchd(cr *appsv1alpha1.APIManager) (*component.SystemSearchd, error) { diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler_test.go b/pkg/3scale/amp/operator/system_searchd_reconciler_test.go index 17c0dfeca..06871933c 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler_test.go @@ -2,6 +2,7 @@ package operator import ( "context" + k8sappsv1 "k8s.io/api/apps/v1" "testing" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -10,14 +11,11 @@ import ( appsv1 "github.com/openshift/api/apps/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" fakeclientset "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -35,7 +33,13 @@ func TestSystemSearchdReconciler(t *testing.T) { objs := []runtime.Object{apimanager} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) if err != nil { t.Fatal(err) } @@ -62,7 +66,7 @@ func TestSystemSearchdReconciler(t *testing.T) { }{ {"PVC", component.SystemSearchdPVCName, &v1.PersistentVolumeClaim{}}, {"Service", component.SystemSearchdServiceName, &v1.Service{}}, - {"DC", component.SystemSearchdDeploymentName, &appsv1.DeploymentConfig{}}, + {"DC", component.SystemSearchdDeploymentName, &k8sappsv1.Deployment{}}, } for _, tc := range cases { @@ -80,128 +84,3 @@ func TestSystemSearchdReconciler(t *testing.T) { }) } } - -func TestUpgradeFromSphinx(t *testing.T) { - var ( - log = logf.Log.WithName("upgrade_test") - ctx = context.TODO() - s = scheme.Scheme - apimanager = testSearchdBasicApimanager() - ) - - s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - - cases := []struct { - testName string - initialState []runtime.Object - }{ - {"Start from scratch", nil}, - {"only sphinx dc/service", []runtime.Object{ - testOldSphinxDC(namespace), testOldSphinxSvc(namespace), - }}, - {"only searchd dc/service", []runtime.Object{ - testsearchdDC(namespace), testsearchdSvc(namespace), - }}, - {"both sphinx and searchd dc/service", []runtime.Object{ - testOldSphinxDC(namespace), testOldSphinxSvc(namespace), - testsearchdDC(namespace), testsearchdSvc(namespace), - }}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - objs := []runtime.Object{} - cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() - clientAPIReader := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() - clientset := fakeclientset.NewSimpleClientset() - recorder := record.NewFakeRecorder(10000) - baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) - baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) - - reconciler := NewSystemSearchdReconciler(baseAPIManagerLogicReconciler) - _, err = reconciler.Reconcile() - if err != nil { - subT.Fatal(err) - } - - // Sphinx Service should not be there - sphinxServiceKey := client.ObjectKey{Name: "system-sphinx", Namespace: namespace} - err = cl.Get(ctx, sphinxServiceKey, &v1.Service{}) - if err == nil { - subT.Fatalf("reading an object expected to be deleted: %s", sphinxServiceKey) - } - if !errors.IsNotFound(err) { - subT.Fatalf("unexpected error reading object %s: %v", sphinxServiceKey, err) - } - - // Sphinx DC should not be there - sphinxDCKey := client.ObjectKey{Name: "system-sphinx", Namespace: namespace} - err = cl.Get(ctx, sphinxDCKey, &appsv1.DeploymentConfig{}) - if err == nil { - subT.Fatalf("reading an object expected to be deleted: %s", sphinxDCKey) - } - if !errors.IsNotFound(err) { - subT.Fatalf("unexpected error reading object %s: %v", sphinxDCKey, err) - } - - opts := &component.SystemSearchdOptions{} - searchd := component.NewSystemSearchd(opts) - // Searchd Service should be there - searchdServiceKey := client.ObjectKey{Name: searchd.Service().Name, Namespace: namespace} - err = cl.Get(ctx, searchdServiceKey, &v1.Service{}) - if err != nil { - subT.Fatalf("error fetching object %s: %v", searchdServiceKey, err) - } - // Searchd DC should be there - searchDCKey := client.ObjectKey{Name: searchd.DeploymentConfig().Name, Namespace: namespace} - err = cl.Get(ctx, searchDCKey, &appsv1.DeploymentConfig{}) - if err != nil { - subT.Fatalf("error fetching object %s: %v", searchDCKey, err) - } - }) - } -} - -func testOldSphinxDC(namespace string) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "system-sphinx", - Namespace: namespace, - }, - } -} - -func testOldSphinxSvc(namespace string) *v1.Service { - return &v1.Service{ - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "system-sphinx", - Namespace: namespace, - }, - } -} - -func testsearchdSvc(namespace string) *v1.Service { - opts := &component.SystemSearchdOptions{} - svc := component.NewSystemSearchd(opts).Service() - svc.Namespace = namespace - return svc -} - -func testsearchdDC(namespace string) *appsv1.DeploymentConfig { - opts := &component.SystemSearchdOptions{} - dc := component.NewSystemSearchd(opts).DeploymentConfig() - dc.Namespace = namespace - return dc -} diff --git a/pkg/3scale/amp/operator/zync_reconciler.go b/pkg/3scale/amp/operator/zync_reconciler.go index 0b45cca5e..9685694c1 100644 --- a/pkg/3scale/amp/operator/zync_reconciler.go +++ b/pkg/3scale/amp/operator/zync_reconciler.go @@ -64,7 +64,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ZyncName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ZyncName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } @@ -91,7 +91,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncQueDeploymentName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncQueDeploymentName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } @@ -127,7 +127,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncDatabaseDeploymentName, r.apiManager.GetNamespace(), r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncDatabaseDeploymentName, r.apiManager.GetNamespace(), false, r.Client()) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/helper/job.go b/pkg/helper/job.go index 7067dd6e5..4c85f531b 100644 --- a/pkg/helper/job.go +++ b/pkg/helper/job.go @@ -1,11 +1,15 @@ package helper import ( + "context" "fmt" "strings" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) // UIDBasedJobName returns a Job name that is compromised of the provided prefix, @@ -26,3 +30,27 @@ func UIDBasedJobName(prefix string, uid types.UID) (string, error) { return jobName, err } + +// HasJobCompleted checks if the provided Job has completed +func HasJobCompleted(jName string, jNamespace string, client k8sclient.Client) bool { + job := &batchv1.Job{} + err := client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: jNamespace, + Name: jName, + }, job) + + // Return false on error + if err != nil { + return false + } + + // Check if Job has completed + jobConditions := job.Status.Conditions + for _, condition := range jobConditions { + if condition.Type == batchv1.JobComplete && condition.Status == corev1.ConditionTrue { + return true + } + } + + return false +} diff --git a/pkg/upgrade/migrate_deployment_config_2.15.go b/pkg/upgrade/migrate_deployment_config_2.15.go index 27339349f..5cfb6a2b1 100644 --- a/pkg/upgrade/migrate_deployment_config_2.15.go +++ b/pkg/upgrade/migrate_deployment_config_2.15.go @@ -15,7 +15,7 @@ import ( // MigrateDeploymentConfigToDeployment verifies the Deployment is healthy and then deletes the corresponding DeploymentConfig // 3scale 2.14 -> 2.15 -func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, client k8sclient.Client) (bool, error) { +func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, overrideDeploymentHealth bool, client k8sclient.Client) (bool, error) { deploymentConfig := &appsv1.DeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: dName, @@ -32,21 +32,31 @@ func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, client return false, fmt.Errorf("error getting deploymentconfig %s: %v", deploymentConfig.Name, err) } - // Verify that the Deployment is healthy + // Check Deployment health deployment := &k8sappsv1.Deployment{} err = client.Get(context.TODO(), k8sclient.ObjectKey{ Namespace: dNamespace, Name: dName, }, deployment) + + // Return error if can't get Deployment if err != nil && !k8serr.IsNotFound(err) { return false, fmt.Errorf("error getting deployment %s: %w", deployment.Name, err) } - if k8serr.IsNotFound(err) || !helper.IsDeploymentAvailable(deployment) { - log.V(1).Info(fmt.Sprintf("deployment %s is not yet available", deployment.Name)) + + // Requeue if Deployment doesn't exist yet + if k8serr.IsNotFound(err) { + log.V(1).Info(fmt.Sprintf("deployment %s does not exist", deployment.Name)) + return false, nil + } + + // Requeue if Deployment isn't healthy and override is set to false, otherwise proceed + if !helper.IsDeploymentAvailable(deployment) && !overrideDeploymentHealth { + log.V(1).Info(fmt.Sprintf("deployment %s is not yet available and overrideDeploymentHealth is set to %t", deployment.Name, overrideDeploymentHealth)) return false, nil } - // Delete the DeploymentConfig because the Deployment replacing it is healthy + // Delete the DeploymentConfig because the Deployment replacing it is healthy or overrideDeploymentHealth is set to true err = client.Delete(context.TODO(), deploymentConfig) if err != nil { if !k8serr.IsNotFound(err) { diff --git a/pkg/upgrade/upgrade_sphinx_reference_2.14.go b/pkg/upgrade/upgrade_sphinx_reference_2.14.go deleted file mode 100644 index a159aabfe..000000000 --- a/pkg/upgrade/upgrade_sphinx_reference_2.14.go +++ /dev/null @@ -1,19 +0,0 @@ -package upgrade - -import ( - appsv1 "github.com/openshift/api/apps/v1" - - "github.com/3scale/3scale-operator/pkg/reconcilers" -) - -// DeploymentConfigPodTemplateLabelsMutator ensures pod template labels are reconciled -func SphinxAddressReference(desired, existing *appsv1.DeploymentConfig) (bool, error) { - var changed bool - changed = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "THINKING_SPHINX_ADDRESS") - - // Not in desired, it should be removed from existing - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "THINKING_SPHINX_CONFIGURATION_FILE") - changed = changed || tmpChanged - - return changed, nil -} diff --git a/pkg/upgrade/upgrade_system_backend_urls_2.14.go b/pkg/upgrade/upgrade_system_backend_urls_2.14.go deleted file mode 100644 index 07974662b..000000000 --- a/pkg/upgrade/upgrade_system_backend_urls_2.14.go +++ /dev/null @@ -1,30 +0,0 @@ -package upgrade - -import ( - appsv1 "github.com/openshift/api/apps/v1" - - "github.com/3scale/3scale-operator/pkg/reconcilers" -) - -// SystemBackendUrls reconciles environment variables for Backend URLs on system DC -func SystemBackendUrls(desired, existing *appsv1.DeploymentConfig) (bool, error) { - var changed bool - - // Remove old env var - tmpChanged := reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "APICAST_BACKEND_ROOT_ENDPOINT") - changed = changed || tmpChanged - - // Remove old env var - tmpChanged = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "BACKEND_ROUTE") - changed = changed || tmpChanged - - // Add new env vars - tmpChanged = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "BACKEND_URL") - changed = changed || tmpChanged - - // Add new env vars - tmpChanged = reconcilers.DeploymentConfigEnvVarReconciler(desired, existing, "BACKEND_PUBLIC_URL") - changed = changed || tmpChanged - - return changed, nil -} From 24d8c0a4fd21edcddf3c049ccf138e6919fa5c80 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Mon, 22 Jan 2024 11:25:31 -0500 Subject: [PATCH 07/23] THREESCALE-10391 Fix zync route management --- pkg/3scale/amp/operator/apicast_reconciler.go | 4 +- pkg/3scale/amp/operator/backend_reconciler.go | 6 +-- .../amp/operator/memcached_reconciler.go | 2 +- pkg/3scale/amp/operator/system_reconciler.go | 4 +- .../amp/operator/system_searchd_reconciler.go | 2 +- pkg/3scale/amp/operator/zync_reconciler.go | 6 +-- .../apimanager_managed_routes_event_mapper.go | 11 ++--- ...anager_managed_routes_event_mapper_test.go | 17 +++---- pkg/upgrade/migrate_deployment_config_2.15.go | 44 ++++++++++++++++++- 9 files changed, 70 insertions(+), 26 deletions(-) diff --git a/pkg/3scale/amp/operator/apicast_reconciler.go b/pkg/3scale/amp/operator/apicast_reconciler.go index a5687fa9c..6ce962e27 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler.go +++ b/pkg/3scale/amp/operator/apicast_reconciler.go @@ -100,7 +100,7 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ApicastStagingName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ApicastStagingName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } @@ -147,7 +147,7 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ApicastProductionName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ApicastProductionName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/backend_reconciler.go b/pkg/3scale/amp/operator/backend_reconciler.go index e8ba14292..4357ae60a 100644 --- a/pkg/3scale/amp/operator/backend_reconciler.go +++ b/pkg/3scale/amp/operator/backend_reconciler.go @@ -39,7 +39,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.BackendCronName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.BackendCronName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } @@ -59,7 +59,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendListenerName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendListenerName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } @@ -96,7 +96,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendWorkerName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.BackendWorkerName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/memcached_reconciler.go b/pkg/3scale/amp/operator/memcached_reconciler.go index b9fa7a6b1..8aa1cee96 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler.go +++ b/pkg/3scale/amp/operator/memcached_reconciler.go @@ -40,7 +40,7 @@ func (r *MemcachedReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemMemcachedDeploymentName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemMemcachedDeploymentName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index 5a6d21022..128cfc07a 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -201,7 +201,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemAppDeploymentName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemAppDeploymentName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } @@ -232,7 +232,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.SystemSidekiqName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.SystemSidekiqName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler.go b/pkg/3scale/amp/operator/system_searchd_reconciler.go index bd98282b2..7a4bb7c2b 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler.go @@ -62,7 +62,7 @@ func (r *SystemSearchdReconciler) Reconcile() (reconcile.Result, error) { // 3scale 2.14 -> 2.15 // Overriding the Deployment health check because the system-searchd PVC is ReadWriteOnce and so it can't be assigned across multiple nodes (pods) - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemSearchdDeploymentName, r.apiManager.GetNamespace(), true, r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemSearchdDeploymentName, r.apiManager.GetNamespace(), true, r.Client(), nil) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/zync_reconciler.go b/pkg/3scale/amp/operator/zync_reconciler.go index 9685694c1..9d6be6808 100644 --- a/pkg/3scale/amp/operator/zync_reconciler.go +++ b/pkg/3scale/amp/operator/zync_reconciler.go @@ -64,7 +64,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ZyncName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.ZyncName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } @@ -91,7 +91,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncQueDeploymentName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncQueDeploymentName, r.apiManager.GetNamespace(), false, r.Client(), r.BaseReconciler.Scheme()) if err != nil { return reconcile.Result{}, err } @@ -127,7 +127,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { } // 3scale 2.14 -> 2.15 - isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncDatabaseDeploymentName, r.apiManager.GetNamespace(), false, r.Client()) + isMigrated, err = upgrade.MigrateDeploymentConfigToDeployment(component.ZyncDatabaseDeploymentName, r.apiManager.GetNamespace(), false, r.Client(), nil) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/handlers/apimanager_managed_routes_event_mapper.go b/pkg/handlers/apimanager_managed_routes_event_mapper.go index 346a176bc..305b996e3 100644 --- a/pkg/handlers/apimanager_managed_routes_event_mapper.go +++ b/pkg/handlers/apimanager_managed_routes_event_mapper.go @@ -2,12 +2,13 @@ package handlers import ( "context" + "github.com/3scale/3scale-operator/pkg/reconcilers" appscommon "github.com/3scale/3scale-operator/apis/apps" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/go-logr/logr" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -62,19 +63,19 @@ func (h *APIManagerRoutesEventMapper) getAPIManagerOwnerReconcileRequest(object return request } - // If the OwnerReference of the received object is a DeploymentConfig and + // If the OwnerReference of the received object is a Deployment and // its name is Zync Que's name then we fetch that Object and recursively // try to find an OwnerReference that is an APIManager. If it is found // we return it. - zyncQueKind := "DeploymentConfig" - zyncQueGroup := appsv1.GroupVersion.Group + zyncQueKind := reconcilers.DeploymentKind + zyncQueGroup := k8sappsv1.GroupName // An alternative to hardcode Zync-Que name would be just try to recurse // OwnerReferences until there are no more of them. That would be // potentially more costly. zyncQueDeploymentName := component.ZyncQueDeploymentName if ref.Kind == zyncQueKind && refGV.Group == zyncQueGroup && ref.Name == zyncQueDeploymentName { h.Logger.V(2).Info("OwnerReference to Zync-Que detected. Recursively looking for APIManager OwnerReferences...") - existing := &appsv1.DeploymentConfig{} + existing := &k8sappsv1.Deployment{} getErr := h.K8sClient.Get(context.Background(), types.NamespacedName{Name: ref.Name, Namespace: object.GetNamespace()}, existing) if getErr != nil { // If there's an error getting the object it might be due to diff --git a/pkg/handlers/apimanager_managed_routes_event_mapper_test.go b/pkg/handlers/apimanager_managed_routes_event_mapper_test.go index 69da83318..97b834f0f 100644 --- a/pkg/handlers/apimanager_managed_routes_event_mapper_test.go +++ b/pkg/handlers/apimanager_managed_routes_event_mapper_test.go @@ -1,6 +1,7 @@ package handlers import ( + "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -18,7 +19,7 @@ import ( appscommon "github.com/3scale/3scale-operator/apis/apps" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" ) func TestAPIManagerRoutesEventMapperMap(t *testing.T) { @@ -35,10 +36,10 @@ func TestAPIManagerRoutesEventMapperMap(t *testing.T) { }, } - zyncQue := &appsv1.DeploymentConfig{ + zyncQue := &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", + Kind: reconcilers.DeploymentKind, + APIVersion: reconcilers.DeploymentAPIVersion, }, ObjectMeta: metav1.ObjectMeta{ Name: component.ZyncQueDeploymentName, @@ -57,12 +58,12 @@ func TestAPIManagerRoutesEventMapperMap(t *testing.T) { s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err := k8sappsv1.AddToScheme(s) if err != nil { t.Fatal(err) } - err = routev1.AddToScheme(s) + err = routev1.Install(s) if err != nil { t.Fatal(err) } @@ -126,8 +127,8 @@ func TestAPIManagerRoutesEventMapperMap(t *testing.T) { Name: "asecret", }, metav1.OwnerReference{ - APIVersion: appsv1.GroupVersion.String(), - Kind: "DeploymentConfig", + APIVersion: reconcilers.DeploymentAPIVersion, + Kind: reconcilers.DeploymentKind, Name: component.ZyncQueDeploymentName, }, }, diff --git a/pkg/upgrade/migrate_deployment_config_2.15.go b/pkg/upgrade/migrate_deployment_config_2.15.go index 5cfb6a2b1..c6a7ac977 100644 --- a/pkg/upgrade/migrate_deployment_config_2.15.go +++ b/pkg/upgrade/migrate_deployment_config_2.15.go @@ -5,17 +5,21 @@ import ( "fmt" appsv1 "github.com/openshift/api/apps/v1" + routev1 "github.com/openshift/api/route/v1" k8sappsv1 "k8s.io/api/apps/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/helper" ) // MigrateDeploymentConfigToDeployment verifies the Deployment is healthy and then deletes the corresponding DeploymentConfig // 3scale 2.14 -> 2.15 -func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, overrideDeploymentHealth bool, client k8sclient.Client) (bool, error) { +func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, overrideDeploymentHealth bool, client k8sclient.Client, scheme *runtime.Scheme) (bool, error) { deploymentConfig := &appsv1.DeploymentConfig{ ObjectMeta: metav1.ObjectMeta{ Name: dName, @@ -56,6 +60,14 @@ func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, overri return false, nil } + // Transfer zync-que route ownership + if deployment.Name == component.ZyncQueDeploymentName && scheme != nil { + err = transferZyncRoutesOwnership(deployment, client, scheme) + if err != nil { + return false, err + } + } + // Delete the DeploymentConfig because the Deployment replacing it is healthy or overrideDeploymentHealth is set to true err = client.Delete(context.TODO(), deploymentConfig) if err != nil { @@ -67,3 +79,33 @@ func MigrateDeploymentConfigToDeployment(dName string, dNamespace string, overri log.Info(fmt.Sprintf("%s Deployment has replaced its corresponding DeploymentConfig", deployment.Name)) return true, nil } + +func transferZyncRoutesOwnership(deployment *k8sappsv1.Deployment, client k8sclient.Client, scheme *runtime.Scheme) error { + listOps := []k8sclient.ListOption{ + k8sclient.InNamespace(deployment.Namespace), + } + + routeList := &routev1.RouteList{} + err := client.List(context.TODO(), routeList, listOps...) + if err != nil { + return fmt.Errorf("failed to list routes to transfer ownership: %w", err) + } + + for _, rt := range routeList.Items { + for _, ownerRef := range rt.ObjectMeta.OwnerReferences { + if ownerRef.Name == component.ZyncQueDeploymentName { + err = controllerutil.SetOwnerReference(deployment, &rt, scheme) + if err != nil { + return fmt.Errorf("failed to set ownerRef for route %v : %w", rt, err) + } + + err = client.Update(context.TODO(), &rt) + if err != nil { + return fmt.Errorf("failed to update ownerRef for route %v : %w", rt, err) + } + } + } + } + + return nil +} From 705b83d703c7c08d110d212f3ba455826a22a41d Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Thu, 18 Jan 2024 17:20:40 -0500 Subject: [PATCH 08/23] THREESCALE-10393 Refactored Deployments to set images directly --- pkg/3scale/amp/component/apicast.go | 32 ++----- pkg/3scale/amp/component/backend.go | 70 +++------------ pkg/3scale/amp/component/memcached.go | 21 +---- pkg/3scale/amp/component/system.go | 58 +++--------- pkg/3scale/amp/component/system_searchd.go | 21 +---- pkg/3scale/amp/component/zync.go | 64 +++---------- pkg/3scale/amp/operator/apicast_reconciler.go | 9 +- .../amp/operator/apicast_reconciler_test.go | 90 ++++++++++--------- pkg/3scale/amp/operator/backend_reconciler.go | 11 ++- .../amp/operator/memcached_reconciler.go | 7 +- pkg/3scale/amp/operator/system_reconciler.go | 4 +- .../amp/operator/system_searchd_reconciler.go | 7 +- pkg/3scale/amp/operator/zync_reconciler.go | 11 ++- pkg/reconcilers/deployment.go | 23 ----- pkg/reconcilers/deployment_test.go | 54 ----------- 15 files changed, 137 insertions(+), 345 deletions(-) diff --git a/pkg/3scale/amp/component/apicast.go b/pkg/3scale/amp/component/apicast.go index ae2d788ad..196e96c3c 100644 --- a/pkg/3scale/amp/component/apicast.go +++ b/pkg/3scale/amp/component/apicast.go @@ -90,7 +90,7 @@ func (apicast *Apicast) ProductionService() *v1.Service { } } -func (apicast *Apicast) StagingDeployment() *k8sappsv1.Deployment { +func (apicast *Apicast) StagingDeployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ @@ -132,7 +132,7 @@ func (apicast *Apicast) StagingDeployment() *k8sappsv1.Deployment { { Ports: apicast.stagingContainerPorts(), Env: apicast.buildApicastStagingEnv(), - Image: "amp-apicast:latest", + Image: containerImage, ImagePullPolicy: v1.PullIfNotPresent, Name: ApicastStagingName, Resources: apicast.Options.StagingResourceRequirements, @@ -165,7 +165,7 @@ func (apicast *Apicast) StagingDeployment() *k8sappsv1.Deployment { } } -func (apicast *Apicast) ProductionDeployment() *k8sappsv1.Deployment { +func (apicast *Apicast) ProductionDeployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ @@ -206,7 +206,7 @@ func (apicast *Apicast) ProductionDeployment() *k8sappsv1.Deployment { InitContainers: []v1.Container{ { Name: ApicastProductionInitContainerName, - Image: "amp-apicast:latest", + Image: containerImage, Command: []string{"sh", "-c", "until $(curl --output /dev/null --silent --fail --head http://system-master:3000/status); do sleep $SLEEP_SECONDS; done"}, Env: []v1.EnvVar{ { @@ -220,7 +220,7 @@ func (apicast *Apicast) ProductionDeployment() *k8sappsv1.Deployment { { Ports: apicast.productionContainerPorts(), Env: apicast.buildApicastProductionEnv(), - Image: "amp-apicast:latest", + Image: containerImage, ImagePullPolicy: v1.PullIfNotPresent, Name: ApicastProductionName, Resources: apicast.Options.ProductionResourceRequirements, @@ -729,19 +729,6 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { func (apicast *Apicast) productionDeploymentAnnotations() map[string]string { annotations := map[string]string{} - // Image trigger annotation should always be present - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: ApicastProductionInitContainerName, - Tag: fmt.Sprintf("amp-apicast:%v", apicast.Options.ImageTag), - }, - { - Name: ApicastProductionName, - Tag: fmt.Sprintf("amp-apicast:%v", apicast.Options.ImageTag), - }, - }) - annotations[reconcilers.DeploymentImageTriggerAnnotation] = imageTriggerString - for _, customPolicy := range apicast.Options.ProductionCustomPolicies { annotations[customPolicy.AnnotationKey()] = customPolicy.AnnotationValue() } @@ -761,15 +748,6 @@ func (apicast *Apicast) productionDeploymentAnnotations() map[string]string { func (apicast *Apicast) stagingDeploymentAnnotations() map[string]string { annotations := map[string]string{} - // Image trigger annotation should always be present - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: ApicastStagingName, - Tag: fmt.Sprintf("amp-apicast:%v", apicast.Options.ImageTag), - }, - }) - annotations[reconcilers.DeploymentImageTriggerAnnotation] = imageTriggerString - for _, customPolicy := range apicast.Options.StagingCustomPolicies { annotations[customPolicy.AnnotationKey()] = customPolicy.AnnotationValue() } diff --git a/pkg/3scale/amp/component/backend.go b/pkg/3scale/amp/component/backend.go index ce94bf952..d83eb9737 100644 --- a/pkg/3scale/amp/component/backend.go +++ b/pkg/3scale/amp/component/backend.go @@ -1,7 +1,6 @@ package component import ( - "fmt" "strconv" "github.com/3scale/3scale-operator/pkg/helper" @@ -62,13 +61,12 @@ func NewBackend(options *BackendOptions) *Backend { return &Backend{Options: options} } -func (backend *Backend) WorkerDeployment() *k8sappsv1.Deployment { +func (backend *Backend) WorkerDeployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: BackendWorkerName, - Labels: backend.Options.CommonWorkerLabels, - Annotations: backend.backendWorkerDeploymentAnnotations(), + Name: BackendWorkerName, + Labels: backend.Options.CommonWorkerLabels, }, Spec: k8sappsv1.DeploymentSpec{ Strategy: k8sappsv1.DeploymentStrategy{ @@ -102,7 +100,7 @@ func (backend *Backend) WorkerDeployment() *k8sappsv1.Deployment { InitContainers: []v1.Container{ { Name: "backend-redis-svc", - Image: "amp-backend:latest", + Image: containerImage, Command: []string{ "/opt/app/entrypoint.sh", "sh", @@ -115,7 +113,7 @@ func (backend *Backend) WorkerDeployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: BackendWorkerName, - Image: "amp-backend:latest", + Image: containerImage, Args: []string{"bin/3scale_backend_worker", "run"}, Env: backend.buildBackendWorkerEnv(), Resources: backend.Options.WorkerResourceRequirements, @@ -146,13 +144,12 @@ func (backend *Backend) WorkerDeployment() *k8sappsv1.Deployment { } } -func (backend *Backend) CronDeployment() *k8sappsv1.Deployment { +func (backend *Backend) CronDeployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: BackendCronName, - Labels: backend.Options.CommonCronLabels, - Annotations: backend.backendCronDeploymentAnnotations(), + Name: BackendCronName, + Labels: backend.Options.CommonCronLabels, }, Spec: k8sappsv1.DeploymentSpec{ Strategy: k8sappsv1.DeploymentStrategy{ @@ -186,7 +183,7 @@ func (backend *Backend) CronDeployment() *k8sappsv1.Deployment { InitContainers: []v1.Container{ { Name: "backend-redis-svc", - Image: "amp-backend:latest", + Image: containerImage, Command: []string{ "/opt/app/entrypoint.sh", "sh", @@ -199,7 +196,7 @@ func (backend *Backend) CronDeployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: "backend-cron", - Image: "amp-backend:latest", + Image: containerImage, Args: []string{"touch /tmp/healthy && backend-cron"}, Env: backend.buildBackendCronEnv(), Resources: backend.Options.CronResourceRequirements, @@ -224,13 +221,12 @@ func (backend *Backend) CronDeployment() *k8sappsv1.Deployment { } } -func (backend *Backend) ListenerDeployment() *k8sappsv1.Deployment { +func (backend *Backend) ListenerDeployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: BackendListenerName, - Labels: backend.Options.CommonListenerLabels, - Annotations: backend.backendListenerDeploymentAnnotations(), + Name: BackendListenerName, + Labels: backend.Options.CommonListenerLabels, }, Spec: k8sappsv1.DeploymentSpec{ Strategy: k8sappsv1.DeploymentStrategy{ @@ -264,7 +260,7 @@ func (backend *Backend) ListenerDeployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: BackendListenerName, - Image: "amp-backend:latest", + Image: containerImage, Args: []string{"bin/3scale_backend", "start", "-e", "production", "-p", "3000", "-x", "/dev/stdout"}, Ports: backend.listenerPorts(), Env: backend.buildBackendListenerEnv(), @@ -543,41 +539,3 @@ func (backend *Backend) workerPorts() []v1.ContainerPort { return ports } - -func (backend *Backend) backendWorkerDeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: BackendInitContainerName, - Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), - }, - { - Name: BackendWorkerName, - Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} - -func (backend *Backend) backendCronDeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: BackendInitContainerName, - Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), - }, - { - Name: BackendCronName, - Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} - -func (backend *Backend) backendListenerDeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: BackendListenerName, - Tag: fmt.Sprintf("amp-backend:%v", backend.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} diff --git a/pkg/3scale/amp/component/memcached.go b/pkg/3scale/amp/component/memcached.go index 316564175..870a3b487 100644 --- a/pkg/3scale/amp/component/memcached.go +++ b/pkg/3scale/amp/component/memcached.go @@ -1,8 +1,6 @@ package component import ( - "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" k8sappsv1 "k8s.io/api/apps/v1" @@ -23,15 +21,14 @@ func NewMemcached(options *MemcachedOptions) *Memcached { return &Memcached{Options: options} } -func (m *Memcached) Deployment() *k8sappsv1.Deployment { +func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { var memcachedReplicas int32 = 1 return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: SystemMemcachedDeploymentName, - Labels: m.Options.DeploymentLabels, - Annotations: m.memcacheDeploymentAnnotations(), + Name: SystemMemcachedDeploymentName, + Labels: m.Options.DeploymentLabels, }, Spec: k8sappsv1.DeploymentSpec{ Strategy: k8sappsv1.DeploymentStrategy{ @@ -56,7 +53,7 @@ func (m *Memcached) Deployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: "memcache", - Image: "system-memcached:latest", + Image: containerImage, Command: []string{"memcached", "-m", "64"}, Ports: []v1.ContainerPort{ {HostPort: 0, @@ -104,13 +101,3 @@ func (m *Memcached) Deployment() *k8sappsv1.Deployment { }, } } - -func (m *Memcached) memcacheDeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: "memcache", - Tag: fmt.Sprintf("system-memcached:%v", m.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} diff --git a/pkg/3scale/amp/component/system.go b/pkg/3scale/amp/component/system.go index 90021dee1..7ad4bda2e 100644 --- a/pkg/3scale/amp/component/system.go +++ b/pkg/3scale/amp/component/system.go @@ -1,7 +1,6 @@ package component import ( - "fmt" "github.com/3scale/3scale-operator/pkg/reconcilers" k8sappsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" @@ -549,13 +548,12 @@ func (system *System) volumesForSystemAppLifecycleHookPods() []v1.VolumeMount { return res } -func (system *System) AppDeployment() *k8sappsv1.Deployment { +func (system *System) AppDeployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: SystemAppDeploymentName, - Labels: system.Options.CommonAppLabels, - Annotations: system.appDeploymentAnnotations(), + Name: SystemAppDeploymentName, + Labels: system.Options.CommonAppLabels, }, Spec: k8sappsv1.DeploymentSpec{ Strategy: k8sappsv1.DeploymentStrategy{ @@ -590,7 +588,7 @@ func (system *System) AppDeployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: SystemAppMasterContainerName, - Image: "amp-system:latest", + Image: containerImage, Args: []string{"env", "TENANT_MODE=master", "PORT=3002", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, Ports: system.appMasterPorts(), Env: system.buildAppMasterContainerEnv(), @@ -641,7 +639,7 @@ func (system *System) AppDeployment() *k8sappsv1.Deployment { }, { Name: SystemAppProviderContainerName, - Image: "amp-system:latest", + Image: containerImage, Args: []string{"env", "TENANT_MODE=provider", "PORT=3000", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, Ports: system.appProviderPorts(), Env: system.buildAppProviderContainerEnv(), @@ -692,7 +690,7 @@ func (system *System) AppDeployment() *k8sappsv1.Deployment { }, { Name: SystemAppDeveloperContainerName, - Image: "amp-system:latest", + Image: containerImage, Args: []string{"env", "PORT=3001", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb"}, Ports: system.appDeveloperPorts(), Env: system.buildAppDeveloperContainerEnv(), @@ -894,13 +892,12 @@ func (system *System) SidekiqPodVolumes() []v1.Volume { return res } -func (system *System) SidekiqDeployment() *k8sappsv1.Deployment { +func (system *System) SidekiqDeployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: SystemSidekiqName, - Labels: system.Options.CommonSidekiqLabels, - Annotations: system.sidekiqDeploymentAnnotations(), + Name: SystemSidekiqName, + Labels: system.Options.CommonSidekiqLabels, }, Spec: k8sappsv1.DeploymentSpec{ Strategy: k8sappsv1.DeploymentStrategy{ @@ -935,7 +932,7 @@ func (system *System) SidekiqDeployment() *k8sappsv1.Deployment { InitContainers: []v1.Container{ { Name: "check-svc", - Image: "amp-system:latest", + Image: containerImage, Command: []string{ "bash", "-c", @@ -947,7 +944,7 @@ func (system *System) SidekiqDeployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: SystemSidekiqName, - Image: "amp-system:latest", + Image: containerImage, Args: []string{"rake", "sidekiq:worker", "RAILS_MAX_THREADS=25"}, Env: system.buildSystemSidekiqContainerEnv(), Resources: *system.Options.SidekiqContainerResourceRequirements, @@ -1340,36 +1337,3 @@ func (system *System) appDeveloperPorts() []v1.ContainerPort { return ports } - -func (system *System) sidekiqDeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: SystemSideKiqInitContainerName, - Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), - }, - { - Name: SystemSidekiqName, - Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} - -func (system *System) appDeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: SystemAppProviderContainerName, - Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), - }, - { - Name: SystemAppDeveloperContainerName, - Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), - }, - { - Name: SystemAppMasterContainerName, - Tag: fmt.Sprintf("amp-system:%v", system.Options.ImageTag), - }, - }) - - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} diff --git a/pkg/3scale/amp/component/system_searchd.go b/pkg/3scale/amp/component/system_searchd.go index 512355edb..5d32e6d13 100644 --- a/pkg/3scale/amp/component/system_searchd.go +++ b/pkg/3scale/amp/component/system_searchd.go @@ -1,7 +1,6 @@ package component import ( - "fmt" "github.com/3scale/3scale-operator/pkg/reconcilers" k8sappsv1 "k8s.io/api/apps/v1" @@ -49,15 +48,14 @@ func (s *SystemSearchd) Service() *v1.Service { } } -func (s *SystemSearchd) Deployment() *k8sappsv1.Deployment { +func (s *SystemSearchd) Deployment(containerImage string) *k8sappsv1.Deployment { var searchdReplicas int32 = 1 return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: SystemSearchdDeploymentName, - Labels: s.Options.Labels, - Annotations: s.DeploymentAnnotations(), + Name: SystemSearchdDeploymentName, + Labels: s.Options.Labels, }, Spec: k8sappsv1.DeploymentSpec{ Replicas: &searchdReplicas, @@ -92,7 +90,7 @@ func (s *SystemSearchd) Deployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: SystemSearchdDeploymentName, - Image: "system-searchd:latest", + Image: containerImage, ImagePullPolicy: v1.PullIfNotPresent, VolumeMounts: []v1.VolumeMount{ { @@ -157,14 +155,3 @@ func (s *SystemSearchd) PVC() *v1.PersistentVolumeClaim { }, } } - -func (s *SystemSearchd) DeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: SystemSearchdDeploymentName, - Tag: fmt.Sprintf("system-searchd:%v", s.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} - -} diff --git a/pkg/3scale/amp/component/zync.go b/pkg/3scale/amp/component/zync.go index b5e730ba4..856be0880 100644 --- a/pkg/3scale/amp/component/zync.go +++ b/pkg/3scale/amp/component/zync.go @@ -1,7 +1,6 @@ package component import ( - "fmt" "github.com/3scale/3scale-operator/pkg/reconcilers" k8sappsv1 "k8s.io/api/apps/v1" @@ -165,13 +164,12 @@ func (zync *Zync) QueRole() *rbacv1.Role { } } -func (zync *Zync) Deployment() *k8sappsv1.Deployment { +func (zync *Zync) Deployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: ZyncName, - Labels: zync.Options.CommonZyncLabels, - Annotations: zync.deploymentAnnotations(), + Name: ZyncName, + Labels: zync.Options.CommonZyncLabels, }, Spec: k8sappsv1.DeploymentSpec{ Replicas: &zync.Options.ZyncReplicas, @@ -192,7 +190,7 @@ func (zync *Zync) Deployment() *k8sappsv1.Deployment { InitContainers: []v1.Container{ { Name: ZyncInitContainerName, - Image: "amp-zync:latest", + Image: containerImage, Command: []string{ "bash", "-c", @@ -220,7 +218,7 @@ func (zync *Zync) Deployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: ZyncName, - Image: "amp-zync:latest", + Image: containerImage, Ports: zync.zyncPorts(), Env: zync.commonZyncEnvVars(), LivenessProbe: &v1.Probe{ @@ -289,13 +287,12 @@ func (zync *Zync) commonZyncEnvVars() []v1.EnvVar { }, } } -func (zync *Zync) QueDeployment() *k8sappsv1.Deployment { +func (zync *Zync) QueDeployment(containerImage string) *k8sappsv1.Deployment { return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: ZyncQueDeploymentName, - Labels: zync.Options.CommonZyncQueLabels, - Annotations: zync.queDeploymentAnnotations(), + Name: ZyncQueDeploymentName, + Labels: zync.Options.CommonZyncQueLabels, }, Spec: k8sappsv1.DeploymentSpec{ Replicas: &zync.Options.ZyncQueReplicas, @@ -333,7 +330,7 @@ func (zync *Zync) QueDeployment() *k8sappsv1.Deployment { Name: "que", Command: []string{"/usr/bin/bash"}, Args: []string{"-c", "bundle exec rake 'que[--worker-count 10]'"}, - Image: "amp-zync:latest", + Image: containerImage, ImagePullPolicy: v1.PullAlways, LivenessProbe: &v1.Probe{ FailureThreshold: 3, @@ -368,15 +365,14 @@ func (zync *Zync) QueDeployment() *k8sappsv1.Deployment { } } -func (zync *Zync) DatabaseDeployment() *k8sappsv1.Deployment { +func (zync *Zync) DatabaseDeployment(containerImage string) *k8sappsv1.Deployment { var zyncDatabaseReplicas int32 = 1 return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ - Name: ZyncDatabaseDeploymentName, - Labels: zync.Options.CommonZyncDatabaseLabels, - Annotations: zync.databaseDeploymentAnnotations(), + Name: ZyncDatabaseDeploymentName, + Labels: zync.Options.CommonZyncDatabaseLabels, }, Spec: k8sappsv1.DeploymentSpec{ Replicas: &zyncDatabaseReplicas, @@ -401,7 +397,7 @@ func (zync *Zync) DatabaseDeployment() *k8sappsv1.Deployment { Containers: []v1.Container{ { Name: "postgresql", - Image: " ", + Image: containerImage, Ports: []v1.ContainerPort{ { ContainerPort: 5432, @@ -572,37 +568,3 @@ func (zync *Zync) zyncPorts() []v1.ContainerPort { return ports } - -func (zync *Zync) deploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: ZyncInitContainerName, - Tag: fmt.Sprintf("amp-zync:%v", zync.Options.ImageTag), - }, - { - Name: ZyncName, - Tag: fmt.Sprintf("amp-zync:%v", zync.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} - -func (zync *Zync) queDeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: "que", - Tag: fmt.Sprintf("amp-zync:%v", zync.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} - -func (zync *Zync) databaseDeploymentAnnotations() map[string]string { - imageTriggerString := reconcilers.CreateImageTriggerAnnotationString([]reconcilers.ContainerImage{ - { - Name: "postgresql", - Tag: fmt.Sprintf("zync-database-postgresql:%v", zync.Options.ImageTag), - }, - }) - return map[string]string{reconcilers.DeploymentImageTriggerAnnotation: imageTriggerString} -} diff --git a/pkg/3scale/amp/operator/apicast_reconciler.go b/pkg/3scale/amp/operator/apicast_reconciler.go index 6ce962e27..a43a4173b 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler.go +++ b/pkg/3scale/amp/operator/apicast_reconciler.go @@ -58,6 +58,11 @@ func NewApicastReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReco } func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + apicast, err := Apicast(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err @@ -94,7 +99,7 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { } // Staging Deployment - err = r.ReconcileDeployment(apicast.StagingDeployment(), reconcilers.DeploymentMutator(stagingMutators...)) + err = r.ReconcileDeployment(apicast.StagingDeployment(ampImages.Options.ApicastImage), reconcilers.DeploymentMutator(stagingMutators...)) if err != nil { return reconcile.Result{}, err } @@ -141,7 +146,7 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { } // Production Deployment - err = r.ReconcileDeployment(apicast.ProductionDeployment(), reconcilers.DeploymentMutator(productionMutators...)) + err = r.ReconcileDeployment(apicast.ProductionDeployment(ampImages.Options.ApicastImage), reconcilers.DeploymentMutator(productionMutators...)) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/apicast_reconciler_test.go b/pkg/3scale/amp/operator/apicast_reconciler_test.go index 8c15b1666..e0779c0d0 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler_test.go +++ b/pkg/3scale/amp/operator/apicast_reconciler_test.go @@ -218,6 +218,10 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { }, }, } + ampImages, err := AmpImages(apimanager) + if err != nil { + t.Fatal(err) + } // Existing Deployment has 1 custom policy defined: P1 // Desired Deployment has 1 custom policy defined: P2 @@ -230,7 +234,7 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { ProductionTracingConfig: &component.APIcastTracingConfig{}, } apicast := component.NewApicast(apicastOptions) - existingProdDeployment := apicast.ProductionDeployment() + existingProdDeployment := apicast.ProductionDeployment(ampImages.Options.ApicastImage) existingProdDeployment.Namespace = namespace // - Policy annotation for P1 added @@ -259,7 +263,7 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { objs := []runtime.Object{apimanager, existingProdDeployment, p2Secret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err = appsv1.AddToScheme(s) if err != nil { t.Fatal(err) } @@ -393,44 +397,6 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { } ) - apicastOptions := &component.ApicastOptions{ - StagingTracingConfig: &component.APIcastTracingConfig{}, - ProductionTracingConfig: &existingTracingConfig1, - } - apicast := component.NewApicast(apicastOptions) - existingProdDeployment := apicast.ProductionDeployment() - existingProdDeployment.Namespace = namespace - - // - Tracing Configuration 1 added into the Production Deployment with the expected key - existingTracingConfig1Found := false - for key := range existingProdDeployment.Annotations { - if existingTracingConfig1.AnnotationKey() == key { - existingTracingConfig1Found = true - } - } - - if !existingTracingConfig1Found { - t.Fatal("tracing config 1 annotation not found. Should have been created") - } - - existingTc1Secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, - ObjectMeta: metav1.ObjectMeta{Name: *existingTracingConfig1.TracingConfigSecretName, Namespace: namespace}, - Data: map[string][]byte{ - "config": []byte("some existing tracing config"), - }, - Type: v1.SecretTypeOpaque, - } - - desiredTc1Secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, - ObjectMeta: metav1.ObjectMeta{Name: *desiredTracingConfig1.TracingConfigSecretName, Namespace: namespace}, - Data: map[string][]byte{ - "config": []byte("some desired tracing config"), - }, - Type: v1.SecretTypeOpaque, - } - apimanager := &appsv1alpha1.APIManager{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -469,12 +435,54 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { }, }, } + ampImages, err := AmpImages(apimanager) + if err != nil { + t.Fatal(err) + } + + apicastOptions := &component.ApicastOptions{ + StagingTracingConfig: &component.APIcastTracingConfig{}, + ProductionTracingConfig: &existingTracingConfig1, + } + apicast := component.NewApicast(apicastOptions) + existingProdDeployment := apicast.ProductionDeployment(ampImages.Options.ApicastImage) + existingProdDeployment.Namespace = namespace + + // - Tracing Configuration 1 added into the Production Deployment with the expected key + existingTracingConfig1Found := false + for key := range existingProdDeployment.Annotations { + if existingTracingConfig1.AnnotationKey() == key { + existingTracingConfig1Found = true + } + } + + if !existingTracingConfig1Found { + t.Fatal("tracing config 1 annotation not found. Should have been created") + } + + existingTc1Secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: *existingTracingConfig1.TracingConfigSecretName, Namespace: namespace}, + Data: map[string][]byte{ + "config": []byte("some existing tracing config"), + }, + Type: v1.SecretTypeOpaque, + } + + desiredTc1Secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: *desiredTracingConfig1.TracingConfigSecretName, Namespace: namespace}, + Data: map[string][]byte{ + "config": []byte("some desired tracing config"), + }, + Type: v1.SecretTypeOpaque, + } // Objects to track in the fake client. objs := []runtime.Object{apimanager, existingProdDeployment, existingTc1Secret, desiredTc1Secret} s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) + err = appsv1.AddToScheme(s) if err != nil { t.Fatal(err) } diff --git a/pkg/3scale/amp/operator/backend_reconciler.go b/pkg/3scale/amp/operator/backend_reconciler.go index 4357ae60a..5586a1c8e 100644 --- a/pkg/3scale/amp/operator/backend_reconciler.go +++ b/pkg/3scale/amp/operator/backend_reconciler.go @@ -22,6 +22,11 @@ func NewBackendReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReco } func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + backend, err := Backend(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err @@ -33,7 +38,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { cronDeploymentMutator = append(cronDeploymentMutator, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeployment(backend.CronDeployment(), reconcilers.DeploymentMutator(cronDeploymentMutator...)) + err = r.ReconcileDeployment(backend.CronDeployment(ampImages.Options.BackendImage), reconcilers.DeploymentMutator(cronDeploymentMutator...)) if err != nil { return reconcile.Result{}, err } @@ -53,7 +58,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { listenerDeploymentMutator = append(listenerDeploymentMutator, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeployment(backend.ListenerDeployment(), reconcilers.DeploymentMutator(listenerDeploymentMutator...)) + err = r.ReconcileDeployment(backend.ListenerDeployment(ampImages.Options.BackendImage), reconcilers.DeploymentMutator(listenerDeploymentMutator...)) if err != nil { return reconcile.Result{}, err } @@ -90,7 +95,7 @@ func (r *BackendReconciler) Reconcile() (reconcile.Result, error) { workerDeploymentMutator = append(workerDeploymentMutator, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeployment(backend.WorkerDeployment(), reconcilers.DeploymentMutator(workerDeploymentMutator...)) + err = r.ReconcileDeployment(backend.WorkerDeployment(ampImages.Options.BackendImage), reconcilers.DeploymentMutator(workerDeploymentMutator...)) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/memcached_reconciler.go b/pkg/3scale/amp/operator/memcached_reconciler.go index 8aa1cee96..45f2c6405 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler.go +++ b/pkg/3scale/amp/operator/memcached_reconciler.go @@ -19,6 +19,11 @@ func NewMemcachedReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicRe } func (r *MemcachedReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + memcached, err := Memcached(r.apiManager) if err != nil { return reconcile.Result{}, err @@ -34,7 +39,7 @@ func (r *MemcachedReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, ) - err = r.ReconcileDeployment(memcached.Deployment(), mutator) + err = r.ReconcileDeployment(memcached.Deployment(ampImages.Options.SystemMemcachedImage), mutator) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index 128cfc07a..a73225a8f 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -173,7 +173,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { systemAppDeploymentMutators = append(systemAppDeploymentMutators, reconcilers.DeploymentReplicasMutator) } if !systemComponentNotReady { - err = r.ReconcileDeployment(system.AppDeployment(), reconcilers.DeploymentMutator(systemAppDeploymentMutators...)) + err = r.ReconcileDeployment(system.AppDeployment(ampImages.Options.SystemImage), reconcilers.DeploymentMutator(systemAppDeploymentMutators...)) if err != nil { return reconcile.Result{}, err } @@ -226,7 +226,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { sidekiqDeploymentMutators = append(sidekiqDeploymentMutators, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeployment(system.SidekiqDeployment(), reconcilers.DeploymentMutator(sidekiqDeploymentMutators...)) + err = r.ReconcileDeployment(system.SidekiqDeployment(ampImages.Options.SystemImage), reconcilers.DeploymentMutator(sidekiqDeploymentMutators...)) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler.go b/pkg/3scale/amp/operator/system_searchd_reconciler.go index 7a4bb7c2b..954acb9ed 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler.go @@ -20,6 +20,11 @@ func NewSystemSearchdReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLog } func (r *SystemSearchdReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + searchd, err := SystemSearchd(r.apiManager) if err != nil { return reconcile.Result{}, err @@ -55,7 +60,7 @@ func (r *SystemSearchdReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentProbesMutator, reconcilers.DeploymentArgsMutator, ) - err = r.ReconcileDeployment(searchd.Deployment(), searchdDeploymentMutator) + err = r.ReconcileDeployment(searchd.Deployment(ampImages.Options.SystemSearchdImage), searchdDeploymentMutator) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/zync_reconciler.go b/pkg/3scale/amp/operator/zync_reconciler.go index 9d6be6808..2973ea0db 100644 --- a/pkg/3scale/amp/operator/zync_reconciler.go +++ b/pkg/3scale/amp/operator/zync_reconciler.go @@ -22,6 +22,11 @@ func NewZyncReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReconci } func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { + ampImages, err := AmpImages(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + zync, err := Zync(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err @@ -58,7 +63,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { if r.apiManager.Spec.Zync.AppSpec.Replicas != nil { zyncMutators = append(zyncMutators, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeployment(zync.Deployment(), reconcilers.DeploymentMutator(zyncMutators...)) + err = r.ReconcileDeployment(zync.Deployment(ampImages.Options.ZyncImage), reconcilers.DeploymentMutator(zyncMutators...)) if err != nil { return reconcile.Result{}, err } @@ -85,7 +90,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { if r.apiManager.Spec.Zync.QueSpec.Replicas != nil { zyncQueMutators = append(zyncQueMutators, reconcilers.DeploymentReplicasMutator) } - err = r.ReconcileDeployment(zync.QueDeployment(), reconcilers.DeploymentMutator(zyncQueMutators...)) + err = r.ReconcileDeployment(zync.QueDeployment(ampImages.Options.ZyncImage), reconcilers.DeploymentMutator(zyncQueMutators...)) if err != nil { return reconcile.Result{}, err } @@ -121,7 +126,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, ) - err = r.ReconcileDeployment(zync.DatabaseDeployment(), zyncDBDMutator) + err = r.ReconcileDeployment(zync.DatabaseDeployment(ampImages.Options.ZyncDatabasePostgreSQLImage), zyncDBDMutator) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/reconcilers/deployment.go b/pkg/reconcilers/deployment.go index 33046eac3..e1853670b 100644 --- a/pkg/reconcilers/deployment.go +++ b/pkg/reconcilers/deployment.go @@ -1,7 +1,6 @@ package reconcilers import ( - "encoding/json" "fmt" "reflect" @@ -305,25 +304,3 @@ func DeploymentProbesMutator(desired, existing *k8sappsv1.Deployment) (bool, err return updated, nil } - -// CreateImageTriggerAnnotationString creates the annotation for Deployments to leverage existing ImageStreamTags -func CreateImageTriggerAnnotationString(containers []ContainerImage) string { - var triggers []ImageTrigger - - for _, container := range containers { - triggers = append(triggers, ImageTrigger{ - From: ImageTriggerFrom{ - Kind: "ImageStreamTag", - Name: container.Tag, - }, - FieldPath: fmt.Sprintf(`spec.template.spec.containers[?(@.name=="%v")].image`, container.Name), - }) - } - - if len(triggers) > 0 { - jsonData, _ := json.Marshal(triggers) - return string(jsonData) - } - - return "" -} diff --git a/pkg/reconcilers/deployment_test.go b/pkg/reconcilers/deployment_test.go index d1986c764..cf9bfa3c3 100644 --- a/pkg/reconcilers/deployment_test.go +++ b/pkg/reconcilers/deployment_test.go @@ -1153,57 +1153,3 @@ func TestDeploymentAnnotationsMutator(t *testing.T) { }) } } - -func TestCreateImageTriggerAnnotationString(t *testing.T) { - type args struct { - containers []ContainerImage - } - tests := []struct { - name string - args args - want string - }{ - { - name: "NoContainers", - args: args{ - containers: []ContainerImage{}, - }, - want: "", - }, - { - name: "OneContainer", - args: args{ - containers: []ContainerImage{ - { - Name: "container1", - Tag: "test-image:1.1", - }, - }, - }, - want: `[{"from":{"kind":"ImageStreamTag","name":"test-image:1.1"},"fieldPath":"spec.template.spec.containers[?(@.name==\"container1\")].image"}]`, - }, - { - name: "TwoContainers", - args: args{ - containers: []ContainerImage{ - { - Name: "container1", - Tag: "test-image:1.1", - }, - { - Name: "container2", - Tag: "test-image:2.2", - }, - }, - }, - want: `[{"from":{"kind":"ImageStreamTag","name":"test-image:1.1"},"fieldPath":"spec.template.spec.containers[?(@.name==\"container1\")].image"},{"from":{"kind":"ImageStreamTag","name":"test-image:2.2"},"fieldPath":"spec.template.spec.containers[?(@.name==\"container2\")].image"}]`, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := CreateImageTriggerAnnotationString(tt.args.containers); got != tt.want { - t.Errorf("CreateImageTriggerAnnotationString() = %v, want %v", got, tt.want) - } - }) - } -} From 903852b7c18dc26986300ad303a805f0cdb3561b Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Fri, 19 Jan 2024 10:57:50 -0500 Subject: [PATCH 09/23] THREESCALE-10393 Migrate MySQL DeploymentConfig to Deployment --- pkg/3scale/amp/component/system_mysql.go | 118 +++++++++-------- .../operator/system_mysql_options_provider.go | 5 +- .../system_mysql_options_provider_test.go | 9 +- .../amp/operator/system_mysql_reconciler.go | 46 +++++-- .../operator/system_mysql_reconciler_test.go | 120 ++++++++++++++++++ 5 files changed, 224 insertions(+), 74 deletions(-) create mode 100644 pkg/3scale/amp/operator/system_mysql_reconciler_test.go diff --git a/pkg/3scale/amp/component/system_mysql.go b/pkg/3scale/amp/component/system_mysql.go index 0a22b73ef..f808410f3 100644 --- a/pkg/3scale/amp/component/system_mysql.go +++ b/pkg/3scale/amp/component/system_mysql.go @@ -1,10 +1,10 @@ package component import ( - "fmt" - "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -34,14 +34,14 @@ func (mysql *SystemMysql) Service() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "system-mysql", Protocol: v1.ProtocolTCP, Port: 3306, TargetPort: intstr.FromInt(3306), }, }, - Selector: map[string]string{"deploymentConfig": "system-mysql"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-mysql"}, }, } } @@ -117,40 +117,26 @@ func (mysql *SystemMysql) PersistentVolumeClaim() *v1.PersistentVolumeClaim { } } -func (mysql *SystemMysql) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (mysql *SystemMysql) Deployment(containerImage string) *k8sappsv1.Deployment { + var mysqlReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: SystemMySQLDeploymentName, Labels: mysql.Options.DeploymentLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange}, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-mysql", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-mysql:%s", mysql.Options.ImageTag), - }, - }, + Replicas: &mysqlReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemMySQLDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemMySQLDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: mysql.Options.PodTemplateLabels, Annotations: mysql.Options.PodTemplateAnnotations, @@ -160,30 +146,46 @@ func (mysql *SystemMysql) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: mysql.Options.Tolerations, ServiceAccountName: "amp", //TODO make this configurable via flag Volumes: []v1.Volume{ - v1.Volume{ + { Name: "mysql-storage", - VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "mysql-storage", - ReadOnly: false}}, - }, v1.Volume{ + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "mysql-storage", + ReadOnly: false, + }, + }, + }, + { Name: "mysql-extra-conf", - VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "mysql-extra-conf"}}}, - }, v1.Volume{ + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "mysql-extra-conf", + }, + }, + }, + }, + { Name: "mysql-main-conf", - VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "mysql-main-conf"}}}}, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "mysql-main-conf", + }, + }, + }, + }, }, Containers: []v1.Container{ - v1.Container{ + { Name: "system-mysql", - Image: "system-mysql:latest", + Image: containerImage, Ports: []v1.ContainerPort{ - v1.ContainerPort{HostPort: 0, + { + HostPort: 0, ContainerPort: 3306, - Protocol: v1.ProtocolTCP}, + Protocol: v1.ProtocolTCP, + }, }, Env: []v1.EnvVar{ helper.EnvVarFromSecret("MYSQL_USER", SystemSecretSystemDatabaseSecretName, SystemSecretSystemDatabaseUserFieldName), @@ -198,25 +200,30 @@ func (mysql *SystemMysql) DeploymentConfig() *appsv1.DeploymentConfig { }, Resources: mysql.Options.ContainerResourceRequirements, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "mysql-storage", ReadOnly: false, MountPath: "/var/lib/mysql/data", - }, v1.VolumeMount{ + }, + { Name: "mysql-extra-conf", ReadOnly: false, MountPath: "/etc/my-extra.d", - }, v1.VolumeMount{ + }, + { Name: "mysql-main-conf", ReadOnly: false, - MountPath: "/etc/my-extra"}, + MountPath: "/etc/my-extra", + }, }, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 3306}}, + Type: intstr.Int, + IntVal: 3306, + }, + }, }, InitialDelaySeconds: 30, TimeoutSeconds: 0, @@ -227,7 +234,8 @@ func (mysql *SystemMysql) DeploymentConfig() *appsv1.DeploymentConfig { ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ - Command: []string{"/bin/sh", "-i", "-c", "MYSQL_PWD=\"$MYSQL_PASSWORD\" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE -e 'SELECT 1'"}}, + Command: []string{"/bin/sh", "-i", "-c", "MYSQL_PWD=\"$MYSQL_PASSWORD\" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE -e 'SELECT 1'"}, + }, }, InitialDelaySeconds: 10, TimeoutSeconds: 5, diff --git a/pkg/3scale/amp/operator/system_mysql_options_provider.go b/pkg/3scale/amp/operator/system_mysql_options_provider.go index 4872bed20..e71ca57d9 100644 --- a/pkg/3scale/amp/operator/system_mysql_options_provider.go +++ b/pkg/3scale/amp/operator/system_mysql_options_provider.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" "net/url" "strings" @@ -145,7 +146,7 @@ func (s *SystemMysqlOptionsProvider) setResourceRequirementsOptions() { s.mysqlOptions.ContainerResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if s.apimanager.Spec.System.DatabaseSpec != nil && @@ -209,7 +210,7 @@ func (s *SystemMysqlOptionsProvider) podTemplateLabels() map[string]string { } } - labels["deploymentConfig"] = "system-mysql" + labels[reconcilers.DeploymentLabelSelector] = "system-mysql" return labels } diff --git a/pkg/3scale/amp/operator/system_mysql_options_provider_test.go b/pkg/3scale/amp/operator/system_mysql_options_provider_test.go index 823265592..b02b66bb7 100644 --- a/pkg/3scale/amp/operator/system_mysql_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_mysql_options_provider_test.go @@ -10,6 +10,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -41,10 +42,10 @@ func testSystemMysqlDeploymentLabels() map[string]string { func testSystemMysqlPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "mysql", - "deploymentConfig": "system-mysql", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "mysql", + reconcilers.DeploymentLabelSelector: "system-mysql", } addExpectedMeteringLabels(labels, "system-mysql", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/system_mysql_reconciler.go b/pkg/3scale/amp/operator/system_mysql_reconciler.go index 389e36e95..36835fe91 100644 --- a/pkg/3scale/amp/operator/system_mysql_reconciler.go +++ b/pkg/3scale/amp/operator/system_mysql_reconciler.go @@ -4,6 +4,7 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -19,29 +20,48 @@ func NewSystemMySQLReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogic } func (r *SystemMySQLReconciler) Reconcile() (reconcile.Result, error) { + systemMySQLImage, err := SystemMySQLImage(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + systemMySQL, err := SystemMySQL(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } - // DC - dcMutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // MySQL Deployment + deploymentMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, ) - err = r.ReconcileDeploymentConfig(systemMySQL.DeploymentConfig(), dcMutator) + err = r.ReconcileDeployment(systemMySQL.Deployment(systemMySQLImage.Options.Image), deploymentMutator) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + // Overriding the Deployment health check because the mysql-storage PVC is ReadWriteOnce and so it can't be assigned across multiple nodes (pods) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemMySQLDeploymentName, r.apiManager.GetNamespace(), true, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + // Service - err = r.ReconcileService(systemMySQL.Service(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(systemMySQL.Service(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -58,7 +78,7 @@ func (r *SystemMySQLReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // PCV + // PVC err = r.ReconcilePersistentVolumeClaim(systemMySQL.PersistentVolumeClaim(), reconcilers.CreateOnlyMutator) if err != nil { return reconcile.Result{}, err diff --git a/pkg/3scale/amp/operator/system_mysql_reconciler_test.go b/pkg/3scale/amp/operator/system_mysql_reconciler_test.go new file mode 100644 index 000000000..bc5e02f19 --- /dev/null +++ b/pkg/3scale/amp/operator/system_mysql_reconciler_test.go @@ -0,0 +1,120 @@ +package operator + +import ( + "context" + "testing" + + appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" + "github.com/3scale/3scale-operator/pkg/3scale/amp/component" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + appsv1 "github.com/openshift/api/apps/v1" + imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + fakeclientset "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +func TestSystemMySQLReconcilerCreate(t *testing.T) { + var ( + appLabel = "someLabel" + name = "example-apimanager" + namespace = "operator-unittest" + trueValue = true + imageURL = "mysql:test" + wildcardDomain = "test.3scale.net" + tenantName = "someTenant" + log = logf.Log.WithName("operator_test") + ) + + ctx := context.TODO() + + apimanager := &appsv1alpha1.APIManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1alpha1.APIManagerSpec{ + APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ + AppLabel: &appLabel, + ImageStreamTagImportInsecure: &trueValue, + ResourceRequirementsEnabled: &trueValue, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + }, + System: &appsv1alpha1.SystemSpec{ + DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ + MySQL: &appsv1alpha1.SystemMySQLSpec{ + Image: &imageURL, + }, + }, + }, + }, + } + s := scheme.Scheme + s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) + err := imagev1.Install(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) + if err != nil { + t.Fatal(err) + } + + // Objects to track in the fake client. + objs := []runtime.Object{} + + // Create a fake client to mock API calls. + cl := fake.NewFakeClient(objs...) + clientAPIReader := fake.NewFakeClient(objs...) + clientset := fakeclientset.NewSimpleClientset() + recorder := record.NewFakeRecorder(10000) + + baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) + baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) + + reconciler := NewSystemMySQLReconciler(baseAPIManagerLogicReconciler) + _, err = reconciler.Reconcile() + if err != nil { + t.Fatal(err) + } + + cases := []struct { + testName string + objName string + obj client.Object + }{ + {"systemMySQL_Deployment", "system-mysql", &k8sappsv1.Deployment{}}, + {"systemMySQL_Service", "system-mysql", &v1.Service{}}, + {"systemMySQL_Main_CM", "mysql-main-conf", &v1.ConfigMap{}}, + {"systemMySQL_Extra_CM", "mysql-extra-conf", &v1.ConfigMap{}}, + {"systemMySQL_PVC", "mysql-storage", &v1.PersistentVolumeClaim{}}, + {"systemDatabaseSecret", component.SystemSecretSystemDatabaseSecretName, &v1.Secret{}}, + } + + for _, tc := range cases { + t.Run(tc.testName, func(subT *testing.T) { + obj := tc.obj + namespacedName := types.NamespacedName{ + Name: tc.objName, + Namespace: namespace, + } + err = cl.Get(context.TODO(), namespacedName, obj) + // object must exist, that is all required to be tested + if err != nil { + subT.Errorf("error fetching object %s: %v", tc.objName, err) + } + }) + } +} From 0d561dcd34283a9d8703470db63e7b233b7dddff Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Fri, 19 Jan 2024 12:37:08 -0500 Subject: [PATCH 10/23] THREESCALE-10393 Migrate PostgreSQL DeploymentConfig to Deployment --- pkg/3scale/amp/component/system_postgresql.go | 62 ++++++++----------- .../system_postgresql_options_provider.go | 5 +- ...system_postgresql_options_provider_test.go | 9 +-- .../operator/system_postgresql_reconciler.go | 44 +++++++++---- .../system_postgresql_reconciler_test.go | 11 +++- 5 files changed, 75 insertions(+), 56 deletions(-) diff --git a/pkg/3scale/amp/component/system_postgresql.go b/pkg/3scale/amp/component/system_postgresql.go index ca6462a90..33f44eed7 100644 --- a/pkg/3scale/amp/component/system_postgresql.go +++ b/pkg/3scale/amp/component/system_postgresql.go @@ -1,10 +1,10 @@ package component import ( - "fmt" - "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -34,14 +34,14 @@ func (p *SystemPostgreSQL) Service() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "system-postgresql", Protocol: v1.ProtocolTCP, Port: 5432, TargetPort: intstr.FromInt(5432), }, }, - Selector: map[string]string{"deploymentConfig": "system-postgresql"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-postgresql"}, }, } } @@ -63,7 +63,7 @@ func (p *SystemPostgreSQL) DataPersistentVolumeClaim() *v1.PersistentVolumeClaim }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ - v1.PersistentVolumeAccessMode("ReadWriteOnce"), + "ReadWriteOnce", }, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ @@ -76,40 +76,29 @@ func (p *SystemPostgreSQL) DataPersistentVolumeClaim() *v1.PersistentVolumeClaim } } -func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ +func (p *SystemPostgreSQL) Deployment(containerImage string) *k8sappsv1.Deployment { + var postgresReplicas int32 = 1 + + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", + Kind: reconcilers.DeploymentLabelSelector, APIVersion: "apps.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: SystemPostgreSQLDeploymentName, Labels: p.Options.DeploymentLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange}, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-postgresql", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-postgresql:%s", p.Options.ImageTag), - }, - }, + Replicas: &postgresReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemPostgreSQLDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemPostgreSQLDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: p.Options.PodTemplateLabels, Annotations: p.Options.PodTemplateAnnotations, @@ -119,7 +108,7 @@ func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { Tolerations: p.Options.Tolerations, ServiceAccountName: "amp", //TODO make this configurable via flag Volumes: []v1.Volume{ - v1.Volume{ + { Name: "postgresql-data", VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ @@ -129,11 +118,11 @@ func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { }, }, Containers: []v1.Container{ - v1.Container{ + { Name: "system-postgresql", - Image: "system-postgresql:latest", + Image: containerImage, Ports: []v1.ContainerPort{ - v1.ContainerPort{ + { ContainerPort: 5432, Protocol: v1.ProtocolTCP, }, @@ -148,7 +137,7 @@ func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { }, Resources: p.Options.ContainerResourceRequirements, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "postgresql-data", MountPath: "/var/lib/pgsql/data", }, @@ -171,7 +160,8 @@ func (p *SystemPostgreSQL) DeploymentConfig() *appsv1.DeploymentConfig { ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ - Command: []string{"/bin/sh", "-i", "-c", "psql -h 127.0.0.1 -U $POSTGRESQL_USER -q -d $POSTGRESQL_DATABASE -c 'SELECT 1'"}}, + Command: []string{"/bin/sh", "-i", "-c", "psql -h 127.0.0.1 -U $POSTGRESQL_USER -q -d $POSTGRESQL_DATABASE -c 'SELECT 1'"}, + }, }, InitialDelaySeconds: 10, PeriodSeconds: 30, diff --git a/pkg/3scale/amp/operator/system_postgresql_options_provider.go b/pkg/3scale/amp/operator/system_postgresql_options_provider.go index 91c71aff3..e37e1ab05 100644 --- a/pkg/3scale/amp/operator/system_postgresql_options_provider.go +++ b/pkg/3scale/amp/operator/system_postgresql_options_provider.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" "net/url" "strings" @@ -132,7 +133,7 @@ func (s *SystemPostgresqlOptionsProvider) setResourceRequirementsOptions() { s.options.ContainerResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if s.apimanager.Spec.System.DatabaseSpec != nil && @@ -194,7 +195,7 @@ func (s *SystemPostgresqlOptionsProvider) podTemplateLabels() map[string]string } } - labels["deploymentConfig"] = "system-postgresql" + labels[reconcilers.DeploymentLabelSelector] = "system-postgresql" return labels } diff --git a/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go b/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go index 877ddb72e..bbede0a6d 100644 --- a/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go @@ -2,6 +2,7 @@ package operator import ( "fmt" + "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "strings" "testing" @@ -41,10 +42,10 @@ func testSystemPostgreSQLDeploymentLabels() map[string]string { func testSystemPostgreSQLPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "postgresql", - "deploymentConfig": "system-postgresql", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "postgresql", + reconcilers.DeploymentLabelSelector: "system-postgresql", } addExpectedMeteringLabels(labels, "system-postgresql", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/system_postgresql_reconciler.go b/pkg/3scale/amp/operator/system_postgresql_reconciler.go index 8dbcc0e83..bf395270b 100644 --- a/pkg/3scale/amp/operator/system_postgresql_reconciler.go +++ b/pkg/3scale/amp/operator/system_postgresql_reconciler.go @@ -4,6 +4,7 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -20,29 +21,48 @@ func NewSystemPostgreSQLReconciler(baseAPIManagerLogicReconciler *BaseAPIManager } func (r *SystemPostgreSQLReconciler) Reconcile() (reconcile.Result, error) { + systemPostgreSQLImage, err := SystemPostgreSQLImage(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + systemPostgreSQL, err := SystemPostgreSQL(r.apiManager, r.Client()) if err != nil { return reconcile.Result{}, err } - // DC - dcMutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, + // PostgreSQL Deployment + deploymentMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, ) - err = r.ReconcileDeploymentConfig(systemPostgreSQL.DeploymentConfig(), dcMutator) + err = r.ReconcileDeployment(systemPostgreSQL.Deployment(systemPostgreSQLImage.Options.Image), deploymentMutator) if err != nil { return reconcile.Result{}, err } + // 3scale 2.14 -> 2.15 + // Overriding the Deployment health check because the postgresql-data PVC is ReadWriteOnce and so it can't be assigned across multiple nodes (pods) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(component.SystemPostgreSQLDeploymentName, r.apiManager.GetNamespace(), true, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + // Service - err = r.ReconcileService(systemPostgreSQL.Service(), reconcilers.CreateOnlyMutator) + err = r.ReconcileService(systemPostgreSQL.Service(), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go b/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go index 59f2e331b..39736b7fe 100644 --- a/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go @@ -10,6 +10,7 @@ import ( appsv1 "github.com/openshift/api/apps/v1" imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -60,7 +61,13 @@ func TestSystemPostgreSQLReconcilerCreate(t *testing.T) { } s := scheme.Scheme s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := imagev1.AddToScheme(s) + err := imagev1.Install(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) if err != nil { t.Fatal(err) } @@ -88,7 +95,7 @@ func TestSystemPostgreSQLReconcilerCreate(t *testing.T) { objName string obj client.Object }{ - {"systemPostgreSQL_DC", "system-postgresql", &appsv1.DeploymentConfig{}}, + {"systemPostgreSQL_Deployment", "system-postgresql", &k8sappsv1.Deployment{}}, {"systemPostgreSQL_Service", "system-postgresql", &v1.Service{}}, {"systemPostgreSQL_PVC", "postgresql-data", &v1.PersistentVolumeClaim{}}, {"systemDatabaseSecret", component.SystemSecretSystemDatabaseSecretName, &v1.Secret{}}, From cb07aa06e98f645f80a7eae17158e2fe14816113 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Fri, 19 Jan 2024 14:13:30 -0500 Subject: [PATCH 11/23] THREESCALE-10393 Migrate Redis DeploymentConfigs to Deployments --- pkg/3scale/amp/component/redis.go | 209 ++++++++---------- .../amp/operator/redis_options_provider.go | 7 +- .../operator/redis_options_provider_test.go | 17 +- pkg/3scale/amp/operator/redis_reconciler.go | 55 +++-- .../amp/operator/redis_reconciler_test.go | 14 +- pkg/upgrade/redis_2.14.go | 48 ---- 6 files changed, 155 insertions(+), 195 deletions(-) delete mode 100644 pkg/upgrade/redis_2.14.go diff --git a/pkg/3scale/amp/component/redis.go b/pkg/3scale/amp/component/redis.go index 88ed07f52..55afc57b0 100644 --- a/pkg/3scale/amp/component/redis.go +++ b/pkg/3scale/amp/component/redis.go @@ -1,12 +1,13 @@ package component import ( - "fmt" "path" "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/reconcilers" + imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,22 +27,22 @@ func NewRedis(options *RedisOptions) *Redis { return &Redis{Options: options} } -func (redis *Redis) BackendDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: redis.buildDeploymentConfigTypeMeta(), - ObjectMeta: redis.buildDeploymentConfigObjectMeta(), - Spec: redis.buildDeploymentConfigSpec(), +func (redis *Redis) BackendDeployment() *k8sappsv1.Deployment { + return &k8sappsv1.Deployment{ + TypeMeta: redis.buildDeploymentTypeMeta(), + ObjectMeta: redis.buildDeploymentObjectMeta(), + Spec: redis.buildDeploymentSpec(), } } -func (redis *Redis) buildDeploymentConfigTypeMeta() metav1.TypeMeta { +func (redis *Redis) buildDeploymentTypeMeta() metav1.TypeMeta { return metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", + Kind: reconcilers.DeploymentKind, + APIVersion: reconcilers.DeploymentAPIVersion, } } -func (redis *Redis) buildDeploymentConfigObjectMeta() metav1.ObjectMeta { +func (redis *Redis) buildDeploymentObjectMeta() metav1.ObjectMeta { return metav1.ObjectMeta{ Name: backendRedisObjectMetaName, Labels: redis.Options.BackendRedisLabels, @@ -51,63 +52,45 @@ func (redis *Redis) buildDeploymentConfigObjectMeta() metav1.ObjectMeta { const ( redisConfigVolumeName = "redis-config" - backendRedisObjectMetaName = "backend-redis" - backendRedisDCSelectorName = backendRedisObjectMetaName - backendRedisStorageVolumeName = "backend-redis-storage" - backendRedisConfigMapKey = "redis.conf" - backendRedisContainerName = "backend-redis" - backendRedisConfigPath = "/etc/redis.d/" + backendRedisObjectMetaName = "backend-redis" + backendRedisDeploymentSelectorName = backendRedisObjectMetaName + backendRedisStorageVolumeName = "backend-redis-storage" + backendRedisConfigMapKey = "redis.conf" + backendRedisContainerName = "backend-redis" + backendRedisConfigPath = "/etc/redis.d/" ) -func (redis *Redis) buildDeploymentConfigSpec() appsv1.DeploymentConfigSpec { - return appsv1.DeploymentConfigSpec{ +func (redis *Redis) buildDeploymentSpec() k8sappsv1.DeploymentSpec { + var redisReplicas int32 = 1 + + return k8sappsv1.DeploymentSpec{ Template: redis.buildPodTemplateSpec(), Strategy: redis.buildDeploymentStrategy(), - Selector: redis.buildDeploymentConfigSelector(), - Replicas: 1, - Triggers: redis.buildDeploymentConfigTriggers(), + Selector: redis.buildDeploymentSelector(), + Replicas: &redisReplicas, } } -func (redis *Redis) buildDeploymentStrategy() appsv1.DeploymentStrategy { - return appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, +func (redis *Redis) buildDeploymentStrategy() k8sappsv1.DeploymentStrategy { + return k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, } } func (redis *Redis) getSelectorLabels() map[string]string { return map[string]string{ - "deploymentConfig": backendRedisDCSelectorName, + reconcilers.DeploymentLabelSelector: backendRedisDeploymentSelectorName, } } -func (redis *Redis) buildDeploymentConfigSelector() map[string]string { - return redis.getSelectorLabels() -} - -func (redis *Redis) buildDeploymentConfigTriggers() appsv1.DeploymentTriggerPolicies { - return appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange, - }, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "backend-redis", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("backend-redis:%s", redis.Options.BackendImageTag), - }, - }, - }, +func (redis *Redis) buildDeploymentSelector() *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchLabels: redis.getSelectorLabels(), } } -func (redis *Redis) buildPodTemplateSpec() *v1.PodTemplateSpec { - return &v1.PodTemplateSpec{ +func (redis *Redis) buildPodTemplateSpec() v1.PodTemplateSpec { + return v1.PodTemplateSpec{ Spec: v1.PodSpec{ Affinity: redis.Options.BackendRedisAffinity, Tolerations: redis.Options.BackendRedisTolerations, @@ -126,7 +109,7 @@ func (redis *Redis) buildPodTemplateSpec() *v1.PodTemplateSpec { func (redis *Redis) buildPodVolumes() []v1.Volume { return []v1.Volume{ - v1.Volume{ + { Name: backendRedisStorageVolumeName, VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ @@ -134,7 +117,7 @@ func (redis *Redis) buildPodVolumes() []v1.Volume { }, }, }, - v1.Volume{ + { Name: redisConfigVolumeName, VolumeSource: v1.VolumeSource{ ConfigMap: &v1.ConfigMapVolumeSource{ @@ -142,7 +125,7 @@ func (redis *Redis) buildPodVolumes() []v1.Volume { Name: redisConfigVolumeName, }, Items: []v1.KeyToPath{ - v1.KeyToPath{ + { Key: backendRedisConfigMapKey, Path: backendRedisConfigMapKey, }, @@ -155,8 +138,8 @@ func (redis *Redis) buildPodVolumes() []v1.Volume { func (redis *Redis) buildPodContainers() []v1.Container { return []v1.Container{ - v1.Container{ - Image: "backend-redis:latest", + { + Image: redis.Options.BackendImage, ImagePullPolicy: v1.PullIfNotPresent, Name: backendRedisContainerName, Env: redis.buildEnv(), @@ -204,13 +187,13 @@ func (redis *Redis) buildPodContainerLivenessProbe() *v1.Probe { func (redis *Redis) buildPodContainerVolumeMounts() []v1.VolumeMount { return []v1.VolumeMount{ - v1.VolumeMount{ + { Name: backendRedisStorageVolumeName, // https://github.com/sclorg/redis-container/ images have // redis data directory hardcoded on /var/lib/redis/data MountPath: "/var/lib/redis/data", }, - v1.VolumeMount{ + { Name: redisConfigVolumeName, MountPath: backendRedisConfigPath, }, @@ -248,7 +231,7 @@ func (redis *Redis) buildServiceSpec() v1.ServiceSpec { func (redis *Redis) buildServicePorts() []v1.ServicePort { return []v1.ServicePort{ - v1.ServicePort{ + { Port: 6379, TargetPort: intstr.FromInt(6379), Protocol: v1.ProtocolTCP, @@ -258,7 +241,7 @@ func (redis *Redis) buildServicePorts() []v1.ServicePort { func (redis *Redis) buildServiceSelector() map[string]string { return map[string]string{ - "deploymentConfig": backendRedisDCSelectorName, + reconcilers.DeploymentLabelSelector: backendRedisDeploymentSelectorName, } } @@ -430,41 +413,27 @@ func (redis *Redis) BackendRedisSecret() *v1.Secret { ////// Begin System Redis -func (redis *Redis) SystemDeploymentConfig() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, +func (redis *Redis) SystemDeployment() *k8sappsv1.Deployment { + var redisReplicas int32 = 1 + + return &k8sappsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ Name: SystemRedisDeploymentName, Labels: redis.Options.SystemRedisLabels, }, - Spec: appsv1.DeploymentConfigSpec{ - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.DeploymentStrategyTypeRecreate, + Spec: k8sappsv1.DeploymentSpec{ + Strategy: k8sappsv1.DeploymentStrategy{ + Type: k8sappsv1.RecreateDeploymentStrategyType, }, MinReadySeconds: 0, - Triggers: appsv1.DeploymentTriggerPolicies{ - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnConfigChange}, - appsv1.DeploymentTriggerPolicy{ - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - Automatic: true, - ContainerNames: []string{ - "system-redis", - }, - From: v1.ObjectReference{ - Kind: "ImageStreamTag", - Name: fmt.Sprintf("system-redis:%s", redis.Options.SystemImageTag), - }, - }, + Replicas: &redisReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + reconcilers.DeploymentLabelSelector: SystemRedisDeploymentName, }, }, - Replicas: 1, - Selector: map[string]string{"deploymentConfig": SystemRedisDeploymentName}, - Template: &v1.PodTemplateSpec{ + Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: redis.Options.SystemRedisPodTemplateLabels, Annotations: redis.Options.SystemRedisPodTemplateAnnotations, @@ -474,43 +443,57 @@ func (redis *Redis) SystemDeploymentConfig() *appsv1.DeploymentConfig { Tolerations: redis.Options.SystemRedisTolerations, ServiceAccountName: "amp", //TODO make this configurable via flag Volumes: []v1.Volume{ - v1.Volume{ + { Name: "system-redis-storage", - VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "system-redis-storage", - ReadOnly: false}}, - }, v1.Volume{ + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "system-redis-storage", + ReadOnly: false, + }, + }, + }, { Name: "redis-config", - VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "redis-config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "redis-config", + }, + Items: []v1.KeyToPath{ + { + Key: "redis.conf", + Path: "redis.conf", + }, + }, }, - Items: []v1.KeyToPath{ - v1.KeyToPath{ - Key: "redis.conf", - Path: "redis.conf"}}}}}, + }, + }, }, Containers: []v1.Container{ - v1.Container{ + { Name: "system-redis", - Image: "system-redis:latest", + Image: redis.Options.SystemImage, Env: redis.buildEnv(), Resources: *redis.Options.SystemRedisContainerResourceRequirements, VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "system-redis-storage", ReadOnly: false, MountPath: "/var/lib/redis/data", - }, v1.VolumeMount{ + }, + { Name: "redis-config", ReadOnly: false, - MountPath: "/etc/redis.d/"}, + MountPath: "/etc/redis.d/", + }, }, LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{TCPSocket: &v1.TCPSocketAction{ - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 6379}}, + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 6379, + }, + }, }, InitialDelaySeconds: 10, TimeoutSeconds: 0, @@ -521,7 +504,8 @@ func (redis *Redis) SystemDeploymentConfig() *appsv1.DeploymentConfig { ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ - Command: []string{"container-entrypoint", "bash", "-c", "redis-cli set liveness-probe \"`date`\" | grep OK"}}, + Command: []string{"container-entrypoint", "bash", "-c", "redis-cli set liveness-probe \"`date`\" | grep OK"}, + }, }, InitialDelaySeconds: 30, TimeoutSeconds: 5, @@ -535,7 +519,8 @@ func (redis *Redis) SystemDeploymentConfig() *appsv1.DeploymentConfig { }, PriorityClassName: redis.Options.SystemRedisPriorityClassName, TopologySpreadConstraints: redis.Options.SystemRedisTopologySpreadConstraints, - }}, + }, + }, }, } } @@ -552,14 +537,14 @@ func (redis *Redis) SystemService() *v1.Service { }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ - v1.ServicePort{ + { Name: "redis", Protocol: v1.ProtocolTCP, Port: 6379, TargetPort: intstr.FromInt(6379), }, }, - Selector: map[string]string{"deploymentConfig": "system-redis"}, + Selector: map[string]string{reconcilers.DeploymentLabelSelector: "system-redis"}, }, } } @@ -576,7 +561,7 @@ func (redis *Redis) SystemPVC() *v1.PersistentVolumeClaim { }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ - v1.PersistentVolumeAccessMode("ReadWriteOnce"), + "ReadWriteOnce", }, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{"storage": resource.MustParse("1Gi")}, diff --git a/pkg/3scale/amp/operator/redis_options_provider.go b/pkg/3scale/amp/operator/redis_options_provider.go index f9b55b2af..043b0b428 100644 --- a/pkg/3scale/amp/operator/redis_options_provider.go +++ b/pkg/3scale/amp/operator/redis_options_provider.go @@ -7,6 +7,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -168,7 +169,7 @@ func (r *RedisOptionsProvider) setResourceRequirementsOptions() { r.options.SystemRedisContainerResourceRequirements = &v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if r.apimanager.Spec.Backend.RedisResources != nil { @@ -234,7 +235,7 @@ func (r *RedisOptionsProvider) systemRedisPodTemplateLabels() map[string]string labels[k] = v } - labels["deploymentConfig"] = "system-redis" + labels[reconcilers.DeploymentLabelSelector] = "system-redis" return labels } @@ -250,7 +251,7 @@ func (r *RedisOptionsProvider) backendRedisPodTemplateLabels() map[string]string labels[k] = v } - labels["deploymentConfig"] = "backend-redis" + labels[reconcilers.DeploymentLabelSelector] = "backend-redis" return labels } diff --git a/pkg/3scale/amp/operator/redis_options_provider_test.go b/pkg/3scale/amp/operator/redis_options_provider_test.go index 2901f3bb7..5ceb06156 100644 --- a/pkg/3scale/amp/operator/redis_options_provider_test.go +++ b/pkg/3scale/amp/operator/redis_options_provider_test.go @@ -8,6 +8,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -34,10 +35,10 @@ func testRedisSystemRedisLabels() map[string]string { func testRedisSystemRedisPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "system", - "threescale_component_element": "redis", - "deploymentConfig": "system-redis", + "app": appLabel, + "threescale_component": "system", + "threescale_component_element": "redis", + reconcilers.DeploymentLabelSelector: "system-redis", } addExpectedMeteringLabels(labels, "system-redis", helper.ApplicationType) @@ -61,10 +62,10 @@ func testRedisBackendRedisLabels() map[string]string { func testRedisBackendRedisPodTemplateLabels() map[string]string { labels := map[string]string{ - "app": appLabel, - "threescale_component": "backend", - "threescale_component_element": "redis", - "deploymentConfig": "backend-redis", + "app": appLabel, + "threescale_component": "backend", + "threescale_component_element": "redis", + reconcilers.DeploymentLabelSelector: "backend-redis", } addExpectedMeteringLabels(labels, "backend-redis", helper.ApplicationType) diff --git a/pkg/3scale/amp/operator/redis_reconciler.go b/pkg/3scale/amp/operator/redis_reconciler.go index 556cab850..10c17ae65 100644 --- a/pkg/3scale/amp/operator/redis_reconciler.go +++ b/pkg/3scale/amp/operator/redis_reconciler.go @@ -1,8 +1,9 @@ package operator import ( - appsv1 "github.com/openshift/api/apps/v1" + "github.com/3scale/3scale-operator/pkg/upgrade" imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -10,15 +11,14 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" - "github.com/3scale/3scale-operator/pkg/upgrade" ) -// RedisDependencyReconciler is a generic DependencyReconciler that reconciles +// RedisReconciler is a generic DependencyReconciler that reconciles // an internal Redis instance using the Redis options type RedisReconciler struct { *BaseAPIManagerLogicReconciler - DeploymentConfig func(redis *component.Redis) *appsv1.DeploymentConfig + Deployment func(redis *component.Redis) *k8sappsv1.Deployment Service func(redis *component.Redis) *corev1.Service ConfigMap func(redis *component.Redis) *corev1.ConfigMap PersistentVolumeClaim func(redis *component.Redis) *corev1.PersistentVolumeClaim @@ -32,7 +32,7 @@ func NewSystemRedisDependencyReconciler(baseAPIManagerLogicReconciler *BaseAPIMa return &RedisReconciler{ BaseAPIManagerLogicReconciler: baseAPIManagerLogicReconciler, - DeploymentConfig: (*component.Redis).SystemDeploymentConfig, + Deployment: (*component.Redis).SystemDeployment, Service: (*component.Redis).SystemService, ConfigMap: (*component.Redis).ConfigMap, PersistentVolumeClaim: (*component.Redis).SystemPVC, @@ -45,7 +45,7 @@ func NewBackendRedisDependencyReconciler(baseAPIManagerLogicReconciler *BaseAPIM return &RedisReconciler{ BaseAPIManagerLogicReconciler: baseAPIManagerLogicReconciler, - DeploymentConfig: (*component.Redis).BackendDeploymentConfig, + Deployment: (*component.Redis).BackendDeployment, Service: (*component.Redis).BackendService, ConfigMap: (*component.Redis).ConfigMap, PersistentVolumeClaim: (*component.Redis).BackendPVC, @@ -60,25 +60,38 @@ func (r *RedisReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - dcMutator := reconcilers.DeploymentConfigMutator( - reconcilers.DeploymentConfigImageChangeTriggerMutator, - reconcilers.DeploymentConfigContainerResourcesMutator, - reconcilers.DeploymentConfigAffinityMutator, - reconcilers.DeploymentConfigTolerationsMutator, - reconcilers.DeploymentConfigPodTemplateLabelsMutator, - reconcilers.DeploymentConfigPriorityClassMutator, - reconcilers.DeploymentConfigTopologySpreadConstraintsMutator, - reconcilers.DeploymentConfigPodTemplateAnnotationsMutator, - // 3scale 2.13 -> 2.14 - upgrade.Redis6CommandArgsEnv, + deploymentMutator := reconcilers.DeploymentMutator( + reconcilers.DeploymentContainerResourcesMutator, + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, ) - err = r.ReconcileDeploymentConfig(r.DeploymentConfig(redis), dcMutator) + redisDeployment := r.Deployment(redis) + err = r.ReconcileDeployment(redisDeployment, deploymentMutator) if err != nil { return reconcile.Result{}, err } - // redis Service - err = r.ReconcileService(r.Service(redis), reconcilers.CreateOnlyMutator) + // 3scale 2.14 -> 2.15 + // Overriding the Deployment health check because the redis PVCs are ReadWriteOnce and so they can't be assigned across multiple nodes (pods) + isMigrated, err := upgrade.MigrateDeploymentConfigToDeployment(redisDeployment.Name, r.apiManager.GetNamespace(), true, r.Client(), nil) + if err != nil { + return reconcile.Result{}, err + } + if !isMigrated { + return reconcile.Result{Requeue: true}, nil + } + + serviceMutators := []reconcilers.MutateFn{ + reconcilers.CreateOnlyMutator, + reconcilers.ServiceSelectorMutator, + } + + // Service + err = r.ReconcileService(r.Service(redis), reconcilers.ServiceMutator(serviceMutators...)) if err != nil { return reconcile.Result{}, err } @@ -103,7 +116,7 @@ func (r *RedisReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // Redis Secret + // Secret err = r.ReconcileSecret(r.Secret(redis), reconcilers.DefaultsOnlySecretMutator) if err != nil { return reconcile.Result{}, err diff --git a/pkg/3scale/amp/operator/redis_reconciler_test.go b/pkg/3scale/amp/operator/redis_reconciler_test.go index 7ddadda56..206b912ee 100644 --- a/pkg/3scale/amp/operator/redis_reconciler_test.go +++ b/pkg/3scale/amp/operator/redis_reconciler_test.go @@ -7,6 +7,8 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/reconcilers" + k8sappsv1 "k8s.io/api/apps/v1" + appsv1 "github.com/openshift/api/apps/v1" imagev1 "github.com/openshift/api/image/v1" v1 "k8s.io/api/core/v1" @@ -60,7 +62,13 @@ func TestRedisBackendDCReconcilerCreate(t *testing.T) { if err != nil { t.Fatal(err) } - err = appsv1.AddToScheme(s) + err = k8sappsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } + + // 3scale 2.14 -> 2.15 + err = appsv1.Install(s) if err != nil { t.Fatal(err) } @@ -89,7 +97,7 @@ func TestRedisBackendDCReconcilerCreate(t *testing.T) { objName string obj client.Object }{ - {"backend-redis", &appsv1.DeploymentConfig{}}, + {"backend-redis", &k8sappsv1.Deployment{}}, {"backend-redis", &v1.Service{}}, {"redis-config", &v1.ConfigMap{}}, {"backend-redis-storage", &v1.PersistentVolumeClaim{}}, @@ -99,7 +107,7 @@ func TestRedisBackendDCReconcilerCreate(t *testing.T) { objName string obj client.Object }{ - {"system-redis", &appsv1.DeploymentConfig{}}, + {"system-redis", &k8sappsv1.Deployment{}}, {"system-redis-storage", &v1.PersistentVolumeClaim{}}, {"system-redis", &imagev1.ImageStream{}}, {"system-redis", &v1.Service{}}, diff --git a/pkg/upgrade/redis_2.14.go b/pkg/upgrade/redis_2.14.go deleted file mode 100644 index f0429bfa2..000000000 --- a/pkg/upgrade/redis_2.14.go +++ /dev/null @@ -1,48 +0,0 @@ -package upgrade - -import ( - "fmt" - "reflect" - - "github.com/3scale/3scale-operator/pkg/common" - "github.com/3scale/3scale-operator/pkg/helper" - appsv1 "github.com/openshift/api/apps/v1" -) - -// Redis6CommandArgsEnv reconciles environment variables, command and args -func Redis6CommandArgsEnv(desired, existing *appsv1.DeploymentConfig) (bool, error) { - var updated bool - - desiredName := common.ObjectInfo(desired) - - if len(desired.Spec.Template.Spec.Containers) != 1 { - return false, fmt.Errorf("%s desired spec.template.spec.containers length changed to '%d', should be 1", desiredName, len(desired.Spec.Template.Spec.Containers)) - } - - if len(existing.Spec.Template.Spec.Containers) != 1 { - log.Info(fmt.Sprintf("%s spec.template.spec.containers length changed to '%d', recreating dc", desiredName, len(existing.Spec.Template.Spec.Containers))) - existing.Spec.Template.Spec.Containers = desired.Spec.Template.Spec.Containers - updated = true - } - - // Env Vars added in 2.14 - tmpChanged := helper.EnvVarReconciler( - desired.Spec.Template.Spec.Containers[0].Env, - &existing.Spec.Template.Spec.Containers[0].Env, - "REDIS_CONF") - updated = updated || tmpChanged - - // Command updated in 2.14 - if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[0].Command, desired.Spec.Template.Spec.Containers[0].Command) { - existing.Spec.Template.Spec.Containers[0].Command = desired.Spec.Template.Spec.Containers[0].Command - updated = true - } - - // Args updated in 2.14 - if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[0].Args, desired.Spec.Template.Spec.Containers[0].Args) { - existing.Spec.Template.Spec.Containers[0].Args = desired.Spec.Template.Spec.Containers[0].Args - updated = true - } - - return updated, nil -} From e6a53691eb00b7b38596cd61167f99f044ce359d Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Thu, 25 Jan 2024 16:33:05 -0500 Subject: [PATCH 12/23] THREESCALE-10698 Removed/Updated DeploymentConfig references in code --- apis/apps/v1alpha1/apimanager_types.go | 2 +- .../apps/apimanager_controller_test.go | 24 +- .../apps/apimanager_status_reconciler.go | 10 +- pkg/3scale/amp/component/evaluation.go | 8 +- .../amp/operator/backend_options_provider.go | 2 +- .../base_apimanager_logic_reconciler.go | 5 - .../operator/memcached_options_provider.go | 2 +- .../amp/operator/redis_reconciler_test.go | 2 +- .../amp/operator/system_options_provider.go | 2 +- .../system_searchd_options_provider.go | 2 +- .../system_searchd_reconciler_test.go | 2 +- .../apimanager_managed_routes_event_mapper.go | 2 +- pkg/helper/deploymentconfig.go | 40 - pkg/helper/envvarutils.go | 4 +- pkg/reconcilers/deployment.go | 11 +- pkg/reconcilers/deployment_test.go | 8 +- pkg/reconcilers/deploymentconfig.go | 324 ----- pkg/reconcilers/deploymentconfig_test.go | 1276 ----------------- pkg/reconcilers/secret.go | 2 +- pkg/restore/apimanager_restore.go | 10 +- 20 files changed, 46 insertions(+), 1692 deletions(-) delete mode 100644 pkg/helper/deploymentconfig.go delete mode 100644 pkg/reconcilers/deploymentconfig.go delete mode 100644 pkg/reconcilers/deploymentconfig_test.go diff --git a/apis/apps/v1alpha1/apimanager_types.go b/apis/apps/v1alpha1/apimanager_types.go index 4967df91c..072b7e779 100644 --- a/apis/apps/v1alpha1/apimanager_types.go +++ b/apis/apps/v1alpha1/apimanager_types.go @@ -130,7 +130,7 @@ func (s *APIManagerStatus) Equals(other *APIManagerStatus, logger logr.Logger) b // APIManager is the Schema for the apimanagers API // +kubebuilder:resource:path=apimanagers,scope=Namespaced // +operator-sdk:csv:customresourcedefinitions:displayName="APIManager" -// +operator-sdk:csv:customresourcedefinitions:resources={{"DeploymentConfig","apps.openshift.io/v1"}} +// +operator-sdk:csv:customresourcedefinitions:resources={{"Deployment","apps.openshift.io/v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"PersistentVolumeClaim","v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"Service","v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"Route","route.openshift.io/v1"}} diff --git a/controllers/apps/apimanager_controller_test.go b/controllers/apps/apimanager_controller_test.go index 977f57834..c66467ddc 100644 --- a/controllers/apps/apimanager_controller_test.go +++ b/controllers/apps/apimanager_controller_test.go @@ -12,8 +12,8 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - appsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -143,10 +143,10 @@ var _ = Describe("APIManager controller", func() { return err == nil }, 5*time.Minute, 5*time.Second).Should(BeTrue()) - fmt.Fprintf(GinkgoWriter, "Waiting for all APIManager managed DeploymentConfigs\n") - err = waitForAllAPIManagerStandardDeploymentConfigs(testNamespace, 5*time.Second, 15*time.Minute, GinkgoWriter) + fmt.Fprintf(GinkgoWriter, "Waiting for all APIManager managed Deployments\n") + err = waitForAllAPIManagerStandardDeployments(testNamespace, 5*time.Second, 15*time.Minute, GinkgoWriter) Expect(err).ToNot(HaveOccurred()) - fmt.Fprintf(GinkgoWriter, "All APIManager managed DeploymentConfigs are ready\n") + fmt.Fprintf(GinkgoWriter, "All APIManager managed Deployments are ready\n") fmt.Fprintf(GinkgoWriter, "Waiting for all APIManager managed Routes\n") err = waitForAllAPIManagerStandardRoutes(testNamespace, 5*time.Second, 15*time.Minute, wildcardDomain, GinkgoWriter) @@ -164,8 +164,8 @@ var _ = Describe("APIManager controller", func() { }) }) -func waitForAllAPIManagerStandardDeploymentConfigs(namespace string, retryInterval, timeout time.Duration, w io.Writer) error { - deploymentConfigNames := []string{ // TODO gather this from constants/somewhere centralized +func waitForAllAPIManagerStandardDeployments(namespace string, retryInterval, timeout time.Duration, w io.Writer) error { + deploymentNames := []string{ // TODO gather this from constants/somewhere centralized "apicast-production", "apicast-staging", "backend-cron", @@ -183,23 +183,23 @@ func waitForAllAPIManagerStandardDeploymentConfigs(namespace string, retryInterv "zync-database", } - for _, dcName := range deploymentConfigNames { - lookupKey := types.NamespacedName{Name: dcName, Namespace: namespace} - createdDeployment := &appsv1.DeploymentConfig{} + for _, dName := range deploymentNames { + lookupKey := types.NamespacedName{Name: dName, Namespace: namespace} + createdDeployment := &k8sappsv1.Deployment{} Eventually(func() bool { err := testK8sClient.Get(context.Background(), lookupKey, createdDeployment) if err != nil { return false } - if helper.IsDeploymentConfigAvailable(createdDeployment) { - fmt.Fprintf(w, "DeploymentConfig '%s' available\n", dcName) + if helper.IsDeploymentAvailable(createdDeployment) { + fmt.Fprintf(w, "Deployment '%s' available\n", dName) return true } availableReplicas := createdDeployment.Status.AvailableReplicas desiredReplicas := createdDeployment.Spec.Replicas - fmt.Fprintf(w, "Waiting for full availability of %s DeploymentConfig (%d/%d)\n", dcName, availableReplicas, desiredReplicas) + fmt.Fprintf(w, "Waiting for full availability of %s Deployment (%d/%d)\n", dName, availableReplicas, desiredReplicas) return false }, timeout, retryInterval).Should(BeTrue()) diff --git a/controllers/apps/apimanager_status_reconciler.go b/controllers/apps/apimanager_status_reconciler.go index e89c69538..f01589d50 100644 --- a/controllers/apps/apimanager_status_reconciler.go +++ b/controllers/apps/apimanager_status_reconciler.go @@ -127,14 +127,14 @@ func (s *APIManagerStatusReconciler) expectedDeploymentNames(instance *appsv1alp func (s *APIManagerStatusReconciler) deploymentsAvailable(existingDeployments []k8sappsv1.Deployment) bool { expectedDeploymentNames := s.expectedDeploymentNames(s.apimanagerResource) for _, deploymentName := range expectedDeploymentNames { - foundExistingDCIdx := -1 - for idx, existingDC := range existingDeployments { - if existingDC.Name == deploymentName { - foundExistingDCIdx = idx + foundExistingDeploymentIdx := -1 + for idx, existingDeployment := range existingDeployments { + if existingDeployment.Name == deploymentName { + foundExistingDeploymentIdx = idx break } } - if foundExistingDCIdx == -1 || !helper.IsDeploymentAvailable(&existingDeployments[foundExistingDCIdx]) { + if foundExistingDeploymentIdx == -1 || !helper.IsDeploymentAvailable(&existingDeployments[foundExistingDeploymentIdx]) { return false } } diff --git a/pkg/3scale/amp/component/evaluation.go b/pkg/3scale/amp/component/evaluation.go index d6a88495d..e22422a9f 100644 --- a/pkg/3scale/amp/component/evaluation.go +++ b/pkg/3scale/amp/component/evaluation.go @@ -2,7 +2,7 @@ package component import ( "github.com/3scale/3scale-operator/pkg/common" - appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ) @@ -15,10 +15,10 @@ func NewEvaluation() *Evaluation { func (evaluation *Evaluation) RemoveContainersResourceRequestsAndLimits(objects []common.KubernetesObject) { for _, obj := range objects { - dc, ok := obj.(*appsv1.DeploymentConfig) + deployment, ok := obj.(*k8sappsv1.Deployment) if ok { - for containerIdx := range dc.Spec.Template.Spec.Containers { - container := &dc.Spec.Template.Spec.Containers[containerIdx] + for containerIdx := range deployment.Spec.Template.Spec.Containers { + container := &deployment.Spec.Template.Spec.Containers[containerIdx] container.Resources = v1.ResourceRequirements{} } } diff --git a/pkg/3scale/amp/operator/backend_options_provider.go b/pkg/3scale/amp/operator/backend_options_provider.go index 0e75e8cfb..9224c856d 100644 --- a/pkg/3scale/amp/operator/backend_options_provider.go +++ b/pkg/3scale/amp/operator/backend_options_provider.go @@ -122,7 +122,7 @@ func (o *OperatorBackendOptionsProvider) setResourceRequirementsOptions() { o.backendOptions.CronResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if o.apimanager.Spec.Backend.ListenerSpec.Resources != nil { diff --git a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go index b6822fb6a..c99a04b83 100644 --- a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go +++ b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go @@ -10,7 +10,6 @@ import ( "github.com/go-logr/logr" grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" - appsv1 "github.com/openshift/api/apps/v1" imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -61,10 +60,6 @@ func (r *BaseAPIManagerLogicReconciler) ReconcileImagestream(desired *imagev1.Im return r.ReconcileResource(&imagev1.ImageStream{}, desired, mutatefn) } -func (r *BaseAPIManagerLogicReconciler) ReconcileDeploymentConfig(desired *appsv1.DeploymentConfig, mutatefn reconcilers.MutateFn) error { - return r.ReconcileResource(&appsv1.DeploymentConfig{}, desired, mutatefn) -} - func (r *BaseAPIManagerLogicReconciler) ReconcileDeployment(desired *k8sappsv1.Deployment, mutatefn reconcilers.MutateFn) error { return r.ReconcileResource(&k8sappsv1.Deployment{}, desired, mutatefn) } diff --git a/pkg/3scale/amp/operator/memcached_options_provider.go b/pkg/3scale/amp/operator/memcached_options_provider.go index 2829f9928..ef8584832 100644 --- a/pkg/3scale/amp/operator/memcached_options_provider.go +++ b/pkg/3scale/amp/operator/memcached_options_provider.go @@ -49,7 +49,7 @@ func (m *MemcachedOptionsProvider) setResourceRequirementsOptions() { m.memcachedOptions.ResourceRequirements = v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if m.apimanager.Spec.System.MemcachedResources != nil { diff --git a/pkg/3scale/amp/operator/redis_reconciler_test.go b/pkg/3scale/amp/operator/redis_reconciler_test.go index 206b912ee..72b49faf2 100644 --- a/pkg/3scale/amp/operator/redis_reconciler_test.go +++ b/pkg/3scale/amp/operator/redis_reconciler_test.go @@ -23,7 +23,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -func TestRedisBackendDCReconcilerCreate(t *testing.T) { +func TestRedisBackendDeploymentReconcilerCreate(t *testing.T) { var ( appLabel = "someLabel" name = "example-apimanager" diff --git a/pkg/3scale/amp/operator/system_options_provider.go b/pkg/3scale/amp/operator/system_options_provider.go index c7ed93688..db8674d90 100644 --- a/pkg/3scale/amp/operator/system_options_provider.go +++ b/pkg/3scale/amp/operator/system_options_provider.go @@ -374,7 +374,7 @@ func (s *SystemOptionsProvider) setResourceRequirementsOptions() { s.options.SidekiqContainerResourceRequirements = &v1.ResourceRequirements{} } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if s.apimanager.Spec.System.AppSpec.MasterContainerResources != nil { diff --git a/pkg/3scale/amp/operator/system_searchd_options_provider.go b/pkg/3scale/amp/operator/system_searchd_options_provider.go index 235d47886..e13db4fea 100644 --- a/pkg/3scale/amp/operator/system_searchd_options_provider.go +++ b/pkg/3scale/amp/operator/system_searchd_options_provider.go @@ -49,7 +49,7 @@ func (s *SystemSearchdOptionsProvider) setResourceRequirementsOptions() { if *s.apimanager.Spec.ResourceRequirementsEnabled { s.options.ContainerResourceRequirements = component.DefaultSearchdContainerResourceRequirements() } - // DeploymentConfig-level ResourceRequirements CR fields have priority over + // Deployment-level ResourceRequirements CR fields have priority over // spec.resourceRequirementsEnabled, overwriting that setting when they are // defined if s.apimanager.Spec.System.SearchdSpec.Resources != nil { diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler_test.go b/pkg/3scale/amp/operator/system_searchd_reconciler_test.go index 06871933c..a9e97bf40 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler_test.go @@ -66,7 +66,7 @@ func TestSystemSearchdReconciler(t *testing.T) { }{ {"PVC", component.SystemSearchdPVCName, &v1.PersistentVolumeClaim{}}, {"Service", component.SystemSearchdServiceName, &v1.Service{}}, - {"DC", component.SystemSearchdDeploymentName, &k8sappsv1.Deployment{}}, + {"Deployment", component.SystemSearchdDeploymentName, &k8sappsv1.Deployment{}}, } for _, tc := range cases { diff --git a/pkg/handlers/apimanager_managed_routes_event_mapper.go b/pkg/handlers/apimanager_managed_routes_event_mapper.go index 305b996e3..c903118d9 100644 --- a/pkg/handlers/apimanager_managed_routes_event_mapper.go +++ b/pkg/handlers/apimanager_managed_routes_event_mapper.go @@ -46,7 +46,7 @@ func (h *APIManagerRoutesEventMapper) getAPIManagerOwnerReconcileRequest(object h.Logger.V(2).Info("Evaluating OwnerReference", "GroupVersion", refGV, "Kind", ref.Kind, "Name", ref.Name) // Compare the OwnerReference Group and Kind against the APIManager Kind - // or the name of the Zync Que DeploymentConfig. + // or the name of the Zync Que Deployment. // If the OwnerReference of the received object is an APIManager we // return a reconcile Request using the name from the OwnerReference and the diff --git a/pkg/helper/deploymentconfig.go b/pkg/helper/deploymentconfig.go deleted file mode 100644 index ae59579ff..000000000 --- a/pkg/helper/deploymentconfig.go +++ /dev/null @@ -1,40 +0,0 @@ -package helper - -import ( - "errors" - - appsv1 "github.com/openshift/api/apps/v1" - corev1 "k8s.io/api/core/v1" -) - -// IsDeploymentConfigAvailable returns true when the provided DeploymentConfig -// has the "Available" condition set to true -func IsDeploymentConfigAvailable(dc *appsv1.DeploymentConfig) bool { - dcConditions := dc.Status.Conditions - for _, dcCondition := range dcConditions { - if dcCondition.Type == appsv1.DeploymentAvailable && dcCondition.Status == corev1.ConditionTrue { - return true - } - } - return false -} - -func FindDeploymentTriggerOnImageChange(triggerPolicies []appsv1.DeploymentTriggerPolicy) (int, error) { - result := -1 - for i := range triggerPolicies { - if triggerPolicies[i].Type == appsv1.DeploymentTriggerOnImageChange { - result = i - break - } - } - - if result == -1 { - return -1, errors.New("no imageChangeParams deployment trigger policy found") - } - - return result, nil -} - -func IsDeploymentConfigDeleting(dc *appsv1.DeploymentConfig) bool { - return dc.GetDeletionTimestamp() != nil -} diff --git a/pkg/helper/envvarutils.go b/pkg/helper/envvarutils.go index d4131de67..e5a086c95 100644 --- a/pkg/helper/envvarutils.go +++ b/pkg/helper/envvarutils.go @@ -123,10 +123,10 @@ func RemoveDuplicateEnvVars(envVars []v1.EnvVar) []v1.EnvVar { return result } -// EnvVarReconciler implements basic env var reconcilliation. +// EnvVarReconciler implements basic env var reconciliation. // Added when in desired and not in existing // Updated when in desired and in existing but not equal -// Removed when not in desired and exists in existing DC +// Removed when not in desired and exists in existing Deployment func EnvVarReconciler(desired []v1.EnvVar, existing *[]v1.EnvVar, envVar string) bool { update := false diff --git a/pkg/reconcilers/deployment.go b/pkg/reconcilers/deployment.go index e1853670b..a4e25f36a 100644 --- a/pkg/reconcilers/deployment.go +++ b/pkg/reconcilers/deployment.go @@ -15,10 +15,9 @@ import ( ) const ( - DeploymentKind = "Deployment" - DeploymentAPIVersion = "apps/v1" - DeploymentLabelSelector = "deployment" - DeploymentImageTriggerAnnotation = "image.openshift.io/triggers" + DeploymentKind = "Deployment" + DeploymentAPIVersion = "apps/v1" + DeploymentLabelSelector = "deployment" ) type ContainerImage struct { @@ -156,10 +155,10 @@ func DeploymentContainerResourcesMutator(desired, existing *k8sappsv1.Deployment } // DeploymentEnvVarReconciler implements basic env var reconciliation deployments. -// Existing and desired DC must have same number of containers +// Existing and desired Deployment must have same number of containers // Added when in desired and not in existing // Updated when in desired and in existing but not equal -// Removed when not in desired and exists in existing DC +// Removed when not in desired and exists in existing Deployment func DeploymentEnvVarReconciler(desired, existing *k8sappsv1.Deployment, envVar string) bool { updated := false diff --git a/pkg/reconcilers/deployment_test.go b/pkg/reconcilers/deployment_test.go index cf9bfa3c3..443ab7a70 100644 --- a/pkg/reconcilers/deployment_test.go +++ b/pkg/reconcilers/deployment_test.go @@ -874,11 +874,11 @@ func TestDeploymentArgsMutator(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got, err := DeploymentArgsMutator(tt.args.desired, tt.args.existing) if (err != nil) != tt.wantErr { - t.Errorf("DeploymentConfigArgsMutator() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("DeploymentArgsMutator() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("DeploymentConfigArgsMutator() = %v, want %v", got, tt.want) + t.Errorf("DeploymentArgsMutator() = %v, want %v", got, tt.want) } }) } @@ -1083,11 +1083,11 @@ func TestDeploymentProbesMutator(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got, err := DeploymentProbesMutator(tt.args.desired, tt.args.existing) if (err != nil) != tt.wantErr { - t.Errorf("DeploymentConfigProbesMutator() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("DeploymentProbesMutator() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("DeploymentConfigProbesMutator() = %v, want %v", got, tt.want) + t.Errorf("DeploymentProbesMutator() = %v, want %v", got, tt.want) } }) } diff --git a/pkg/reconcilers/deploymentconfig.go b/pkg/reconcilers/deploymentconfig.go deleted file mode 100644 index 26caca55e..000000000 --- a/pkg/reconcilers/deploymentconfig.go +++ /dev/null @@ -1,324 +0,0 @@ -package reconcilers - -import ( - "fmt" - "reflect" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - appsv1 "github.com/openshift/api/apps/v1" - "k8s.io/apimachinery/pkg/api/resource" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/3scale/3scale-operator/pkg/common" - "github.com/3scale/3scale-operator/pkg/helper" -) - -// DCMutateFn is a function which mutates the existing DeploymentConfig into it's desired state. -type DCMutateFn func(desired, existing *appsv1.DeploymentConfig) (bool, error) - -func DeploymentConfigMutator(opts ...DCMutateFn) MutateFn { - return func(existingObj, desiredObj common.KubernetesObject) (bool, error) { - existing, ok := existingObj.(*appsv1.DeploymentConfig) - if !ok { - return false, fmt.Errorf("%T is not a *appsv1.DeploymentConfig", existingObj) - } - desired, ok := desiredObj.(*appsv1.DeploymentConfig) - if !ok { - return false, fmt.Errorf("%T is not a *appsv1.DeploymentConfig", desiredObj) - } - - update := false - - // Loop through each option - for _, opt := range opts { - tmpUpdate, err := opt(desired, existing) - if err != nil { - return false, err - } - update = update || tmpUpdate - } - - return update, nil - } -} - -// GenericBackendMutators returns the generic mutators for backend -func GenericBackendMutators() []DCMutateFn { - return []DCMutateFn{ - DeploymentConfigImageChangeTriggerMutator, - DeploymentConfigContainerResourcesMutator, - DeploymentConfigAffinityMutator, - DeploymentConfigTolerationsMutator, - DeploymentConfigPodTemplateLabelsMutator, - DeploymentConfigPriorityClassMutator, - DeploymentConfigTopologySpreadConstraintsMutator, - DeploymentConfigPodTemplateAnnotationsMutator, - DeploymentConfigArgsMutator, - DeploymentConfigProbesMutator, - } -} - -func DeploymentConfigReplicasMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - update := false - - if desired.Spec.Replicas != existing.Spec.Replicas { - existing.Spec.Replicas = desired.Spec.Replicas - update = true - } - - return update, nil -} - -func DeploymentConfigAffinityMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - if !reflect.DeepEqual(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity) { - diff := cmp.Diff(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity) - log.Info(fmt.Sprintf("%s spec.template.spec.Affinity has changed: %s", common.ObjectInfo(desired), diff)) - existing.Spec.Template.Spec.Affinity = desired.Spec.Template.Spec.Affinity - updated = true - } - - return updated, nil -} - -func DeploymentConfigTolerationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - if !reflect.DeepEqual(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations) { - diff := cmp.Diff(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations) - log.Info(fmt.Sprintf("%s spec.template.spec.Tolerations has changed: %s", common.ObjectInfo(desired), diff)) - existing.Spec.Template.Spec.Tolerations = desired.Spec.Template.Spec.Tolerations - updated = true - } - - return updated, nil -} - -func DeploymentConfigContainerResourcesMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - desiredName := common.ObjectInfo(desired) - update := false - - if len(desired.Spec.Template.Spec.Containers) != 1 { - return false, fmt.Errorf("%s desired spec.template.spec.containers length changed to '%d', should be 1", desiredName, len(desired.Spec.Template.Spec.Containers)) - } - - if len(existing.Spec.Template.Spec.Containers) != 1 { - log.Info(fmt.Sprintf("%s spec.template.spec.containers length changed to '%d', recreating dc", desiredName, len(existing.Spec.Template.Spec.Containers))) - existing.Spec.Template.Spec.Containers = desired.Spec.Template.Spec.Containers - update = true - } - - if !helper.CmpResources(&existing.Spec.Template.Spec.Containers[0].Resources, &desired.Spec.Template.Spec.Containers[0].Resources) { - diff := cmp.Diff(existing.Spec.Template.Spec.Containers[0].Resources, desired.Spec.Template.Spec.Containers[0].Resources, cmpopts.IgnoreUnexported(resource.Quantity{})) - log.Info(fmt.Sprintf("%s spec.template.spec.containers[0].resources have changed: %s", desiredName, diff)) - existing.Spec.Template.Spec.Containers[0].Resources = desired.Spec.Template.Spec.Containers[0].Resources - update = true - } - - return update, nil -} - -// DeploymentConfigEnvVarReconciler implements basic env var reconcilliation deployment configs. -// Existing and desired DC must have same number of containers -// Added when in desired and not in existing -// Updated when in desired and in existing but not equal -// Removed when not in desired and exists in existing DC -func DeploymentConfigEnvVarReconciler(desired, existing *appsv1.DeploymentConfig, envVar string) bool { - updated := false - - if len(desired.Spec.Template.Spec.Containers) != len(existing.Spec.Template.Spec.Containers) { - log.Info("[WARNING] not reconciling deployment config", - "name", client.ObjectKeyFromObject(desired), - "reason", "existing and desired do not have same number of containers") - return false - } - - if len(desired.Spec.Template.Spec.InitContainers) != len(existing.Spec.Template.Spec.InitContainers) { - log.Info("[WARNING] not reconciling deployment config", - "name", client.ObjectKeyFromObject(desired), - "reason", "existing and desired do not have same number of init containers") - return false - } - - // Init Containers - for idx := range existing.Spec.Template.Spec.InitContainers { - tmpChanged := helper.EnvVarReconciler( - desired.Spec.Template.Spec.InitContainers[idx].Env, - &existing.Spec.Template.Spec.InitContainers[idx].Env, - envVar) - updated = updated || tmpChanged - } - - // Containers - for idx := range existing.Spec.Template.Spec.Containers { - tmpChanged := helper.EnvVarReconciler( - desired.Spec.Template.Spec.Containers[idx].Env, - &existing.Spec.Template.Spec.Containers[idx].Env, - envVar) - updated = updated || tmpChanged - } - - // Pre Hook pod - if existing.Spec.Strategy.RollingParams != nil && - existing.Spec.Strategy.RollingParams.Pre != nil && - existing.Spec.Strategy.RollingParams.Pre.ExecNewPod != nil && - desired.Spec.Strategy.RollingParams != nil && - desired.Spec.Strategy.RollingParams.Pre != nil && - desired.Spec.Strategy.RollingParams.Pre.ExecNewPod != nil { - // reconcile Pre Hook - tmpChanged := helper.EnvVarReconciler( - desired.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env, - &existing.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env, - envVar) - updated = updated || tmpChanged - } - - // Post Hook pod - if existing.Spec.Strategy.RollingParams != nil && - existing.Spec.Strategy.RollingParams.Post != nil && - existing.Spec.Strategy.RollingParams.Post.ExecNewPod != nil && - desired.Spec.Strategy.RollingParams != nil && - desired.Spec.Strategy.RollingParams.Post != nil && - desired.Spec.Strategy.RollingParams.Post.ExecNewPod != nil { - // reconcile Pre Hook - tmpChanged := helper.EnvVarReconciler( - desired.Spec.Strategy.RollingParams.Post.ExecNewPod.Env, - &existing.Spec.Strategy.RollingParams.Post.ExecNewPod.Env, - envVar) - updated = updated || tmpChanged - } - - return updated -} - -// DeploymentConfigImageChangeTriggerMutator ensures image change triggers are reconciled -func DeploymentConfigImageChangeTriggerMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - desiredDeploymentTriggerImageChangePos, err := helper.FindDeploymentTriggerOnImageChange(desired.Spec.Triggers) - if err != nil { - return false, fmt.Errorf("unexpected: '%s' in DeploymentConfig '%s'", err, desired.Name) - - } - existingDeploymentTriggerImageChangePos, err := helper.FindDeploymentTriggerOnImageChange(existing.Spec.Triggers) - if err != nil { - return false, fmt.Errorf("unexpected: '%s' in DeploymentConfig '%s'", err, existing.Name) - } - - desiredDeploymentTriggerImageChangeParams := desired.Spec.Triggers[desiredDeploymentTriggerImageChangePos].ImageChangeParams - existingDeploymentTriggerImageChangeParams := existing.Spec.Triggers[existingDeploymentTriggerImageChangePos].ImageChangeParams - - if !reflect.DeepEqual(existingDeploymentTriggerImageChangeParams.From.Name, desiredDeploymentTriggerImageChangeParams.From.Name) { - existingDeploymentTriggerImageChangeParams.From.Name = desiredDeploymentTriggerImageChangeParams.From.Name - return true, nil - } - - return false, nil -} - -// DeploymentConfigPodTemplateLabelsMutator ensures pod template labels are reconciled -func DeploymentConfigPodTemplateLabelsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - helper.MergeMapStringString(&updated, &existing.Spec.Template.Labels, desired.Spec.Template.Labels) - - return updated, nil -} - -// DeploymentConfigRemoveDuplicateEnvVarMutator ensures pod env vars are not duplicated -func DeploymentConfigRemoveDuplicateEnvVarMutator(_, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - for idx := range existing.Spec.Template.Spec.Containers { - prunedEnvs := helper.RemoveDuplicateEnvVars(existing.Spec.Template.Spec.Containers[idx].Env) - if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[idx].Env, prunedEnvs) { - existing.Spec.Template.Spec.Containers[idx].Env = prunedEnvs - updated = true - } - } - - return updated, nil -} - -// DeploymentConfigPriorityClassMutator ensures priorityclass is reconciled -func DeploymentConfigPriorityClassMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - if existing.Spec.Template.Spec.PriorityClassName != desired.Spec.Template.Spec.PriorityClassName { - existing.Spec.Template.Spec.PriorityClassName = desired.Spec.Template.Spec.PriorityClassName - updated = true - } - - return updated, nil -} - -// DeploymentConfigStrategyMutator ensures desired strategy -func DeploymentConfigStrategyMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - if !reflect.DeepEqual(existing.Spec.Strategy, desired.Spec.Strategy) { - existing.Spec.Strategy = desired.Spec.Strategy - updated = true - } - - return updated, nil -} - -// DeploymentConfigTopologySpreadConstraintsMutator ensures TopologySpreadConstraints is reconciled -func DeploymentConfigTopologySpreadConstraintsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - if !reflect.DeepEqual(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints) { - diff := cmp.Diff(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints) - log.Info(fmt.Sprintf("%s spec.template.spec.TopologySpreadConstraints has changed: %s", common.ObjectInfo(desired), diff)) - existing.Spec.Template.Spec.TopologySpreadConstraints = desired.Spec.Template.Spec.TopologySpreadConstraints - updated = true - } - - return updated, nil -} - -// DeploymentConfigPodTemplateAnnotationsMutator ensures Pod Template Annotations is reconciled -func DeploymentConfigPodTemplateAnnotationsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - helper.MergeMapStringString(&updated, &existing.Spec.Template.Annotations, desired.Spec.Template.Annotations) - - return updated, nil -} - -// DeploymentConfigArgsMutator ensures Args are reconciled -func DeploymentConfigArgsMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - for i, desiredContainer := range desired.Spec.Template.Spec.Containers { - existingContainer := &existing.Spec.Template.Spec.Containers[i] - - if !reflect.DeepEqual(existingContainer.Args, desiredContainer.Args) { - existingContainer.Args = desiredContainer.Args - updated = true - } - } - - return updated, nil -} - -// DeploymentConfigProbesMutator ensures probes are reconciled -func DeploymentConfigProbesMutator(desired, existing *appsv1.DeploymentConfig) (bool, error) { - updated := false - - for i, desiredContainer := range desired.Spec.Template.Spec.Containers { - existingContainer := &existing.Spec.Template.Spec.Containers[i] - - if !reflect.DeepEqual(existingContainer.LivenessProbe, desiredContainer.LivenessProbe) { - existingContainer.LivenessProbe = desiredContainer.LivenessProbe - updated = true - } - - if !reflect.DeepEqual(existingContainer.ReadinessProbe, desiredContainer.ReadinessProbe) { - existingContainer.ReadinessProbe = desiredContainer.ReadinessProbe - updated = true - } - } - - return updated, nil -} diff --git a/pkg/reconcilers/deploymentconfig_test.go b/pkg/reconcilers/deploymentconfig_test.go deleted file mode 100644 index 50428b7cf..000000000 --- a/pkg/reconcilers/deploymentconfig_test.go +++ /dev/null @@ -1,1276 +0,0 @@ -package reconcilers - -import ( - "reflect" - "testing" - - "github.com/3scale/3scale-operator/pkg/helper" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - appsv1 "github.com/openshift/api/apps/v1" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -func TestDeploymentConfigReplicasMutator(t *testing.T) { - dcFactory := func() *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Replicas: 3, - }, - } - } - - cases := []struct { - testName string - desired func() *appsv1.DeploymentConfig - expectedResult bool - }{ - {"NothingToReconcile", func() *appsv1.DeploymentConfig { return dcFactory() }, false}, - {"ReplicasReconcile", - func() *appsv1.DeploymentConfig { - desired := dcFactory() - desired.Spec.Replicas = desired.Spec.Replicas + 1000 - return desired - }, true, - }, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory() - update, err := DeploymentConfigReplicasMutator(tc.desired(), existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - if existing.Spec.Replicas != tc.desired().Spec.Replicas { - subT.Fatalf("replica reconciliation failed, existing: %d, desired: %d", existing.Spec.Replicas, tc.desired().Spec.Replicas) - } - - }) - } -} - -func TestDeploymentConfigContainerResourcesMutator(t *testing.T) { - emptyResourceRequirements := corev1.ResourceRequirements{ - Limits: corev1.ResourceList{}, - Requests: corev1.ResourceList{}, - } - notEmptyResources := corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("110Mi"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("200m"), - corev1.ResourceMemory: resource.MustParse("220Mi"), - }, - } - dcFactory := func(resources corev1.ResourceRequirements) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{ - Name: "container1", - Resources: resources, - }, - }, - }, - }, - }, - } - } - - cases := []struct { - testName string - existingResources corev1.ResourceRequirements - desiredResources corev1.ResourceRequirements - expectedResult bool - }{ - {"NothingToReconcile", emptyResourceRequirements, emptyResourceRequirements, false}, - {"NothingToReconcileWithResources", notEmptyResources, notEmptyResources, false}, - {"AddResources", emptyResourceRequirements, notEmptyResources, true}, - {"RemoveResources", notEmptyResources, emptyResourceRequirements, true}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingResources) - desired := dcFactory(tc.desiredResources) - update, err := DeploymentConfigContainerResourcesMutator(desired, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - if !helper.CmpResources(&existing.Spec.Template.Spec.Containers[0].Resources, &desired.Spec.Template.Spec.Containers[0].Resources) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Containers[0].Resources, desired.Spec.Template.Spec.Containers[0].Resources, cmpopts.IgnoreUnexported(resource.Quantity{}))) - } - }) - } -} - -func TestDeploymentConfigAffinityMutator(t *testing.T) { - testAffinity1 := &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - corev1.NodeSelectorTerm{ - MatchFields: []corev1.NodeSelectorRequirement{ - v1.NodeSelectorRequirement{ - Key: "key1", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"val1"}, - }, - }, - }, - }, - }, - }, - } - testAffinity2 := &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - corev1.NodeSelectorTerm{ - MatchFields: []corev1.NodeSelectorRequirement{ - v1.NodeSelectorRequirement{ - Key: "key2", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"val2"}, - }, - }, - }, - }, - }, - }, - } - dcFactory := func(affinity *corev1.Affinity) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Affinity: affinity, - }, - }, - }, - } - } - - cases := []struct { - testName string - existingAffinity *corev1.Affinity - desiredAffinity *corev1.Affinity - expectedResult bool - }{ - {"NothingToReconcile", nil, nil, false}, - {"EqualAffinities", testAffinity1, testAffinity1, false}, - {"DifferentAffinities", testAffinity1, testAffinity2, true}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingAffinity) - desired := dcFactory(tc.desiredAffinity) - update, err := DeploymentConfigAffinityMutator(desired, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - if !reflect.DeepEqual(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Affinity, desired.Spec.Template.Spec.Affinity)) - } - }) - } -} - -func TestDeploymentConfigTolerationsMutator(t *testing.T) { - testTolerations1 := []corev1.Toleration{ - corev1.Toleration{ - Key: "key1", - Effect: corev1.TaintEffectNoExecute, - Operator: corev1.TolerationOpEqual, - Value: "val1", - }, - corev1.Toleration{ - Key: "key2", - Effect: corev1.TaintEffectNoExecute, - Operator: corev1.TolerationOpEqual, - Value: "val2", - }, - } - testTolerations2 := []corev1.Toleration{ - corev1.Toleration{ - Key: "key3", - Effect: corev1.TaintEffectNoExecute, - Operator: corev1.TolerationOpEqual, - Value: "val3", - }, - corev1.Toleration{ - Key: "key4", - Effect: corev1.TaintEffectNoExecute, - Operator: corev1.TolerationOpEqual, - Value: "val4", - }, - } - dcFactory := func(toleration []corev1.Toleration) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Tolerations: toleration, - }, - }, - }, - } - } - - cases := []struct { - testName string - existingTolerations []corev1.Toleration - desiredTolerations []corev1.Toleration - expectedResult bool - }{ - {"NothingToReconcile", nil, nil, false}, - {"EqualAffinities", testTolerations1, testTolerations1, false}, - {"DifferentAffinities", testTolerations1, testTolerations2, true}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingTolerations) - desired := dcFactory(tc.desiredTolerations) - update, err := DeploymentConfigTolerationsMutator(desired, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - if !reflect.DeepEqual(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Tolerations, desired.Spec.Template.Spec.Tolerations)) - } - }) - } - -} - -func TestDeploymentConfigEnvVarReconciler(t *testing.T) { - t.Run("DifferentNumberOfContainers", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{ - Name: "container1", - }, - }, - }, - }, - }, - } - existing := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{ - Name: "container1", - }, - corev1.Container{ - Name: "container2", - }, - }, - }, - }, - }, - } - - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") - if update { - subT.Fatal("expected not to be updated") - } - }) - - t.Run("DifferentNumberOfInitContainers", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{ - Name: "container1", - }, - }, - InitContainers: []corev1.Container{ - corev1.Container{ - Name: "initcontainer1", - }, - }, - }, - }, - }, - } - existing := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{ - Name: "container1", - }, - }, - InitContainers: []corev1.Container{ - corev1.Container{ - Name: "initcontainer1", - }, - corev1.Container{ - Name: "initcontainer2", - }, - }, - }, - }, - }, - } - - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") - if update { - subT.Fatal("expected not to be updated") - } - }) - - t.Run("ContainersEnvVarReconciled", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{ - Name: "container1", - Env: []corev1.EnvVar{ - {Name: "A", Value: "valueA"}, - }, - }, - corev1.Container{ - Name: "container2", - Env: []corev1.EnvVar{}, - }, - }, - }, - }, - }, - } - existing := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{ - Name: "container1", - Env: []corev1.EnvVar{}, - }, - corev1.Container{ - Name: "container2", - Env: []corev1.EnvVar{ - {Name: "A", Value: "valueA"}, - }, - }, - }, - }, - }, - }, - } - - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") - if !update { - subT.Fatal("expected not be updated") - } - - for i := range []int{0, 1} { - if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[i].Env, desired.Spec.Template.Spec.Containers[i].Env) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Containers[i].Env, desired.Spec.Template.Spec.Containers[i].Env)) - } - } - - }) - - t.Run("InitContainersEnvVarReconciled", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - InitContainers: []corev1.Container{ - corev1.Container{ - Name: "intcontainer1", - Env: []corev1.EnvVar{ - {Name: "A", Value: "valueA"}, - }, - }, - corev1.Container{ - Name: "intcontainer2", - Env: []corev1.EnvVar{}, - }, - }, - }, - }, - }, - } - existing := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - InitContainers: []corev1.Container{ - corev1.Container{ - Name: "initcontainer1", - Env: []corev1.EnvVar{}, - }, - corev1.Container{ - Name: "initcontainer2", - Env: []corev1.EnvVar{ - {Name: "A", Value: "valueA"}, - }, - }, - }, - }, - }, - }, - } - - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") - if !update { - subT.Fatal("expected not be updated") - } - - for i := range []int{0, 1} { - if !reflect.DeepEqual(existing.Spec.Template.Spec.InitContainers[i].Env, desired.Spec.Template.Spec.InitContainers[i].Env) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.InitContainers[i].Env, desired.Spec.Template.Spec.InitContainers[i].Env)) - } - } - }) - - t.Run("PreHookEnvVarReconciled", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{}, - }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - Pre: &appsv1.LifecycleHook{ - ExecNewPod: &appsv1.ExecNewPodHook{ - Env: []corev1.EnvVar{ - {Name: "A", Value: "valueA"}, - }, - }, - }, - }, - }, - }, - } - existing := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{}, - }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - Pre: &appsv1.LifecycleHook{ - ExecNewPod: &appsv1.ExecNewPodHook{ - Env: []corev1.EnvVar{}, - }, - }, - }, - }, - }, - } - - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") - if !update { - subT.Fatal("expected not be updated") - } - - if !reflect.DeepEqual(existing.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env, desired.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env) { - subT.Fatal(cmp.Diff(existing.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env, desired.Spec.Strategy.RollingParams.Pre.ExecNewPod.Env)) - } - }) - - t.Run("PostHookEnvVarReconciled", func(subT *testing.T) { - desired := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{}, - }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - Post: &appsv1.LifecycleHook{ - ExecNewPod: &appsv1.ExecNewPodHook{ - Env: []corev1.EnvVar{ - {Name: "A", Value: "valueA"}, - }, - }, - }, - }, - }, - }, - } - existing := &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: "myDC", Namespace: "myNS"}, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{}, - }, - Strategy: appsv1.DeploymentStrategy{ - RollingParams: &appsv1.RollingDeploymentStrategyParams{ - Post: &appsv1.LifecycleHook{ - ExecNewPod: &appsv1.ExecNewPodHook{ - Env: []corev1.EnvVar{}, - }, - }, - }, - }, - }, - } - - update := DeploymentConfigEnvVarReconciler(desired, existing, "A") - if !update { - subT.Fatal("expected not be updated") - } - - if !reflect.DeepEqual(existing.Spec.Strategy.RollingParams.Post.ExecNewPod.Env, desired.Spec.Strategy.RollingParams.Post.ExecNewPod.Env) { - subT.Fatal(cmp.Diff(existing.Spec.Strategy.RollingParams.Post.ExecNewPod.Env, desired.Spec.Strategy.RollingParams.Post.ExecNewPod.Env)) - } - }) -} - -func TestDeploymentConfigImageChangeTriggerMutator(t *testing.T) { - dcFactory := func(triggers []appsv1.DeploymentTriggerPolicy) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Triggers: triggers, - }, - } - } - - sliceCopy := func(a []appsv1.DeploymentTriggerPolicy) []appsv1.DeploymentTriggerPolicy { - return append(a[:0:0], a...) - } - - triggersA := []appsv1.DeploymentTriggerPolicy{ - { - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - From: corev1.ObjectReference{ - Name: "imagestreamA", - }, - }, - }, - } - - triggersB := []appsv1.DeploymentTriggerPolicy{ - { - Type: appsv1.DeploymentTriggerOnImageChange, - ImageChangeParams: &appsv1.DeploymentTriggerImageChangeParams{ - From: corev1.ObjectReference{ - Name: "imagestreamB", - }, - }, - }, - } - - cases := []struct { - testName string - existingTriggers []appsv1.DeploymentTriggerPolicy - desiredTriggers []appsv1.DeploymentTriggerPolicy - expectedResult bool - }{ - {"NothingToReconcile", sliceCopy(triggersA), sliceCopy(triggersA), false}, - {"DifferentName", sliceCopy(triggersA), sliceCopy(triggersB), true}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingTriggers) - desired := dcFactory(tc.desiredTriggers) - update, err := DeploymentConfigImageChangeTriggerMutator(desired, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - // It should be tested changes in triggers on image change only, but good enough for now - if !reflect.DeepEqual(existing.Spec.Triggers, desired.Spec.Triggers) { - subT.Fatal(cmp.Diff(existing.Spec.Triggers, desired.Spec.Triggers)) - } - }) - } -} - -func TestDeploymentConfigPodTemplateLabelsMutator(t *testing.T) { - dcFactory := func(labels map[string]string) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - }, - }, - } - } - - mapCopy := func(originalMap map[string]string) map[string]string { - // Create the target map - targetMap := make(map[string]string) - - // Copy from the original map to the target map - for key, value := range originalMap { - targetMap[key] = value - } - - return targetMap - } - - labelsA := map[string]string{"a": "1", "a2": "2"} - labelsB := map[string]string{"a": "other", "b": "1"} - - cases := []struct { - testName string - existingLabels map[string]string - desiredLabels map[string]string - expectedResult bool - expectedNewLabels map[string]string - }{ - {"NothingToReconcile", mapCopy(labelsA), mapCopy(labelsA), false, mapCopy(labelsA)}, - {"LabelsReconciled", mapCopy(labelsB), mapCopy(labelsA), true, map[string]string{ - "a": "1", "a2": "2", "b": "1", - }}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingLabels) - desired := dcFactory(tc.desiredLabels) - update, err := DeploymentConfigPodTemplateLabelsMutator(desired, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - if !reflect.DeepEqual(existing.Spec.Template.Labels, tc.expectedNewLabels) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Labels, tc.expectedNewLabels)) - } - }) - } -} - -func TestDeploymentConfigRemoveDuplicateEnvVarMutator(t *testing.T) { - dcFactory := func(envs []corev1.EnvVar) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container1", - Env: envs, - }, - { - Name: "container2", - Env: envs, - }, - }, - }, - }, - }, - } - } - - envsA := []corev1.EnvVar{{Name: "a", Value: "1"}, {Name: "b", Value: "2"}} - envsB := []corev1.EnvVar{{Name: "a", Value: "1"}, {Name: "a", Value: "1"}, {Name: "b", Value: "2"}} - - cases := []struct { - testName string - existingEnvs []corev1.EnvVar - expectedResult bool - expectedNewEnvs []corev1.EnvVar - }{ - {"NothingToReconcile", envsA, false, envsA}, - {"EnvsReconciled", envsB, true, envsA}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingEnvs) - update, err := DeploymentConfigRemoveDuplicateEnvVarMutator(nil, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - for idx := range existing.Spec.Template.Spec.Containers { - if !reflect.DeepEqual(existing.Spec.Template.Spec.Containers[idx].Env, tc.expectedNewEnvs) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.Containers[idx].Env, tc.expectedNewEnvs)) - } - } - }) - } -} - -func TestDeploymentConfigTopologySpreadConstraintsMutator(t *testing.T) { - testTopologySpreadConstraint1 := []corev1.TopologySpreadConstraint{ - corev1.TopologySpreadConstraint{ - TopologyKey: "topologyKey1", - WhenUnsatisfiable: "DoNotSchedule", - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "3scale-api-management"}, - }, - }, - } - testTopologySpreadConstraint2 := []corev1.TopologySpreadConstraint{ - corev1.TopologySpreadConstraint{ - TopologyKey: "topologyKey2", - WhenUnsatisfiable: "ScheduleAnyway", - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "3scale-api-management", "threescale_component": "system"}, - }, - }, - } - - dcFactory := func(topologySpreadConstraint []corev1.TopologySpreadConstraint) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - TopologySpreadConstraints: topologySpreadConstraint, - }, - }, - }, - } - } - - cases := []struct { - testName string - existingTopologySpreadConstraints []corev1.TopologySpreadConstraint - desiredTopologySpreadConstraints []corev1.TopologySpreadConstraint - expectedResult bool - }{ - {"NothingToReconcile", nil, nil, false}, - {"EqualTopologies", testTopologySpreadConstraint1, testTopologySpreadConstraint1, false}, - {"DifferentTopologie", testTopologySpreadConstraint1, testTopologySpreadConstraint2, true}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingTopologySpreadConstraints) - desired := dcFactory(tc.desiredTopologySpreadConstraints) - update, err := DeploymentConfigTopologySpreadConstraintsMutator(desired, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - if !reflect.DeepEqual(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Spec.TopologySpreadConstraints, desired.Spec.Template.Spec.TopologySpreadConstraints)) - } - }) - } - -} - -func TestDeploymentConfigPodTemplateAnnotationsMutator(t *testing.T) { - dcFactory := func(annotations map[string]string) *appsv1.DeploymentConfig { - return &appsv1.DeploymentConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "DeploymentConfig", - APIVersion: "apps.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "myDC", - Namespace: "myNS", - }, - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: annotations, - }, - }, - }, - } - } - - mapCopy := func(originalMap map[string]string) map[string]string { - // Create the target map - targetMap := make(map[string]string) - - // Copy from the original map to the target map - for key, value := range originalMap { - targetMap[key] = value - } - - return targetMap - } - - annotationsA := map[string]string{"a": "1", "a2": "2"} - annotationsB := map[string]string{"a": "other", "b": "1"} - - cases := []struct { - testName string - existingAnnotations map[string]string - desiredAnnotations map[string]string - expectedResult bool - expectedNewAnnotations map[string]string - }{ - {"NothingToReconcile", mapCopy(annotationsA), mapCopy(annotationsA), false, mapCopy(annotationsA)}, - {"AnnotationsReconciled", mapCopy(annotationsB), mapCopy(annotationsA), true, map[string]string{ - "a": "1", "a2": "2", "b": "1", - }}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - existing := dcFactory(tc.existingAnnotations) - desired := dcFactory(tc.desiredAnnotations) - update, err := DeploymentConfigPodTemplateAnnotationsMutator(desired, existing) - if err != nil { - subT.Fatal(err) - } - if update != tc.expectedResult { - subT.Fatalf("result failed, expected: %t, got: %t", tc.expectedResult, update) - } - if !reflect.DeepEqual(existing.Spec.Template.Annotations, tc.expectedNewAnnotations) { - subT.Fatal(cmp.Diff(existing.Spec.Template.Annotations, tc.expectedNewAnnotations)) - } - }) - } -} - -func TestDeploymentConfigArgsMutator(t *testing.T) { - type args struct { - desired *appsv1.DeploymentConfig - existing *appsv1.DeploymentConfig - } - tests := []struct { - name string - args args - want bool - wantErr bool - }{ - { - name: "No Args Update Required", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - {Args: []string{"testArg"}}, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - {Args: []string{"testArg"}}, - }, - }, - }, - }, - }, - }, - want: false, - wantErr: false, - }, - { - name: "Args Update Required", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - {Args: []string{"testArg1", "testArg2"}}, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - {Args: []string{"testArg1"}}, - }, - }, - }, - }, - }, - }, - want: true, - wantErr: false, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := DeploymentConfigArgsMutator(tt.args.desired, tt.args.existing) - if (err != nil) != tt.wantErr { - t.Errorf("DeploymentConfigArgsMutator() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("DeploymentConfigArgsMutator() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestDeploymentConfigProbesMutator(t *testing.T) { - type args struct { - desired *appsv1.DeploymentConfig - existing *appsv1.DeploymentConfig - } - tests := []struct { - name string - args args - want bool - wantErr bool - }{ - { - name: "Liveness Probe Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.FromInt(9306), - }, - }, - InitialDelaySeconds: 60, - PeriodSeconds: 10, - }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: nil, - }, - }, - }, - }, - }, - }, - }, - want: true, - wantErr: false, - }, - { - name: "Liveness Probe Not Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.FromInt(9306), - }, - }, - InitialDelaySeconds: 60, - PeriodSeconds: 10, - }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - LivenessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.FromInt(9306), - }, - }, - InitialDelaySeconds: 60, - PeriodSeconds: 10, - }, - }, - }, - }, - }, - }, - }, - }, - want: false, - wantErr: false, - }, - { - name: "Readiness Probe Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status", - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 3000}}, - }, - InitialDelaySeconds: 30, - TimeoutSeconds: 5, - }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: nil, - }, - }, - }, - }, - }, - }, - }, - want: true, - wantErr: false, - }, - { - name: "Readiness Probe Not Updated", - args: args{ - desired: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status", - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 3000}}, - }, - InitialDelaySeconds: 30, - TimeoutSeconds: 5, - }, - }, - }, - }, - }, - }, - }, - existing: &appsv1.DeploymentConfig{ - Spec: appsv1.DeploymentConfigSpec{ - Template: &v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status", - Port: intstr.IntOrString{ - Type: intstr.Type(intstr.Int), - IntVal: 3000}}, - }, - InitialDelaySeconds: 30, - TimeoutSeconds: 5, - }, - }, - }, - }, - }, - }, - }, - }, - want: false, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := DeploymentConfigProbesMutator(tt.args.desired, tt.args.existing) - if (err != nil) != tt.wantErr { - t.Errorf("DeploymentConfigProbesMutator() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("DeploymentConfigProbesMutator() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/reconcilers/secret.go b/pkg/reconcilers/secret.go index fb17c8990..d68dba497 100644 --- a/pkg/reconcilers/secret.go +++ b/pkg/reconcilers/secret.go @@ -9,7 +9,7 @@ import ( ) // DefaultsOnlySecretMutator is useful for secrets pre-created by the user and when not all the fields are created. -// Fields referenced from deployment configs must exist, +// Fields referenced from deployments must exist, // so defaults only reconciliation makes sure they exist with default values when user does doe pre-create them func DefaultsOnlySecretMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { existing, ok := existingObj.(*v1.Secret) diff --git a/pkg/restore/apimanager_restore.go b/pkg/restore/apimanager_restore.go index 68e7849ba..603e78246 100644 --- a/pkg/restore/apimanager_restore.go +++ b/pkg/restore/apimanager_restore.go @@ -395,13 +395,13 @@ func (b *APIManagerRestore) restoreSecretsAndConfigMapsContainerArgs() string { func (b *APIManagerRestore) zyncResyncDomainsContainerArgs() string { return ` - dcname="system-sidekiq" - dcpods=$(oc get pods --ignore-not-found=true -l deploymentconfig=${dcname} --no-headers=true -o custom-columns=:metadata.name) - if [ -z "${dcpods}" ]; then - echo "No pods found for Deployment ${dcname}" + dname="system-sidekiq" + dpods=$(oc get pods --ignore-not-found=true -l deployment=${dname} --no-headers=true -o custom-columns=:metadata.name) + if [ -z "${dpods}" ]; then + echo "No pods found for Deployment ${dname}" exit 1 fi - podname=$(echo -n $dcpods | awk '{print $1}') + podname=$(echo -n $dpods | awk '{print $1}') oc exec ${podname} -- bash -c "bundle exec rake zync:resync:domains" ` } From 117108ccb6625ed135292e2a54defd11ac50a7bc Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Thu, 25 Jan 2024 16:37:44 -0500 Subject: [PATCH 13/23] THREESCALE-10698 Updated DeploymentConfig references in docs --- doc/apimanager-reference.md | 4 ++-- doc/operator-application-capabilities.md | 8 ++++---- doc/operator-user-guide.md | 20 ++++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/apimanager-reference.md b/doc/apimanager-reference.md index 913de8d62..3bea95d9b 100644 --- a/doc/apimanager-reference.md +++ b/doc/apimanager-reference.md @@ -91,7 +91,7 @@ One APIManager custom resource per project is allowed. | AppLabel | `appLabel` | string | No | `3scale-api-management` | The value of the `app` label that will be applied to the API management solution | TenantName | `tenantName` | string | No | `3scale` | Tenant name under the root that Admin UI will be available with -admin suffix. | ImageStreamTagImportInsecure | `imageStreamTagImportInsecure` | bool | No | `false` | Set to true if the server may bypass certificate verification or connect directly over HTTP during image import | -| ImagePullSecrets | `imagePullSecrets` | \[\][corev1.LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#localobjectreference-v1-core) | No | `[ { name: "threescale-registry-auth" } ]` | List of image pull secrets to be used on the managed DeploymentConfigs ServiceAccounts. See [imagePullSecrets field in K8s ServiceAccount documentation](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#serviceaccount-v1-core) for details on Image pull secrets. If not specified, `threescale-registry-auth` is used. Secret names that contain `dockercfg-` or `token-` anywhere in part of its name cannot be specified. If an update to this attribute is performed the corresponding DeploymentConfig pods have to be redeployed by the user to make the changes effective | +| ImagePullSecrets | `imagePullSecrets` | \[\][corev1.LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#localobjectreference-v1-core) | No | `[ { name: "threescale-registry-auth" } ]` | List of image pull secrets to be used on the managed Deployments ServiceAccounts. See [imagePullSecrets field in K8s ServiceAccount documentation](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#serviceaccount-v1-core) for details on Image pull secrets. If not specified, `threescale-registry-auth` is used. Secret names that contain `dockercfg-` or `token-` anywhere in part of its name cannot be specified. If an update to this attribute is performed the corresponding Deployment pods have to be redeployed by the user to make the changes effective | | ResourceRequirementsEnabled | `resourceRequirementsEnabled` | bool | No | `true` | When true, 3Scale API management solution is deployed with the optimal resource requirements and limits. Setting this to false removes those resource requirements. ***Warning*** Only set it to false for development and evaluation environments. When set to `true`, default compute resources are set for the APIManager components. See [Default APIManager components compute resources](#Default-APIManager-components-compute-resources) to see the default assigned values | | ApicastSpec | `apicast` | \*ApicastSpec | No | See [ApicastSpec](#ApicastSpec) | Spec of the Apicast part | | BackendSpec | `backend` | \*BackendSpec | No | See [BackendSpec](#BackendSpec) reference | Spec of the Backend part | @@ -660,7 +660,7 @@ Each element of the Condition array has the following fields: * The *status* field is a string, with possible values **True**, **False**, and **Unknown**. * The *type* field is a string indicating the type of the condition. The types are: * `Available`: An APIManager is in `Available` state when *all* of the following scenarios are true: - * All expected DeploymentConfigs to be deployed exist and have the `Available` condition set to true + * All expected Deployments to be deployed exist and have the `Available` condition set to true * All 3scale default OpenShift routes exist and have the Admitted condition set to true. The default routes are: * Master route * Backend Listener route diff --git a/doc/operator-application-capabilities.md b/doc/operator-application-capabilities.md index aea388cd7..ad01564bd 100644 --- a/doc/operator-application-capabilities.md +++ b/doc/operator-application-capabilities.md @@ -28,8 +28,8 @@ The following diagram shows available custom resource definitions and their rela * [Backend custom resource status field](#backend-custom-resource-status-field) * [Link your 3scale backend to your 3scale tenant or provider account](#link-your-3scale-backend-to-your-3scale-tenant-or-provider-account) * [Product custom resource](#product-custom-resource) - * [Product Deployment Config: Apicast Hosted](#product-deployment-config-apicast-hosted) - * [Product Deployment Config:Apicast Self Managed](#product-deployment-configapicast-self-managed) + * [Product Deployment: Apicast Hosted](#product-deployment-apicast-hosted) + * [Product Deployment: Apicast Self Managed](#product-deployment-apicast-self-managed) * [Product authentication types](#product-authentication-types) * [User Key](#user-key) * [AppID and AppKey pair](#appid-and-appkey-pair) @@ -365,7 +365,7 @@ spec: name: "OperatedProduct 1" ``` -### Product Deployment Config: Apicast Hosted +### Product Deployment: Apicast Hosted Configure your product with *Apicast Hosted* deployment mode @@ -380,7 +380,7 @@ spec: apicastHosted: {} ``` -### Product Deployment Config:Apicast Self Managed +### Product Deployment: Apicast Self Managed Configure your product with *Apicast Self Managed* deployment mode diff --git a/doc/operator-user-guide.md b/doc/operator-user-guide.md index 56300ce47..9bc72ba3c 100644 --- a/doc/operator-user-guide.md +++ b/doc/operator-user-guide.md @@ -503,25 +503,25 @@ spec: Check [*APIManager DatabaseSpec*](apimanager-reference.md#DatabaseSpec) for reference. #### Enabling Pod Disruption Budgets -The 3scale API Management solution DeploymentConfigs deployed and managed by the +The 3scale API Management solution Deployments deployed and managed by the APIManager will be configured with Kubernetes Pod Disruption Budgets enabled. A Pod Disruption Budget limits the number of pods related to an application -(in this case, pods of a DeploymentConfig) that are down simultaneously +(in this case, pods of a Deployment) that are down simultaneously from **voluntary disruptions**. -When enabling the Pod Disruption Budgets for non-database DeploymentConfigs will +When enabling the Pod Disruption Budgets for non-database Deployments will be set with a setting of maximum of 1 unavailable pod at any given time. -Database-related DeploymentConfigs are excluded from this configuration. -Additionally, `system-sphinx` DeploymentConfig is also excluded. +Database-related Deployments are excluded from this configuration. +Additionally, `system-sphinx` Deployment is also excluded. For details about the behavior of Pod Disruption Budgets, what they perform and what constitutes a 'voluntary disruption' see the following [Kubernetes Documentation](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) Pods which are deleted or unavailable due to a rolling upgrade to an application -do count against the disruption budget, but the DeploymentConfigs are not +do count against the disruption budget, but the Deployments are not limited by Pod Disruption Budgets when doing rolling upgrades or they are scaled up/down. @@ -719,7 +719,7 @@ Check [Expanding persistent volumes](https://docs.openshift.com/container-platfo #### Setting custom PriorityClassName PriorityClassName specifies the Pod priority. See [here](https://docs.openshift.com/container-platform/4.13/nodes/pods/nodes-pods-priority.html) for more information. -It be can be customized through APIManager CR `priorityClassName` attribute for each DeploymentConfig. +It be can be customized through APIManager CR `priorityClassName` attribute for each Deployment. Example for apicast-staging and backend-listener: ```yaml apiVersion: apps.3scale.net/v1alpha1 @@ -739,7 +739,7 @@ spec: #### Setting custom TopologySpreadConstraints TopologySpreadConstraints specifies how to spread matching pods among the given topology. See [here](https://docs.openshift.com/container-platform/4.13/nodes/scheduling/nodes-scheduler-pod-topology-spread-constraints.html) for more information. -It be can be customized through APIManager CR `topologySpreadConstraints` attribute for each DeploymentConfig. +It be can be customized through APIManager CR `topologySpreadConstraints` attribute for each Deployment. Example for apicast-staging and backend-listener: ```yaml apiVersion: apps.3scale.net/v1alpha1 @@ -772,7 +772,7 @@ spec: ``` #### Setting custom labels -[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) can be customized through the APIManager CR `labels` attribute for each DeploymentConfig and are applied to their pods. +[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) can be customized through the APIManager CR `labels` attribute for each Deployment and are applied to their pods. Example for apicast-staging and backend-listener: ```yaml apiVersion: apps.3scale.net/v1alpha1 @@ -912,7 +912,7 @@ spec: ``` #### Pod Disruption Budget -Whether Pod Disruption Budgets are enabled for non-database DeploymentConfigs +Whether Pod Disruption Budgets are enabled for non-database Deployments ```yaml apiVersion: apps.3scale.net/v1alpha1 From 8bb7010b17d3f8e532d25b3ca7b31e0d11c7e695 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Thu, 25 Jan 2024 17:16:16 -0500 Subject: [PATCH 14/23] THREESCALE-10698 Updated DeploymentConfig references in GrafanaDashboards --- .../backend-grafana-dashboard-1.json.tpl | 392 +++++++++--------- .../system-grafana-dashboard-1.json.tpl | 84 ++-- .../zync-grafana-dashboard-1.json.tpl | 74 ++-- pkg/assets/bindata.go | 6 +- 4 files changed, 278 insertions(+), 278 deletions(-) diff --git a/pkg/assets/assets/monitoring/backend-grafana-dashboard-1.json.tpl b/pkg/assets/assets/monitoring/backend-grafana-dashboard-1.json.tpl index c048b73a6..1b88a6e83 100644 --- a/pkg/assets/assets/monitoring/backend-grafana-dashboard-1.json.tpl +++ b/pkg/assets/assets/monitoring/backend-grafana-dashboard-1.json.tpl @@ -3621,15 +3621,15 @@ }, "id": 13, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "Pods ($deploymentConfig)", + "title": "Pods ($deployment)", "type": "row" }, { @@ -3689,7 +3689,7 @@ } ], "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -3704,7 +3704,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -3783,7 +3783,7 @@ } ], "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -3798,7 +3798,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -3876,7 +3876,7 @@ } ], "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -3891,7 +3891,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -3969,7 +3969,7 @@ } ], "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -3984,7 +3984,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -4041,7 +4041,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -4053,7 +4053,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -4062,21 +4062,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -4161,7 +4161,7 @@ "renderer": "flot", "repeatedByRow": false, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -4173,7 +4173,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{ pod }}`}}", @@ -4235,13 +4235,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 13, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "Pods ($deploymentConfig)", + "title": "Pods ($deployment)", "type": "row" }, { @@ -4304,7 +4304,7 @@ "repeatPanelId": 30, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4319,7 +4319,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -4401,7 +4401,7 @@ "repeatPanelId": 32, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4416,7 +4416,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -4497,7 +4497,7 @@ "repeatPanelId": 37, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4512,7 +4512,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -4593,7 +4593,7 @@ "repeatPanelId": 36, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4608,7 +4608,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -4668,7 +4668,7 @@ "repeatPanelId": 11, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4680,7 +4680,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -4689,21 +4689,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -4790,7 +4790,7 @@ "repeatPanelId": 9, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -4802,7 +4802,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{ pod }}`}}", @@ -4864,13 +4864,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 13, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "Pods ($deploymentConfig)", + "title": "Pods ($deployment)", "type": "row" }, { @@ -4933,7 +4933,7 @@ "repeatPanelId": 30, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -4948,7 +4948,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5030,7 +5030,7 @@ "repeatPanelId": 32, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5045,7 +5045,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -5126,7 +5126,7 @@ "repeatPanelId": 37, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5141,7 +5141,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5222,7 +5222,7 @@ "repeatPanelId": 36, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5237,7 +5237,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5297,7 +5297,7 @@ "repeatPanelId": 11, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5309,7 +5309,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -5318,21 +5318,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -5419,7 +5419,7 @@ "repeatPanelId": 9, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -5431,7 +5431,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{ pod }}`}}", @@ -5493,13 +5493,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 13, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "Pods ($deploymentConfig)", + "title": "Pods ($deployment)", "type": "row" }, { @@ -5562,7 +5562,7 @@ "repeatPanelId": 30, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5577,7 +5577,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5659,7 +5659,7 @@ "repeatPanelId": 32, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5674,7 +5674,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -5755,7 +5755,7 @@ "repeatPanelId": 37, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5770,7 +5770,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5851,7 +5851,7 @@ "repeatPanelId": 36, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5866,7 +5866,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod))", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -5926,7 +5926,7 @@ "repeatPanelId": 11, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -5938,7 +5938,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -5947,21 +5947,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig.*'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment.*'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -6048,7 +6048,7 @@ "repeatPanelId": 9, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -6060,7 +6060,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{pod}}`}}", @@ -6118,15 +6118,15 @@ }, "id": 4, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "CPU Usage ($deploymentConfig)", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -6163,7 +6163,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -6175,7 +6175,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -6239,13 +6239,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 4, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "CPU Usage ($deploymentConfig)", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -6285,7 +6285,7 @@ "repeatPanelId": 64, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -6297,7 +6297,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -6361,13 +6361,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 4, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "CPU Usage ($deploymentConfig)", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -6407,7 +6407,7 @@ "repeatPanelId": 64, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -6419,7 +6419,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -6483,13 +6483,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 4, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "CPU Usage ($deploymentConfig)", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -6529,7 +6529,7 @@ "repeatPanelId": 64, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -6541,7 +6541,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -6601,15 +6601,15 @@ }, "id": 5, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "CPU Quota ($deploymentConfig)", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -6648,7 +6648,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -6769,7 +6769,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6778,7 +6778,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6787,7 +6787,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6796,7 +6796,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6805,7 +6805,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6865,13 +6865,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 5, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "CPU Quota ($deploymentConfig)", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -6913,7 +6913,7 @@ "repeatPanelId": 1, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -7034,7 +7034,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7043,7 +7043,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7052,7 +7052,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7061,7 +7061,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7070,7 +7070,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7130,13 +7130,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 5, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "CPU Quota ($deploymentConfig)", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -7178,7 +7178,7 @@ "repeatPanelId": 1, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -7299,7 +7299,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7308,7 +7308,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7317,7 +7317,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7326,7 +7326,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7335,7 +7335,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7395,13 +7395,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 5, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "CPU Quota ($deploymentConfig)", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -7443,7 +7443,7 @@ "repeatPanelId": 1, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -7564,7 +7564,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7573,7 +7573,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7582,7 +7582,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7591,7 +7591,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7600,7 +7600,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -7656,15 +7656,15 @@ }, "id": 6, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "Memory Usage ($deploymentConfig)", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -7700,7 +7700,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -7712,7 +7712,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -7776,13 +7776,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 6, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "Memory Usage ($deploymentConfig)", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -7821,7 +7821,7 @@ "repeatPanelId": 2, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -7833,7 +7833,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -7897,13 +7897,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 6, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "Memory Usage ($deploymentConfig)", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -7942,7 +7942,7 @@ "repeatPanelId": 2, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -7954,7 +7954,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -8018,13 +8018,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 6, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "Memory Usage ($deploymentConfig)", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -8063,7 +8063,7 @@ "repeatPanelId": 2, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -8075,7 +8075,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace', pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -8135,15 +8135,15 @@ }, "id": 7, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "Memory Quota ($deploymentConfig)", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -8182,7 +8182,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -8303,7 +8303,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8312,7 +8312,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8321,7 +8321,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8330,7 +8330,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8339,7 +8339,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8399,13 +8399,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 7, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "Memory Quota ($deploymentConfig)", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -8447,7 +8447,7 @@ "repeatPanelId": 3, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -8568,7 +8568,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8577,7 +8577,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8586,7 +8586,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8595,7 +8595,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8604,7 +8604,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8664,13 +8664,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 7, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "Memory Quota ($deploymentConfig)", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -8712,7 +8712,7 @@ "repeatPanelId": 3, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -8833,7 +8833,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8842,7 +8842,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8851,7 +8851,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8860,7 +8860,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8869,7 +8869,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -8929,13 +8929,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 7, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "Memory Quota ($deploymentConfig)", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -8977,7 +8977,7 @@ "repeatPanelId": 3, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -9098,7 +9098,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9107,7 +9107,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9116,7 +9116,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9125,7 +9125,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9134,7 +9134,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -9190,15 +9190,15 @@ }, "id": 15, "panels": [], - "repeat": "deploymentConfig", + "repeat": "deployment", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" } }, - "title": "Network Usage ($deploymentConfig)", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -9234,7 +9234,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -9246,7 +9246,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -9328,7 +9328,7 @@ "points": false, "renderer": "flot", "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-cron", "value": "backend-cron" @@ -9340,7 +9340,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9402,13 +9402,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 15, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" } }, - "title": "Network Usage ($deploymentConfig)", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -9447,7 +9447,7 @@ "repeatPanelId": 17, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -9459,7 +9459,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9544,7 +9544,7 @@ "repeatPanelId": 18, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-listener", "value": "backend-listener" @@ -9556,7 +9556,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9618,13 +9618,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 15, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" } }, - "title": "Network Usage ($deploymentConfig)", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -9663,7 +9663,7 @@ "repeatPanelId": 17, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -9675,7 +9675,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9760,7 +9760,7 @@ "repeatPanelId": 18, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-redis", "value": "backend-redis" @@ -9772,7 +9772,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9834,13 +9834,13 @@ "repeatIteration": 1600791173100, "repeatPanelId": 15, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" } }, - "title": "Network Usage ($deploymentConfig)", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -9879,7 +9879,7 @@ "repeatPanelId": 17, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -9891,7 +9891,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{ pod }}`}}", @@ -9976,7 +9976,7 @@ "repeatPanelId": 18, "repeatedByRow": true, "scopedVars": { - "deploymentConfig": { + "deployment": { "selected": false, "text": "backend-worker", "value": "backend-worker" @@ -9988,7 +9988,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -10096,9 +10096,9 @@ "definition": "label_values(kube_pod_info{namespace='$namespace',pod=~'backend-.*'}, pod)", "hide": 0, "includeAll": true, - "label": "deploymentConfig", + "label": "deployment", "multi": false, - "name": "deploymentConfig", + "name": "deployment", "options": [], "query": "label_values(kube_pod_info{namespace='$namespace',pod=~'backend-.*'}, pod)", "refresh": 1, diff --git a/pkg/assets/assets/monitoring/system-grafana-dashboard-1.json.tpl b/pkg/assets/assets/monitoring/system-grafana-dashboard-1.json.tpl index cd77a29bd..8cfadf8fa 100644 --- a/pkg/assets/assets/monitoring/system-grafana-dashboard-1.json.tpl +++ b/pkg/assets/assets/monitoring/system-grafana-dashboard-1.json.tpl @@ -1464,8 +1464,8 @@ }, "id": 30, "panels": [], - "repeat": "deploymentConfig", - "title": "Pods ($deploymentConfig)", + "repeat": "deployment", + "title": "Pods ($deployment)", "type": "row" }, { @@ -1533,7 +1533,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1619,7 +1619,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1705,7 +1705,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1790,7 +1790,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m]))) by (pod)", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m]))) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -1855,7 +1855,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -1864,21 +1864,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -1970,7 +1970,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{pod}}`}}", @@ -2028,8 +2028,8 @@ }, "id": 37, "panels": [], - "repeat": "deploymentConfig", - "title": "CPU Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -2073,7 +2073,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2133,8 +2133,8 @@ }, "id": 39, "panels": [], - "repeat": "deploymentConfig", - "title": "CPU Quota ($deploymentConfig)", + "repeat": "deployment", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -2290,7 +2290,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2299,7 +2299,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2308,7 +2308,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2317,7 +2317,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2326,7 +2326,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2382,8 +2382,8 @@ }, "id": 41, "panels": [], - "repeat": "deploymentConfig", - "title": "Memory Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -2424,7 +2424,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2484,8 +2484,8 @@ }, "id": 43, "panels": [], - "repeat": "deploymentConfig", - "title": "Memory Quota ($deploymentConfig)", + "repeat": "deployment", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -2638,7 +2638,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2647,7 +2647,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2656,7 +2656,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2665,7 +2665,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2674,7 +2674,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2730,8 +2730,8 @@ }, "id": 45, "panels": [], - "repeat": "deploymentConfig", - "title": "Network Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -2772,7 +2772,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2859,7 +2859,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2917,8 +2917,8 @@ }, "id": 100, "panels": [], - "repeat": "deploymentConfig", - "title": "Active Record ($deploymentConfig)", + "repeat": "deployment", + "title": "Active Record ($deployment)", "type": "row" }, { @@ -2961,7 +2961,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(label_replace(rails_connection_pool_connections{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'},'connection', '$1', 'state', '(.*)')) by (connection)", + "expr": "sum(label_replace(rails_connection_pool_connections{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'},'connection', '$1', 'state', '(.*)')) by (connection)", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -3049,7 +3049,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum (rails_connection_pool_size{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'})", + "expr": "sum (rails_connection_pool_size{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'})", "format": "time_series", "instant": false, "interval": "1m", @@ -3138,7 +3138,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum (rails_connection_pool_waiting{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'})", + "expr": "sum (rails_connection_pool_waiting{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'})", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -3240,9 +3240,9 @@ "definition": "label_values(kube_pod_info{namespace='$namespace',pod=~'system-.*'}, pod)", "hide": 0, "includeAll": true, - "label": "deploymentConfig", + "label": "deployment", "multi": false, - "name": "deploymentConfig", + "name": "deployment", "options": [], "query": "label_values(kube_pod_info{namespace='$namespace',pod=~'system-.*'}, pod)", "refresh": 1, diff --git a/pkg/assets/assets/monitoring/zync-grafana-dashboard-1.json.tpl b/pkg/assets/assets/monitoring/zync-grafana-dashboard-1.json.tpl index 5e4a4acbc..ebeb6b168 100644 --- a/pkg/assets/assets/monitoring/zync-grafana-dashboard-1.json.tpl +++ b/pkg/assets/assets/monitoring/zync-grafana-dashboard-1.json.tpl @@ -1370,8 +1370,8 @@ }, "id": 13, "panels": [], - "repeat": "deploymentConfig", - "title": "Pods ($deploymentConfig)", + "repeat": "deployment", + "title": "Pods ($deployment)", "type": "row" }, { @@ -1439,7 +1439,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1525,7 +1525,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1611,7 +1611,7 @@ "tableColumn": "", "targets": [ { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1696,7 +1696,7 @@ "tableColumn": "", "targets": [ { - "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m]))) by (pod)", + "expr": "max(sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m]))) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -1761,7 +1761,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total-pods", @@ -1770,21 +1770,21 @@ "step": 10 }, { - "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "avail-pods", "refId": "B" }, { - "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deploymentConfig-[0-9]+'})", + "expr": "sum(kube_replicationcontroller_spec_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'}) - sum(kube_replicationcontroller_status_ready_replicas{namespace='$namespace',replicationcontroller=~'$deployment-[0-9]+'})", "format": "time_series", "intervalFactor": 1, "legendFormat": "unavail-pods", "refId": "C" }, { - "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (node))", + "expr": "count(count(container_memory_usage_bytes{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (node))", "format": "time_series", "intervalFactor": 1, "legendFormat": "used-hosts", @@ -1876,7 +1876,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(delta(kube_pod_container_status_restarts_total{namespace='$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{`{{pod}}`}}", @@ -1934,8 +1934,8 @@ }, "id": 4, "panels": [], - "repeat": "deploymentConfig", - "title": "CPU Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "CPU Usage ($deployment)", "type": "row" }, { @@ -1979,7 +1979,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2039,8 +2039,8 @@ }, "id": 5, "panels": [], - "repeat": "deploymentConfig", - "title": "CPU Quota ($deploymentConfig)", + "repeat": "deployment", + "title": "CPU Quota ($deployment)", "type": "row" }, { @@ -2196,7 +2196,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2205,7 +2205,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2214,7 +2214,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2223,7 +2223,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2232,7 +2232,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:{{ .SumRate }}{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2288,8 +2288,8 @@ }, "id": 6, "panels": [], - "repeat": "deploymentConfig", - "title": "Memory Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "Memory Usage ($deployment)", "type": "row" }, { @@ -2330,7 +2330,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2390,8 +2390,8 @@ }, "id": 7, "panels": [], - "repeat": "deploymentConfig", - "title": "Memory Quota ($deploymentConfig)", + "repeat": "deployment", + "title": "Memory Quota ($deployment)", "type": "row" }, { @@ -2544,7 +2544,7 @@ ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2553,7 +2553,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2562,7 +2562,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2571,7 +2571,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2580,7 +2580,7 @@ "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}) by (pod)", + "expr": "sum(container_memory_usage_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+', container!=''}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2636,8 +2636,8 @@ }, "id": 15, "panels": [], - "repeat": "deploymentConfig", - "title": "Network Usage ($deploymentConfig)", + "repeat": "deployment", + "title": "Network Usage ($deployment)", "type": "row" }, { @@ -2678,7 +2678,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2765,7 +2765,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deploymentConfig-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~'$namespace',pod=~'$deployment-[a-z0-9]+-[a-z0-9]+'}[5m])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{pod}}`}}", @@ -2865,9 +2865,9 @@ "definition": "label_values(kube_pod_info{namespace='$namespace',pod=~'zync-.*'}, pod)", "hide": 0, "includeAll": true, - "label": "deploymentConfig", + "label": "deployment", "multi": false, - "name": "deploymentConfig", + "name": "deployment", "options": [], "query": "label_values(kube_pod_info{namespace='$namespace',pod=~'zync-.*'}, pod)", "refresh": 1, diff --git a/pkg/assets/bindata.go b/pkg/assets/bindata.go index 209cbe8ea..cb4b73b71 100644 --- a/pkg/assets/bindata.go +++ b/pkg/assets/bindata.go @@ -124,7 +124,7 @@ func monitoringApicastGrafanaDashboard2JsonTpl() (*asset, error) { return a, nil } -var _monitoringBackendGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\x6b\x93\x9b\x46\xba\xfe\x9e\x5f\xd1\x87\x64\x37\xe3\x9c\x19\x5b\xe8\x2e\x57\xb9\x4e\xf9\xba\xf1\xae\x9d\x9d\x75\x9c\x9c\xcd\x71\x5c\x5a\x46\xf4\x68\xd8\x41\x40\x00\x8d\x47\x9e\x9a\xfd\xed\xa7\x68\x10\x34\x97\xd6\xe8\x82\xa4\x06\x9e\x7c\x88\x47\x0d\x42\xcd\xdb\x97\xf7\x79\xde\x5b\xdf\x7d\x43\x88\xa2\x59\x96\xed\x6b\xbe\x61\x5b\x9e\xf2\x94\x04\x4d\x84\x28\xa6\xe1\xf9\xca\x53\xf2\x89\x7d\x22\x51\x2b\xbb\x72\x31\x37\x4c\xff\xad\xa5\x3c\x25\xea\x69\xd2\xaa\x6b\xbe\xe6\xd9\x73\x77\x42\x95\xa7\x44\x39\x3b\x23\x7f\x71\xb5\x4b\xcd\xd2\xc8\xd9\x99\xc2\xdd\x46\x2d\xed\xc2\x0c\x6e\xf1\xdd\x39\xe5\xda\xaf\x0c\xbd\xa0\xd5\x98\xd8\xd6\x4b\xdb\xb4\xdd\xe0\x99\xee\xf4\x42\x3b\x69\x9d\x92\xb6\xaa\x9e\x92\x76\xaf\x77\x4a\xd4\x47\xfc\xa3\x2d\x6d\xc6\x7e\xfb\x79\xf2\x3a\xe4\xcf\xe4\xb9\x49\x5d\xdf\xe3\xef\xf3\x17\x0e\xbb\x4f\xd7\xbc\xab\x0b\x5b\x73\x75\x25\xba\x76\xcf\xfe\xfd\xfc\x0d\x21\xf7\xc1\xed\x0a\xd5\x0d\x3f\xd3\x5b\x65\x6a\x51\xff\xad\xae\x3c\x25\xd6\xdc\x34\xc3\x16\x57\x73\xae\x3e\xda\xb6\xe9\x1b\x8e\xf2\x94\xb4\x58\xa3\x11\xdc\x32\x64\x7f\x9a\x86\x75\x1d\xc8\xf5\xd3\x67\xf6\xd1\xd1\x2c\x6a\x7a\xb1\x64\x97\x72\x55\x26\xb6\x69\x6a\x8e\x47\x83\x2f\x5e\x6a\xa6\x17\x8b\x41\x99\xba\x86\x7e\x6e\x27\x43\x13\xca\x2b\x23\xfe\x2f\xca\x53\xd2\xee\x72\x0d\xb7\xcb\xbe\x44\x9f\x17\xc1\xe7\xe5\x8b\xc6\xcf\x66\xfd\x6c\xc7\xf7\x71\xbd\xfb\x1c\xb7\xf9\x86\xcf\x64\xa0\xbc\x33\x3c\x9f\x5a\xd4\x4d\xa4\x19\xcb\xd2\xb5\xbf\x84\x52\x8c\x1e\x1d\xbf\x96\x66\x1a\x9a\xc7\x86\x90\xbd\x40\xf2\xcb\x17\x1a\x6b\x49\xbf\x6a\x30\x24\xef\xa8\x35\xf5\xd9\xeb\xb5\x52\xed\xb4\xe8\x76\x7e\xce\x7d\xc7\x7d\x8c\x6f\xb9\x34\x4c\x93\x17\x95\x58\x9a\xc3\x8c\x34\xd5\xf6\x03\xd2\x54\x8b\xa5\xd9\x19\xc5\x9f\x4d\x3a\xa5\x96\x9e\xfe\x29\xed\x66\x9a\x7d\x8f\x60\xf4\xe7\xae\x4b\x2d\xbf\xe0\xca\x4c\xbb\x2d\x6a\x35\xac\x82\x56\xef\xca\xfe\x92\x5f\x44\xbe\xed\x6b\x66\xc1\xdd\x37\x9a\x39\x4f\x84\x9a\x7b\x19\xd3\xb0\xd8\x55\xfe\x69\xac\xf1\x8b\xa1\xfb\xa9\xe9\x97\x99\xe2\xac\x29\x58\x1e\xe7\xb6\x61\xf9\xef\x6d\xb6\xb0\x59\x43\x32\x2c\xb6\x13\x6f\x37\xc9\x2f\x3a\xd4\x9d\x50\xcb\xd7\xa6\x34\x37\xd2\x4e\xf0\x28\x57\xd3\x8d\x79\xf0\x9d\x76\xba\x3d\x3f\x31\x5c\x6a\xe9\xd4\xa5\x6c\xdb\xb8\x34\x6d\x3f\xf9\x61\x8f\xba\x06\xf5\xfe\x7e\x43\x5d\xd7\xd0\x69\xa6\xd3\x9e\xa3\x4d\x68\xd1\xfc\xf3\x7c\x6d\x72\x9d\xfb\x15\xcf\xa7\x8e\x43\xf5\x77\x86\x95\xef\xb0\xaf\xb9\x53\xea\x7b\xdc\x0e\xca\xef\xa1\xc1\xe6\x72\xeb\xb0\xee\x79\xf3\xd9\x89\xab\xf9\xf4\x44\x73\x0c\xcf\xb6\x34\xdf\x76\xc7\x66\xb4\xd0\xc6\x2e\xf5\x1c\xdb\xf2\xe8\x78\x62\xeb\xd4\xbb\x0b\x76\x38\xd6\xc7\x67\xbf\x2b\xdf\xc5\x1f\x7e\x57\x4e\x5d\xfa\xc7\x9c\x7a\xfe\x38\x58\x8e\xcf\x7e\x57\xb4\xb9\x7f\x65\xbb\xc6\x57\xfa\xbb\x72\xff\x49\x9d\x7d\x7e\xc4\xef\x93\xc1\xa2\xb0\xdd\x99\x16\x4c\x36\xc5\x37\x66\x74\x1c\xca\x24\x7d\x8b\x61\xf9\xd4\xbd\x61\xf3\x46\x51\x67\xc5\xd7\xde\x68\x13\x9f\x6d\xcd\x6a\x2b\x75\x3d\x9c\xf6\x6f\xe2\x1f\x89\xbb\x93\x7e\x8c\x4b\x2f\xd9\x4e\xaa\x3c\x57\xe2\xe6\xfb\xd3\xe3\x48\xcb\xa5\x8e\x3c\xb2\x72\xa9\x23\x90\xd4\x8b\xe3\x4a\xca\xa5\x8e\xed\xfa\x72\x08\x2a\xec\x8b\x40\x4e\x2f\x8f\x3f\xa3\xd8\x84\x1f\xdb\xc1\x9f\x72\x08\x2c\xd3\x29\x81\xe4\x5e\x1d\x5f\x72\x2e\x75\x64\x93\x5b\xdc\x25\x81\xd4\x5e\x73\x52\x8b\xfe\xe2\xf0\xd3\x95\x4b\xbd\x2b\xdb\xd4\x73\xb8\x6a\x46\xdf\xb8\xf6\x8c\x03\x93\x71\xfb\x07\x3a\x8d\xf4\x63\xe6\x0b\x3f\x5f\x19\x97\x7e\xfe\x1b\x19\x84\x46\x22\xb1\x7a\xc4\xa1\x2e\xf1\xe8\xc4\xb6\x74\x72\x72\xb1\x58\xb6\x93\x40\xdc\x8f\x38\x18\x17\xc3\xd7\x3b\x1e\x4d\x68\x2e\x83\xa3\x19\x3c\xe1\x05\xeb\x8e\x53\xc2\x4b\x28\x31\x5e\x82\x41\xc3\xd2\x8d\x1b\x43\x9f\x6b\xa6\x92\x43\x15\xcb\x7b\x18\x6a\x4e\x3a\x70\xab\xdd\x1a\x19\x4c\x76\x31\x9f\x5c\x87\x1a\x94\x7f\xd7\x00\xfb\x44\x88\x22\x10\x47\x01\xfe\xcf\xdc\x5d\x8c\x89\x62\xec\xf3\xe9\x73\xae\x8b\x0b\xed\x96\xae\x50\xdc\x3a\x9d\x18\x33\x8d\x81\xe4\x96\x60\x5e\xba\xf4\x0f\x27\x33\x23\x4d\xed\x82\x9a\xb9\xde\x05\x17\xec\xe9\x0b\xcd\xa3\x69\x38\x1f\xe3\xbe\xdc\xed\x21\xf0\x4b\xff\x30\xf7\x8a\x0f\x2e\xde\xa4\x93\xde\x55\x6e\xff\x2c\xb7\x93\xb9\xe6\xc2\x7e\xe6\x96\xcb\x22\x3f\x15\x34\xd3\x98\x16\xe1\x5d\xd6\xfe\x8e\xde\xc4\x9d\xfe\x86\x7f\x68\x8d\xa9\x48\xaa\x61\x05\x17\xe9\xaa\xa5\x72\x91\x80\xa4\xbf\x9e\x39\xfe\xa2\x98\xbf\xff\x1f\x75\xed\xfc\x15\x10\x18\x10\x98\x35\x60\x80\xe7\xb0\x7b\x9e\xfd\xae\xb4\x6f\x6f\xe5\xd0\xfc\xed\x7f\xfe\x53\x1a\xc6\x92\x88\xa7\xdb\xea\xc8\x21\x9e\x6e\xab\x23\x0d\x4d\xe1\xc5\xd3\x95\x45\x3c\x5d\x69\xd8\x09\x2f\x9e\x91\x2c\xe2\x19\x49\x43\x41\x12\xf1\xf4\x64\xd9\x7b\x7a\xc2\xbd\xa7\x42\x5c\x23\x94\x3f\x09\x64\x0b\xb2\x01\xb2\x01\xb2\x51\x49\xb2\x91\xf5\x7b\x8c\x04\x5c\xa3\x53\x2a\xd7\x00\x6d\x00\x6d\xa0\xe3\x60\xdb\xf5\xc6\xa1\x52\xf1\xc6\xe1\x2e\xbd\x83\x1b\x84\x5c\x2c\xc8\x89\x49\x85\xaa\xfd\x8a\x6a\xfe\x4c\x73\xf6\xa7\xd6\xef\xee\xfe\x75\x77\x47\x4c\x4a\xee\xef\xff\x75\x7f\xbf\x06\xbb\x38\xa6\x86\x7f\xbe\x94\xdf\xc3\x2a\x3e\x78\x18\x09\x87\x87\x18\x56\x74\x8b\x07\xa5\x0f\xa5\xdf\x24\xa5\x3f\xd1\x5c\x3d\xf3\xe0\xa0\xe9\x5c\xd3\x75\xc3\x9a\xe6\x67\x4e\x70\xf1\x83\x3d\xb7\xf4\xcc\xc3\xe3\x9e\x4e\xa2\xd8\x97\xcc\x03\xe3\x90\x98\x6f\xdf\x3c\x7f\xf5\xba\xfd\x9c\x9f\xa3\xec\x2b\x3f\x4f\xb4\x70\x09\x7b\x7f\xa4\x46\x60\x79\xf5\x8a\xce\xa2\x75\xe4\x53\xd7\xb1\x4d\xcd\xa7\x7f\x77\x35\x6b\x9a\x62\x34\xc1\x56\x6d\x5b\xa1\x76\x6e\x3d\xee\x15\xac\x0f\xdb\xd1\x26\x86\xbf\xc8\xaf\xc1\x00\x91\x24\x9b\x9e\xef\x2d\x57\xda\x26\x08\xa6\x44\xeb\x68\x1e\xb1\x2c\x37\xfa\x94\x1e\x5e\xda\x30\x5f\xc4\xfb\x42\x5a\xb7\x5d\x19\xd3\x2b\xd3\x98\x5e\xf9\x2f\xa3\x71\x4e\x41\x84\x10\x04\xf5\x56\x82\xa0\x68\x7e\x0a\x81\x47\x16\x4d\x14\xc2\x05\x97\xde\x50\xd7\xa3\xbf\x89\xba\x09\x15\xbc\x6f\x15\xbc\x42\xd5\xee\xa2\x51\xa3\x57\x2f\x47\xb3\x16\xe9\xa4\xa0\xf5\x47\xc3\xf3\xed\xa9\xab\xcd\x84\xf3\x70\xa9\x40\xb3\x23\xa1\xdc\x3e\xcf\xed\x9b\xf9\x0d\x37\x79\xce\x6d\x38\x41\x7f\x9a\xcf\x2e\x18\x22\x4d\x89\x24\xba\xf8\xb3\xf1\x35\xab\x51\x95\x45\xfe\x67\x8a\x35\x22\xaf\x6a\xf8\x7d\xab\x50\x97\x14\x6a\x92\xbc\xb2\x13\x49\xce\x31\x0d\x3f\x9e\x61\x85\x7b\xf5\x22\x7c\xa3\x17\xd1\x7e\x1e\x4c\x7d\x5b\xc9\x5e\x2d\x16\xc6\x22\x27\x8c\x42\xf5\x52\x63\x4e\xa9\x0e\x72\xf2\x0c\xf7\xd3\x01\x48\x25\x48\xa5\x6c\x1a\x8d\x8f\x16\xab\x83\x3e\x3b\x2c\xa5\x74\xa9\x03\x42\x19\xb6\x83\x50\x82\x50\xf2\x0f\x3d\x3e\xa1\xec\x0d\x46\xdd\x37\x6d\x10\xca\xb8\xa1\x38\xdc\x26\x0f\x57\xf6\xc5\x28\x7b\xab\xd3\x09\x1a\xc9\x28\xeb\xa6\x7f\x77\xe2\x93\x22\x75\x0a\x36\x49\xc0\x26\x9b\xc9\x26\xdb\xbd\x9c\x3c\xc3\xbd\xb4\x05\x36\x09\x36\x29\x93\x36\xcb\x64\xd4\xd4\x41\x99\x1d\x8e\x4c\x7e\x60\xc2\x03\x97\x0c\xdb\xc1\x25\xc1\x25\xf9\x87\x1e\x9f\x4b\xbe\x69\x77\x47\xbd\x97\xe0\x92\x71\x43\x21\x97\x2c\x00\x2b\x7b\xe3\x92\x43\x70\xc9\x9a\x6b\xdf\xad\xa9\xe4\x0a\x65\x0a\x26\x49\xc0\x24\x9b\xc9\x24\x3b\x9d\x9c\x3c\xd9\x4e\xda\x07\x93\x04\x93\x94\x4a\x97\x89\x6a\x0e\xd4\x41\xa9\x1d\x23\xe4\xf5\x84\x89\xf1\x11\xd8\x65\xd8\x0e\x76\x09\x76\xc9\x3f\x54\x02\x76\xf9\x66\x34\xec\xb4\xc0\x2e\xe3\x86\x42\x76\x59\x00\x60\xf6\xc5\x2e\xfb\xab\x8b\x0d\x34\x8e\x5d\xd6\x59\x23\x97\x10\x01\xbb\x4a\xc1\x82\x71\x12\x30\xce\x66\x32\xce\xae\xa0\x94\x4b\xbf\x0d\xc6\x09\xc6\x29\x9b\x7e\xcb\xd7\x6a\xab\x83\x76\x3b\x7c\x3c\x2c\xd8\x26\xd8\x26\xd8\xa6\xc4\x6c\xf3\xc5\x70\x30\x78\x35\x02\xdb\x8c\x1b\x0a\xd9\x66\x01\x78\xd9\x1b\xdb\x5c\x5d\x6e\xa2\x91\x6c\xb3\x9e\xda\x78\xe7\xe8\x58\x30\xcd\xd4\x45\x30\xcd\x0d\x98\xe6\x21\x8e\x65\xe8\x8a\xc2\x55\x7b\x49\x36\xf9\x7a\x07\x33\x90\xb7\xc1\xf2\xb4\x34\x93\x3c\x3f\x7f\xab\xe4\x66\x57\x13\x4f\x69\xe8\x0a\xca\x15\xa9\x2a\xce\x69\x00\xa1\x2e\x49\x85\x1b\xd1\xb2\x1b\x6b\x8e\xb1\x75\xe9\x73\x76\x50\x8d\x1c\xd5\x07\xb5\xec\xa1\x39\xfb\x28\x7e\x5a\x8a\xd0\x1c\xc7\x34\x26\xec\xb0\x9f\xf1\x35\x5d\xc8\x22\xbe\x4c\xaf\x04\x82\x2c\xab\x4c\x6a\xd9\x82\x74\xe9\x25\x75\x5d\xea\x8e\x2f\x0d\xd3\xa7\xae\x84\x42\xcd\xf6\x50\x20\xe0\xb2\x0a\xad\x96\x2c\x60\xf9\x04\x2a\x12\x60\x59\xa5\x58\xcb\x10\x20\x75\x5d\x5b\x96\xb9\x18\xf6\x45\x20\xb4\xd7\x32\x09\xed\x86\x5a\xb2\x28\x95\xb0\x2f\x02\xa1\xbd\x91\x48\x68\x33\xea\xbb\xc6\x44\x12\xa9\x45\x9d\x11\x88\xed\x2f\x12\x89\xcd\xa3\xee\x8d\x31\xa1\x63\xdf\xbe\xa6\xb2\xec\x71\xe9\x3e\x09\x84\xf8\xa3\x7c\x42\x94\x4b\x7c\x22\xc1\xbd\x95\x49\x70\xbe\x26\xcb\x46\xc7\xba\x22\x10\xd9\x5f\x25\x12\xd9\xdc\xd3\xa6\x74\x6c\x1a\x33\x43\x16\xc9\xf1\x3d\x12\x08\xf0\x6f\x32\x09\xd0\x37\x4c\xe3\x2b\x43\x50\x92\xc8\x2f\xe9\x90\x40\x7c\xef\x24\x71\x7c\xc6\xd5\xe3\x79\x8b\x15\x8e\xad\xe2\xde\x02\xbe\xce\xc2\x4e\x36\xce\xd7\x59\x23\xdb\x6c\xde\x5f\x28\x32\xce\xb6\xcb\xcd\xaf\xc1\xc1\x55\xf1\x65\x58\x74\xf7\x09\x0f\x1a\x79\x8a\xd5\xce\xb2\x6a\xd0\x91\x56\x25\xc8\xaa\x31\xe7\x5b\x95\x20\xab\xc6\x1c\x76\xb5\xb3\xac\x70\xf2\x15\x77\xa1\x5c\xee\x82\x63\xb0\x08\xc8\x0b\xc8\x4b\xc5\xc9\x4b\x36\xb0\xa4\x27\x28\x59\xae\x0e\x56\x9f\x01\x81\xc0\x12\xd0\x90\x1d\xd5\xfa\x6e\x81\xa2\xe9\x38\x93\x3a\x44\x88\x1e\x30\x5f\x83\x09\x0f\x69\x1a\x61\x3b\xb4\x3f\xb4\x3f\xff\xd0\xe3\xa7\x69\xe0\x3c\xac\x75\xcc\xae\x05\xd0\x65\x5f\x69\x1a\xea\xa0\xbf\x12\x0d\x55\x28\x4f\x03\xba\x78\xbf\xd9\x1a\x62\xd5\x8a\x24\x0d\x76\x09\x49\x1a\xcd\x2b\x07\xd0\x17\xe4\x86\xa8\x03\x9c\x8c\x05\x96\x29\xb1\x66\x13\x05\xe6\xd7\x41\xc7\x1d\x90\x6f\x26\x62\x24\x7f\xa3\x0b\x30\xcf\xa8\x1d\xcc\x13\xcc\x93\x7f\x28\x98\x67\x35\x98\x67\x01\x9c\xd9\x1f\xf3\xac\x4d\xb5\x73\xe8\xe7\x58\xdc\x7b\xe6\xa0\xeb\xa8\x5b\xb0\x51\x02\x36\xda\x4c\x36\x3a\x10\x94\x43\x57\x07\x48\xa6\x07\x1b\xad\x86\xb6\x13\x66\x37\xd7\x41\xf3\x1d\xf2\xf0\xad\x50\x8c\xe4\x4d\x28\x46\x30\xd3\xb0\x1d\xcc\x14\xcc\x94\x7f\x28\x98\x69\x35\x98\x69\x01\xb4\xd9\x1b\x33\x1d\xae\xce\x6e\x01\x33\x6d\x84\xae\xde\xe1\xa8\xae\x35\x54\x2f\x58\x2a\x01\x4b\x6d\x26\x4b\x1d\x0a\x4a\xa8\xab\xc3\xd5\x27\x54\x80\xa5\x82\xa5\x4a\xa2\xf9\xea\xa5\xe9\x8e\xe2\x2f\x05\x23\x8d\xda\xc1\x48\xc1\x48\xf9\x87\x82\x91\x56\x83\x91\x16\xc0\x98\xfd\x31\xd2\xd5\xa7\xcb\x80\x91\xd6\x52\x2f\x97\xe1\x27\x05\xfb\x04\xfb\x04\xfb\x5c\xbe\x7d\xf0\x05\x51\x4d\x9b\xe1\xea\x13\x2b\xc0\x3e\xc1\x3e\x8f\xa9\xe5\x32\xf5\x75\xeb\xa0\xdf\x0e\xc7\x3b\xdf\x5a\x3e\x9d\xba\x61\xe0\xd0\x6b\x26\xc8\xf5\x6a\x42\x80\x81\x82\x81\xe6\x3a\x09\x06\x0a\x06\x2a\x07\x03\xcd\x43\x99\xfd\x31\xd0\x2e\x18\x68\x93\x74\xf3\xd6\xdc\x73\x03\x55\x0b\x16\x4a\xc0\x42\x9b\xc9\x42\x47\xa2\xea\x44\x43\x54\x27\x02\x0b\x95\x58\xd3\xa5\x0f\x2c\xa9\x83\xa6\x3b\x1c\x0b\x7d\xcd\x84\x07\xd6\x19\xb6\x83\x75\x82\x75\xf2\x0f\x05\xeb\xac\x06\xeb\x2c\x80\x2e\xfb\x63\x9d\xa8\x4e\xd4\x28\x5d\xbc\x35\xeb\x5c\xa1\x5a\xc1\x32\x09\x58\x66\x33\x59\xa6\xda\x12\x95\x27\x1a\xa2\x3c\x11\x68\xa6\xbc\xaa\x2d\x7b\xc4\x63\x1d\x74\xdb\xe1\x78\xe6\xfb\x50\x7a\x20\x9a\x61\x3b\x88\x26\x88\x26\xff\x50\x10\xcd\x6a\x10\xcd\x22\xf4\xb2\x3f\xa6\x89\x6a\x44\xcd\x52\xc7\x5b\x53\xcd\x55\xda\x15\x5c\x93\x80\x6b\x36\x94\x6b\xaa\xa2\xe2\x43\x43\x14\x1f\x02\xd7\x94\x57\xb9\x09\xce\xc5\xaf\x83\x8e\x3b\x1c\xe5\xfc\x39\x14\x22\xf9\xc8\x84\x08\xe6\x19\xb6\x83\x79\x82\x79\xf2\x0f\x05\xf3\xac\x08\xf3\x2c\xc0\x32\x7b\x63\x9e\x23\x54\x1b\x6a\xa4\x72\xde\x9a\x80\xae\xa1\x6b\xc1\x43\x09\x78\x68\x43\x79\x68\x5b\x54\x5e\x68\x84\xf2\x42\xe0\xa1\xd2\xab\xba\x7a\x29\xb9\x83\x33\x50\x70\xcf\xa8\x1d\xdc\x13\xdc\x93\x7f\x28\xb8\x67\x45\xb8\x67\x01\x7e\xd9\x1f\xf7\x44\x5d\xa1\x86\x29\xe4\x5d\x59\x27\xf8\x26\xf8\x26\xf8\xe6\xf2\xed\xd9\x0d\xa2\x82\x42\x23\x14\x14\x02\xdf\x94\x58\xbd\xf9\x5a\xcd\xb2\x47\x0e\x48\x36\x03\xd9\x81\x69\x86\xed\x60\x9a\x60\x9a\xfc\x43\xc1\x34\xab\xc2\x34\x0f\x58\x3f\x68\x84\xfa\x41\x4d\x52\xc5\xdb\xd3\x4c\xa1\x66\x05\xc7\x24\xe0\x98\x0d\xe5\x98\x1d\x51\xb9\xa0\x11\xca\x05\x81\x63\xca\xab\xd8\xe6\x9e\x36\xa5\x63\xd3\x98\x19\x35\xd3\x6f\x87\xa3\x9a\xbf\x04\x22\x24\xef\x98\x08\xc1\x38\xc3\x76\x30\x4e\x30\x4e\xfe\xa1\x60\x9c\x15\x61\x9c\x05\x38\x66\x7f\x8c\x13\xb5\x83\x1a\xa8\x98\xb7\x26\x9e\x0f\xea\x59\xf0\x4f\x02\xfe\xd9\x50\xfe\xd9\x15\xd5\x11\x1a\xa1\x8e\x10\xf8\xa7\xc4\x6a\xce\x37\x4c\xe3\x2b\x2b\x46\x5e\x2b\x2d\x77\x40\xfa\x99\x48\x10\xec\x33\x6c\x07\xfb\x04\xfb\xe4\x1f\x0a\xf6\x59\x11\xf6\x59\x80\x62\xf6\xc7\x3e\x51\x4f\xa8\x79\x6a\x79\x7b\xf2\xf9\x80\x96\x05\xf7\x24\xe0\x9e\x2b\xb8\xe7\xc4\x36\x4d\xcd\xf1\x18\x8a\x4a\xef\x01\xc2\xfd\x53\xcd\xec\x9f\xed\xee\x43\x24\xb0\x27\x28\xf0\xd3\xe3\x18\x8a\x66\x51\x33\x87\x31\xa3\x49\xfe\xbf\xb6\x7b\x4d\x5d\x6e\xfb\x5f\x4e\x27\xd7\xfe\xa2\x14\xbe\x56\x75\x29\xf5\x1a\xd2\xec\x16\x4b\xb3\x8b\x6a\x49\x60\xd4\xbb\xa9\xee\x2f\x6c\xa1\x8d\xff\x6d\x5f\x8c\x27\xf6\xdc\x12\xeb\xe5\x7b\xc2\xe9\xde\x60\x35\x0a\xb5\x6f\xa0\x6b\xc6\xe1\xcb\xef\x5b\x03\x07\xfd\xa8\x04\x35\xfe\xab\x7d\x91\xa3\xc2\xff\xb6\x2f\x48\x5a\x90\xe0\xbc\xe5\x72\x5e\x1b\x8c\x57\x3a\xc6\x5b\x5d\x3d\xfd\xb0\xe9\xbb\xdf\x16\xa0\x9e\xd5\x01\xad\xd0\xd3\xd0\xd3\x1b\xe8\x69\x77\x6e\x45\x2a\x76\x3d\x42\x1d\x11\xe9\x0f\xd4\xb1\x5d\xff\xaf\xf6\x45\xad\x68\xf4\xe1\x54\x78\x28\xbf\x40\x6b\xe7\x8d\xda\xe1\x88\xc0\x9c\x0d\xd5\xde\x64\xd5\x7e\x78\x63\x76\x6f\x30\xea\xbe\x69\xc3\x98\x1d\x37\x14\x1b\xb3\x0b\x70\xc9\xbe\x8c\xd9\xbd\xd5\x41\xe6\xf2\xdb\xb2\xa1\x68\x4b\xb6\x57\x0b\xf4\x66\xca\x4c\xbd\xbd\xfe\x84\x81\xba\x76\x06\xea\x3a\x33\xc4\x41\x2b\x27\xd0\x70\xdb\x5c\x1d\x80\x0a\x86\x08\x86\xb8\x7f\xc5\xf5\x93\xed\x1b\x97\x8b\xba\x29\xae\xc3\x31\xc4\x50\x7e\x60\x88\x60\x88\xe9\x97\x01\x43\x5c\x3e\xf8\xf0\x0c\xf1\xa2\x7b\x79\xd9\x6a\x81\x21\xc6\x0d\xc5\x0c\xb1\x00\x97\xec\x8d\x21\xae\x0e\x03\x6f\x02\x43\xac\xa7\xa2\xdd\x9a\x21\x0a\xf4\x26\x18\x22\x18\xe2\x11\x43\x98\x06\xc3\xdc\x9b\x85\xe1\x9a\x49\xa9\xbe\xa2\x10\x26\x97\x3a\x34\x1c\x0a\x9d\x3a\xa6\xbd\x98\x51\xcb\x7f\x69\x5b\x97\xc6\x94\xe3\x13\x13\xdb\xa1\xfa\xaf\x21\x9d\x4d\x8d\x6e\xe6\x1b\x4f\xd3\xbb\x92\x47\x4d\x3a\xf1\xf3\xaf\xcd\x2e\xfa\xf4\x96\xfd\xec\x85\x36\xb9\xa6\x96\x7e\x36\x71\x6d\x2b\xbd\x8e\x19\xde\xca\xdd\x92\x5b\xd2\xf7\xf9\x15\x7a\x6e\xeb\x1e\x39\xf9\x2e\xdb\xbf\x47\x1b\xc4\x67\x4d\xb4\xc9\x15\xfd\x68\xcc\xa8\x3d\xcf\xed\x04\x4c\xe9\xbe\xd0\x26\xd7\x53\x37\x9a\x34\x29\xb5\xc1\x2e\xff\x1a\x75\x3e\x3d\xde\x93\xa5\x99\x20\xd9\xc7\x95\x6f\xdf\xb4\xbb\xa3\xde\x4b\x7e\x21\xb8\xd3\x0b\xed\xa4\xdd\x19\x9c\x12\xb5\x3d\x3a\x25\xdd\xd6\x29\x69\x3d\x1e\x8e\xf8\xed\x56\xf9\xb6\x3d\x1a\x4d\xba\x7d\x25\xb7\xb1\xad\xa1\x8a\x8b\x96\x25\xb7\x28\x2d\xdb\xe2\xf5\xb6\x36\x67\x64\xf5\x2e\xb5\x24\x97\xef\xa7\xb6\x5a\xe9\x65\xb9\xbc\x50\xb0\x36\xb3\x60\x2c\xe6\x3a\xef\x02\x14\x99\x53\x7b\xfc\x1d\xef\xb5\x30\xc0\x4e\xb0\x65\x09\xd7\x51\x27\xb3\x8e\xfa\x0f\x2e\xa3\x82\xc2\x51\x86\xce\x66\xc2\x92\x50\x17\xc2\x84\x4e\x22\x48\x5e\xcf\x29\xab\xb0\xc0\x4c\x73\x1c\xc3\x9a\x7e\x0c\xe7\xa2\x5a\xd4\xbe\x42\xe7\x47\xd4\x25\x5c\x26\xc4\xb7\x09\x5b\x51\x85\x2b\x48\x4d\x16\x8d\x00\xe4\x2f\x1f\xc6\x40\xe3\xea\x87\xb5\x57\x28\xd5\x99\x76\xfb\x4a\xf3\xb5\xf3\xa5\xc5\x82\x9b\x1d\x79\x6b\xc9\xc4\xb6\xac\x70\x7f\x48\xdd\xf3\x31\xdc\x18\x52\x2b\xae\xd8\x94\x62\xce\xa7\x86\xf5\x2b\x75\x3d\xc3\x0e\xb4\x81\xd2\x7f\xdc\x7e\xdc\x4d\x1e\xe6\xd8\x9e\x7f\x69\xdc\xa6\x87\x21\x6a\x7c\x63\x5b\xcb\x6d\x5b\xe9\xb5\xfe\xc4\x5d\x77\x69\xfe\x3b\xac\x4d\xf8\x15\x26\xb3\xf7\x9a\xb3\x62\xac\x2e\x43\xa0\x91\x36\x10\x11\x7e\x1b\xfc\xe9\xc9\xf3\xcc\x05\x3b\xfe\xc2\x0a\x81\x57\x65\x6b\xf6\x1c\xcd\xbd\x36\x43\x23\x12\xd7\xcd\x4b\xc3\x34\x63\xf2\xc3\x76\xbd\x8e\x7a\x4a\x54\x75\x78\xca\xce\xd6\x24\xad\xc7\xea\x30\xb5\xeb\x5d\xce\xcd\x22\x33\x5f\xf0\x64\xfe\x39\xe1\x63\xda\xad\x53\xa2\x8e\x3a\xa9\x07\xac\xc4\xeb\xbe\x76\x61\x06\xcf\x99\xcf\xac\xf4\x0c\xd8\x08\x80\x5f\xcf\x2f\xe8\xd8\xa5\x8e\x69\x4c\x58\xd8\xfb\xc4\xb6\x7c\xd7\x36\x4d\xea\x8e\x3d\x5f\xf3\xe7\xde\xd8\xa5\x9a\xbe\x58\xde\xe2\x71\xf8\xfb\xfb\x04\x7e\x7f\x7f\x5a\xf8\x88\x67\xff\xf9\x3e\xa7\xd7\x1e\xff\xf0\xfd\xfd\xf6\xe1\x8d\x09\xe4\x5e\x89\xb8\x77\x36\x66\x29\xea\x69\x5b\x29\x82\xdf\x4a\xa7\xe5\x29\x85\xf8\x3b\x7b\x65\xe9\xa2\x99\x5b\x96\x61\x4d\x89\x63\xeb\x05\x21\xd7\x9e\x61\x4d\x4d\x1a\x48\x3a\xb9\xc6\xe6\x2b\xbf\x80\x87\xfc\x02\x66\x57\x57\x2f\x60\x3b\xc0\xe9\xca\xb3\xe2\xb5\xdb\x12\x2c\x8e\x87\x16\x2f\xbb\xf1\xa7\x68\xe7\xd5\x6e\xa6\xfb\x41\x22\xe7\xcb\x1d\xad\x00\x8a\x6c\x80\x52\x22\xb8\xb1\x29\x4a\x89\xc0\x0d\x50\xca\x2e\x28\xa5\x5f\x1a\x4a\x69\x17\xa1\x94\xd4\x94\x02\x4e\x29\x1d\xa7\x00\x87\x00\x87\x48\x86\x43\x1c\x3a\x29\x19\x7f\x90\x33\x52\x1f\xf0\x73\x54\x78\xf3\x8b\xa5\xdd\x68\x86\x19\xcc\x01\x40\x1c\x18\x5b\xea\x03\x63\x0a\xfc\x38\xdb\xe2\x98\x01\xac\x2d\xb0\xb6\x00\xe5\xa4\xfa\xd9\x3c\x94\xc3\x52\x40\x4f\x96\xff\xb7\x7c\xcd\xb0\xa8\x3b\x9e\xd1\x99\xed\x2e\xc6\x61\x1d\xc0\x8b\x85\x4f\x85\x20\xc3\xb1\xf5\x22\x48\x71\xf6\x49\x3b\xfb\xda\x3a\x1b\x7d\xfe\xef\xe4\xaf\x00\xe1\x5c\x2c\xc8\x89\x65\xeb\xf4\x51\x93\x8c\x2d\xcc\x97\xa2\x1b\x9e\xef\x1a\x17\x73\x9f\xea\xc4\xb6\xc8\x95\xed\xf9\x80\x25\x65\xc2\x92\x2d\xad\x2b\x7a\xb7\xab\x75\x34\xc0\x92\xdd\x60\xc9\xb0\x34\x58\xd2\x87\x79\xe5\xf0\xc0\x04\xc0\x03\xc0\xe3\xc0\xc0\x63\xa6\xdd\x9e\x78\xf3\xd9\x89\x4e\x4d\x5f\x0b\x6d\x1e\x8e\xad\x8f\x13\x0c\x12\xdb\x3a\x3c\x5f\x73\x7d\x6f\xcc\xe2\xd3\x4b\x81\x21\x9f\x7a\x71\x00\x96\x63\xeb\x8d\x82\x22\xef\xb5\x5b\x66\x10\x21\x4b\xb1\x92\x13\x53\xf3\x7c\xd2\x23\x33\xc3\x9a\xfb\xd4\x2b\x08\xf0\xa8\x1b\x26\xa9\x4e\xb6\xc9\x20\xa3\x76\x1f\x0e\x61\x1a\x0a\xea\x11\xa8\x5c\x16\x46\x09\xd9\x26\x81\x2a\x7f\x3d\x73\xfc\x45\x3e\x5c\x6c\x19\x2f\x9a\xbf\x52\xa7\x14\x15\xa2\x79\xe4\x6b\xf0\x96\x0f\x28\xd6\x75\x53\x55\x7a\xe9\xf6\x8d\x52\x55\xaa\xa2\xd8\x64\x4c\xa9\x39\xac\x81\x7f\x77\x45\xd3\x5e\xa9\x68\xd8\xfa\x38\x4b\x19\xbc\xb9\xfb\xde\x19\xd6\x75\x51\x86\x00\xa7\x90\x52\xed\x81\x90\xd9\x60\x3c\x88\x79\xeb\x1f\xba\xc1\xdc\x09\x05\x92\x8d\x65\xf7\x42\x29\x4d\x4a\x70\x2c\x6d\x34\x34\x73\xeb\xc1\xc1\x79\xb9\xfe\xe0\xd4\xd6\x1e\x36\xf7\xa8\x7e\x96\xb6\x3a\xa5\x85\xf4\x4a\x92\x9c\xba\x73\x5b\x27\x4c\xfe\xe4\x84\xed\x67\xa7\x84\x8d\xef\x29\x99\x5b\xc1\xbf\x8f\x88\x66\xe9\x21\x86\x65\x6f\x93\x18\xd6\x0c\x4e\x45\x21\xb5\xae\xdc\xd4\xba\xbd\xe7\xad\x55\x3a\xb9\x8e\x87\x9d\x28\xad\x56\x48\x65\xfa\x9b\x53\x99\xbc\x05\x91\x51\x99\x72\x2b\xa0\x36\x9e\xc9\x94\xc5\x60\x76\x48\xb6\x0f\x13\x68\xa8\xfe\x62\xf1\x21\x6f\xe0\x06\xbf\xd9\x85\xdf\x1c\xdf\xba\xb6\x6f\x5c\x13\xa6\x31\x3a\xb6\x5e\x89\x82\x01\xe7\xeb\xda\xde\xb6\x80\x2f\xad\x86\xc2\x17\xa0\x94\x74\x27\xa5\x2b\x01\x70\x88\xdc\xcd\x91\xe8\x0c\xb2\x76\xa2\x12\x57\x26\x6f\xa6\xd6\x6b\xd8\xfa\xd6\xa7\x2e\x63\xb9\xac\xa2\x5a\xab\x35\x18\xa9\xea\xa0\xc3\x7b\xe2\xc2\xfb\xce\x83\xe7\xbe\xcd\x64\x8a\x1e\x44\x6d\x2d\x8f\xe1\x78\x40\x75\xc5\xb7\xad\x50\x5f\xc8\xfc\x5c\x13\xf3\xc2\xeb\x2f\x5e\x84\xfd\xdc\xc3\xd7\x72\xfa\xab\xed\xc2\xa4\x0a\x04\x23\x22\x18\x91\xec\x33\x26\x60\xdb\x6d\xbe\x93\xbd\x94\x70\x97\xd4\xe4\xae\x9a\x0e\x68\x4a\xec\x41\x7d\x8d\xe1\x12\x85\x1a\x20\xc5\x74\x13\xc8\x83\x14\xd3\xca\xc3\xa1\x5c\x8a\xe9\xf6\x70\xa8\x53\x04\x87\x10\x04\x89\x20\xc8\x63\x01\x9e\x36\x00\x4f\x9d\x01\x0f\x42\x0e\x90\xcb\x5a\x1f\x2c\x05\xf3\x51\x25\xf0\x52\x3e\x97\x75\x7b\xc0\xd4\x2d\x02\x4c\xb0\x1f\xc1\x7e\x24\x27\x9c\x1a\x00\x4e\x55\x19\x4e\xd5\x36\x48\x50\x22\xf3\x11\x92\x66\x91\x34\x5b\x73\xfc\x93\x4b\x9a\xdd\x1e\xff\xf4\x60\x30\x3a\x3c\x02\x02\xc2\x11\x23\x9c\x3e\x10\x4e\x95\x11\x0e\xb2\x73\x91\x9d\x8b\xec\x5c\xb2\x52\xbf\x6f\x9e\x9d\x3b\x12\x84\xb4\xab\xed\x72\x0f\x83\x6b\x7c\x50\xbb\x44\xe9\xb9\x5b\x87\x12\xaa\x35\xd5\xa0\x32\x86\xc9\x23\x0d\x18\x69\xc0\x48\x03\xae\xf4\xd0\x20\x0d\x18\x69\xc0\x48\x03\x46\x1a\x30\xd2\x80\x65\xe6\x4c\x1b\xa7\x01\xb7\x5b\x79\x9b\x68\xc4\x99\x56\x9f\x2a\x09\xce\x94\xb4\x56\x2b\x11\x78\x73\xae\x34\x02\x55\x42\x46\x31\x32\x8a\x91\x51\x5c\xbd\x8c\xe2\xf5\x92\x75\xe5\xc5\x3c\x75\x4e\x2a\x6e\xab\xa2\x6a\x8a\xed\x61\xa2\xe4\x6a\x97\x54\xec\x52\xdd\xf0\x1e\x50\x5f\xe1\x3d\x2b\x74\x17\xd2\x89\xd7\xc4\xce\x88\x87\x10\x2f\xbf\x4e\xee\xe1\x6b\x86\x43\x24\x80\x10\xe1\xa0\x08\x07\x95\x3e\x58\x42\xa6\x74\xe2\x52\x76\xff\xa6\x84\x49\xd4\xd7\x9c\x2e\x51\x54\x04\x12\x89\x37\x01\x3b\x48\x24\xae\x3c\x10\xca\x26\x12\x6f\x0f\x84\x8a\x8f\xd4\x47\x5c\x28\xe2\x42\x91\x48\x0c\xa8\x53\x2a\xd4\x41\xb8\x02\x52\x88\xeb\x83\xa2\x60\x32\xaa\x04\x52\xca\xa5\x10\xef\x00\x95\xd4\x22\xa8\x04\x9b\x11\x6c\x46\x72\x02\x29\x99\x52\x88\x01\xa4\x90\x3c\x2c\x95\xc9\x08\xc9\xc3\x48\x1e\xae\x39\xf2\xc9\x26\x0f\xef\x80\x7c\x0a\x8b\xef\xc2\x48\x04\x23\x11\x92\x87\x81\x6d\x36\xc2\x36\x48\x1b\x46\xda\x30\xd2\x86\xc9\x4a\xcd\xbe\x71\xda\x70\x5b\x15\x85\xc0\x77\x92\xc8\x30\x84\xc0\xc7\x8d\x48\x1b\xe6\x2f\x55\x5f\x77\xca\x18\x05\x8f\x84\x61\x24\x0c\x23\x61\xb8\xd2\x43\x83\x84\x61\x24\x0c\x23\x61\x18\x09\xc3\x48\x18\x96\x99\x2d\x6d\x9e\x30\xdc\xce\xdb\x41\x23\xb6\xd4\x05\x5b\x42\xc2\xb0\x74\x09\xc3\xf5\x25\x49\xc7\xb7\x02\xee\x1b\x1c\x21\x55\x18\xa9\xc2\x47\x47\x11\x55\x41\x3b\xb5\x4e\x15\x6e\x8b\x4a\x3b\x76\x38\x23\x60\xed\x52\x85\xbf\xd8\xee\xf5\x83\xa5\x2e\xa2\x9b\x56\x68\x2f\x24\x0b\xaf\x89\x9b\x11\xff\x20\x5c\x80\x9d\x56\xee\xe1\x6b\x86\x3f\x24\xde\x66\x04\x7e\x22\xf0\x53\xfa\xe0\x08\x99\x92\x85\x4b\xda\xff\x9b\x12\x1e\x51\x5f\x63\xba\x44\xd1\x10\x48\x17\xde\x04\xee\x20\x5d\xb8\xf2\x50\x28\x97\x2e\xbc\x3d\x14\x1a\x14\x41\x21\x44\x82\x22\x12\x14\xe9\xc2\x00\x3b\xcb\x0b\xd2\x86\xc9\xd4\x2a\x5c\x01\x09\xc3\x99\x65\x28\x37\x8e\x82\xd9\xa8\x12\x58\x29\x9f\x30\xbc\x3d\x58\x1a\x16\x81\x25\xd8\x8d\x60\x37\x92\x13\x4a\xc9\x94\x30\x0c\x28\xb5\x11\x94\xaa\x6d\x70\xa1\x44\x66\x23\xa4\x0c\x23\x65\xb8\xe6\xd8\x27\x97\x32\xbc\x3d\xf6\x29\x2c\xb0\x0b\x43\x11\x0c\x45\x48\x19\x06\xba\x59\x5e\x40\xd2\xb0\xc4\x78\x07\x49\xc3\x55\x0a\x83\xdf\x3c\x69\xb8\x23\x0a\x83\xef\x26\x3d\x47\x18\x7c\xdc\x88\xa4\x61\xfe\x52\x1d\xb4\xa7\x8c\x11\xf1\x48\x1b\x46\xda\x30\xd2\x86\x2b\x3d\x34\x48\x1b\x46\xda\x30\xd2\x86\x91\x36\x8c\xb4\x61\x99\xf9\xd2\xe6\x69\xc3\xdd\xbc\x2d\x34\xe2\x4b\x2a\xf8\x12\xd2\x86\xa5\x4b\x1b\xae\x33\x4d\x3a\xbe\x25\x70\xdf\xf0\x88\x25\x0e\x3b\xb6\x8e\xb4\xe1\x66\x62\x20\x09\x50\x44\x55\xd0\x4e\xad\xd3\x86\xbb\x82\xd2\x8e\x49\xa9\x92\x95\x49\xc3\x79\x95\x72\x58\x25\x34\x71\x39\x12\x44\x0a\x55\x10\xbb\x65\x85\x02\x8a\xb7\x99\x97\xe7\xbf\x90\x5f\x02\xc6\xbb\x63\x02\x70\x75\x40\xea\xe6\x46\xfd\xee\xa0\x78\xba\xf4\x93\xf9\x22\x88\x45\x2c\x01\xb9\xd6\x09\x84\x4a\x64\xb4\xaf\xca\x42\x95\x12\x29\x5a\xb6\x4e\xc7\x31\xe6\x4b\xa3\xc5\xa7\x09\x6e\x9c\x38\xf3\xc8\x9a\xe6\xd1\x89\x6d\xe9\x11\x72\x7c\x7a\x77\x47\x1e\xff\x3c\x9f\x7d\xd0\x7c\x4a\xee\xef\x39\x20\xf9\x9f\x5d\xad\x6c\xe5\x80\xc8\xd5\xf6\xf7\x15\x20\xb2\x34\x13\xfc\x31\x81\x67\xac\x11\x60\x67\x93\x1c\x63\xc2\x9c\xc6\xb5\xcb\x85\x30\x7b\xdd\x62\xc8\xa0\x76\xdb\xeb\x61\xcc\x52\x0a\xd3\x74\x0f\xab\xee\x4c\xc3\xf3\xa9\xf5\xa0\x79\x24\xbe\x0d\xf8\xb4\x1c\x7c\xda\xeb\x89\x26\x5b\x07\x00\x95\x54\x14\xa0\x6e\xbb\xe6\x39\x52\x72\x7c\x73\x69\x89\x3b\x02\x80\xb0\x94\x40\x58\x54\x86\x11\x50\x18\x50\x18\x50\x38\xdd\xc9\x66\x42\xe1\x7e\x5b\x84\x4e\xd6\x34\xb7\x56\x11\x0a\x97\x52\x5e\x18\x20\x78\x93\x69\x26\x8c\xbc\xee\x01\x04\x13\x80\x60\x94\x1a\x07\xfc\x05\xfc\x05\xfc\x05\xfc\x05\xfc\x8d\xee\x3e\x08\xfc\x1d\x08\x23\x1c\xfb\xf5\x85\xbf\x65\x57\x28\x07\x00\x5e\x63\xa2\xa9\xa2\x89\x56\x58\x5f\x12\x00\x18\x00\xb8\x5a\xbb\x01\x20\x30\x20\x30\x01\x04\x06\x04\x06\x04\xae\x18\x04\x1e\x16\x23\x93\x35\xcf\xe8\xa9\x5b\xb8\xed\x3f\xe6\xb6\xaf\x1d\x14\xc8\x4e\x58\xa1\x98\x8c\x7c\x0f\x80\x6e\x2f\xb9\x02\x25\x6a\x8b\xaf\x50\x52\x26\xee\x15\x9d\x01\x05\x7c\xbb\x23\xbe\xd5\xa6\x34\x1a\xbd\x94\x72\x44\x78\x2e\x87\x49\x27\xae\xcd\x26\x7d\x1a\x55\xaf\x44\xaa\x57\xf6\x97\x1f\xa9\xa6\xb3\xd7\x4e\x7f\x2d\x04\x0b\xdc\xeb\x4e\x6c\x33\xb3\xe3\xea\xd4\x9b\x08\xe7\x4a\x89\x20\xd8\xf3\x17\xe6\x2a\xcd\xce\x36\xa0\x40\x52\x1f\xd3\xc8\x22\xdc\x29\x68\x82\x11\x7f\xfb\xed\xb7\xdf\xce\xde\xbf\x3f\x7b\xf5\x8a\xfc\xf8\xe3\xd3\xd9\xec\xa9\x97\xc1\x9c\x8e\xe6\xfb\xd4\xb5\x8a\x9f\xb5\xdc\x08\xaf\x0c\x5d\xa7\xd6\xc3\x19\xfe\x71\xb7\xf2\xc0\x6d\x29\x50\xdb\x8d\xd6\x42\x4e\x19\x27\xb5\x13\x3f\xef\xf2\x42\x5c\xa6\x75\x06\x3e\x87\x58\x38\x3f\x31\x83\x0b\x1f\x63\x58\xa9\xbc\x72\x0d\xd3\x24\xba\xfd\x25\x33\x3b\x83\xdb\x7e\x71\xd3\xe4\x3d\x2b\x42\x56\x1c\x91\x7c\x9b\x2d\x03\x57\x8c\x98\x53\x22\xb6\xe6\xb3\x8b\x2c\x4b\x9b\x5b\x06\x07\x8c\x36\x93\xfe\x07\xfa\xc7\x9c\xe6\x2a\x11\x34\x65\x00\x5e\xc8\x33\x00\xe4\x4f\xcd\x1c\x82\x97\xe5\x0e\x41\xa4\xf6\xd8\xc7\xcd\x06\xe2\x9d\x31\x33\x9a\xba\x0e\x5e\x1d\x7f\x1d\x84\xe2\x6f\xea\x2a\x78\x2d\xc3\x2a\x38\xb7\x75\xf9\xa4\x9f\xc6\xe7\x5b\x09\xff\x89\xfe\xe4\xee\x8e\x3c\xfe\x69\x69\xa9\x23\xf7\xf7\xb9\x86\xb3\x8e\x37\xd1\x4c\x7a\x76\x3d\xbf\xa0\xae\x45\x7d\xea\x9d\x4d\xec\x99\x33\xf7\xe9\x99\x4b\x43\x0a\xe5\x9d\x39\xb6\xfe\x3f\x37\x9a\x7b\x96\x18\x00\x13\xf3\xdf\x9f\x83\x0b\x8e\xad\x3f\xfb\x6e\x3c\x9e\xd0\x6c\xe9\x57\x6e\xc4\x9d\xac\x94\x0f\xbd\xda\x24\x1a\x63\x4e\x2c\x4f\x1e\xff\xf0\x64\x73\xb9\x78\xbe\x6b\x58\xd3\xf5\xe4\x12\xfd\xc5\x19\x16\xeb\x6f\x42\xd6\x2e\x4c\x9a\x35\x1e\x7b\xbe\xc6\xc8\x73\x6e\x65\x6d\x68\x57\x16\x96\x66\x48\xb5\xef\x50\xba\x2e\x5d\xdf\x62\xb9\x0c\xc7\x6e\x84\x98\x98\xac\x27\xb6\x9b\xaa\x00\x56\x3b\x71\xbe\x28\x43\x9c\x52\x4f\x5e\xf2\x84\x60\xc8\xd3\x35\xf2\xf6\xbc\x82\x4c\x86\xb5\x9a\x21\xcc\x57\x58\x3f\xcd\x1a\xf0\xd7\x87\x71\x55\x3e\xec\x91\x64\x1e\x84\xdd\x3c\x92\x5b\x96\x00\x72\x35\xcb\x0b\x06\x21\x3f\x04\x31\x70\xca\x34\xc3\x5b\xd9\x14\x6f\xe5\x37\xe4\x58\x0e\xc6\xa1\xa0\x9e\x8f\xda\x4d\x4e\x8b\xdb\x7f\x8c\x5d\xe2\xee\xa8\x5a\x6e\x25\xdc\x93\xfb\x73\x4f\x0e\x05\xc5\x83\xd4\x6e\x52\xd7\x10\x0e\xca\xca\x39\x28\xb7\x2e\xfa\x9f\xb9\x52\x97\xec\x6c\xf8\x41\xe1\x07\x95\xd0\xfa\x0d\x3f\x28\xfc\xa0\xf0\x83\x96\x3a\x04\xf0\x83\x6e\x33\x08\xf0\x83\xc2\x0f\x0a\x3f\x28\xfc\xa0\xf0\x83\x56\xd3\x14\x2e\x93\x1d\x1a\x7e\xd0\x52\xc5\x09\x3f\x68\xe3\x86\x1c\x7e\xd0\x12\x85\x09\x3f\x68\xc3\x06\x1c\x7e\x50\xf8\x41\xe1\x07\x95\xd0\x0f\x3a\x12\x55\x9d\xee\x25\x26\xf0\xda\xf9\x41\xcb\x2f\xb5\x07\x0f\x68\xa9\x1e\xd0\x91\xa8\x3c\x75\x0f\x29\x9a\xf0\x80\xd6\xa0\x34\x1f\x7c\x9f\xf0\x7d\x4a\x68\xf1\x86\xef\x13\xbe\x4f\xf8\x3e\x4b\x1d\x02\xf8\x3e\xb7\x19\x04\xf8\x3e\xe1\xfb\x84\xef\x13\xbe\x4f\xf8\x3e\xab\x69\xfe\x96\xc9\xf6\x0c\xdf\x67\xa9\xe2\x84\xef\xb3\x71\x43\x0e\xdf\x67\x89\xc2\x84\xef\xb3\x61\x03\x0e\xdf\x27\x7c\x9f\xf0\x7d\xca\xe7\xfb\xec\xb4\x44\xc7\x8c\xf5\x0e\x79\xe2\xee\x81\x7d\x9f\xfb\x38\x67\x01\xde\xcf\x32\xbd\x9f\x9d\x96\xe8\x5c\xb2\x5e\x72\x38\x2f\xbc\x9f\xf0\x7e\x56\xf8\x5c\x06\xf8\x3f\xe1\xff\x94\xd0\xea\x0d\xff\x27\xfc\x9f\xf0\x7f\x96\x3a\x04\xf0\x7f\x6e\x33\x08\xf0\x7f\xc2\xff\x09\xff\x27\xfc\x9f\xf0\x7f\x56\xd3\x04\x2e\x93\xfd\x19\xfe\xcf\x52\xc5\x09\xff\x67\xe3\x86\x1c\xfe\xcf\x12\x85\x09\xff\x67\xc3\x06\x1c\xfe\x4f\xf8\x3f\xe1\xff\x94\xd0\xff\xa9\x0a\xce\x99\x5f\xf3\x94\xf9\x3a\x1d\xb2\xf9\x9e\xce\x6c\x77\x81\x03\xe3\x1f\x9a\x31\x82\x03\xe3\x13\x7f\x39\xfc\x92\xd2\x1e\x0c\x5f\x95\x45\x29\xe5\xa1\xed\x09\x86\x9b\xb1\xad\x22\x82\x95\x17\x0b\x7f\x05\x78\x23\x9b\xa0\xb7\x53\x12\xff\xc6\x7f\x3d\xfb\x1e\x67\xb2\x97\x79\x26\x3b\xbf\xbd\xef\x06\x00\x71\x2c\x3b\x9b\xf2\x00\x79\x4d\x3c\x96\xbd\xa3\x0a\x8e\x65\x57\x7b\xdd\xf5\x30\x63\x29\x11\x73\xfd\xc3\xaa\xb4\x7d\x9c\x9a\x00\xbc\xb9\xde\x7c\x13\x1d\xd4\xde\xeb\x01\x71\xca\x8f\x38\xb7\x5d\xe0\xed\xcc\x95\xba\x9c\x78\x00\x60\x0b\x60\x0b\x60\x0b\x60\x9b\x7d\x13\x00\x5b\xee\xa1\x47\x01\xb6\x6d\xd1\x71\x60\xbd\x35\x8d\xa1\x55\x04\xb6\xe5\x96\xc1\x03\xa4\x5d\x6f\xa6\x89\x0e\xf7\xea\x0d\x00\x69\x01\x69\x2b\xb3\xee\x01\x66\x8f\x01\x66\x1d\x5b\x07\x94\x05\x94\x05\x94\x5d\xa7\x93\xcd\x84\xb2\x1d\x61\x45\xe7\x43\x9e\x6c\x7b\x60\x28\x5b\x76\x56\x33\xc0\xec\x7a\x73\x4d\x58\xa7\x19\x27\xd5\x02\xcc\x56\x69\xe5\x03\xce\xc2\x36\x0b\x40\x0b\x40\x9b\x7b\x13\x00\x5a\xee\xa1\x47\x01\xb4\x5d\x41\x99\x9e\xc1\x7a\x70\xb6\x86\x61\xaa\xa8\xb7\x53\x2a\x8a\xed\x0a\xea\xed\xa0\xda\x4e\x85\xab\xed\x54\x65\x6d\x1f\xab\x14\x0e\xa7\xdb\x50\x09\xa7\xa0\x5b\x85\xe8\x6d\x29\xd2\xa3\x27\xe1\x57\xb0\x18\x8e\x4e\x27\x21\x8a\xdb\x78\x0c\x50\x12\x47\xaa\x61\x68\x6a\x49\x10\x29\x0a\xe3\x44\x63\x81\xda\x38\xa5\x8d\xc3\xf6\x0b\x02\x15\x72\x4a\x1c\x06\x54\xc8\x41\x85\x1c\x54\xc8\x29\xb5\x42\xce\x56\xe6\xe3\x3d\x5b\x8f\x6b\x5d\xff\x26\x12\x73\x79\x02\x96\x54\xa2\xa5\x94\xc0\x39\xe2\xec\xdc\xa4\xc0\x4d\x53\xc6\xf4\x50\x35\x6e\x9a\x22\xcf\x52\xca\xdc\xc8\xbd\x46\x1a\x36\xa2\x52\xd4\xb1\xe1\x3d\x00\xbb\x39\x15\x51\xca\x86\xfd\x87\x52\x36\x75\x28\x65\xd3\x13\x94\xb2\x51\xfb\x89\x0d\x7b\xff\x41\x6f\x89\x47\xb2\x6a\xa9\x89\xf0\x30\xee\xd5\xc3\xd8\x13\xd4\xcd\x51\xfb\x9c\xef\x0d\x3e\xc6\xaa\xf9\x18\xb7\xdd\x27\x3a\x99\x2b\x75\xc9\x6f\x86\x2b\x13\xae\x4c\x29\x6d\xd5\x70\x65\x4a\x31\x0c\x70\x65\x4a\x32\x10\x70\x65\xca\x31\x0e\x70\x65\x4a\x31\x0c\x70\x65\xc2\x95\x09\x57\x26\x5c\x99\x70\x65\x36\xce\xed\x05\x57\x66\xfd\xc6\x14\xae\xcc\x72\xe5\x09\x57\x66\xdd\x46\x14\xae\x4c\xb8\x32\xe1\xca\x94\xd4\x95\x29\xaa\xb1\xdc\x4f\xd2\xee\x6b\xe7\xca\xdc\x4b\x29\x3a\x38\x31\xcb\x75\x62\x8a\x8a\x31\xf7\x91\x28\x09\x27\x66\x0d\x2a\xda\xc1\x7d\x09\xf7\xa5\x94\xf6\x69\xb8\x2f\xa5\x18\x06\xb8\x2f\x25\x19\x08\xb8\x2f\xe5\x18\x07\xb8\x2f\xa5\x18\x06\xb8\x2f\xe1\xbe\x84\xfb\x12\xee\x4b\xb8\x2f\x1b\xe7\xea\x82\xfb\xb2\x7e\x63\x0a\xf7\x65\xb9\xf2\x84\xfb\xb2\x6e\x23\x0a\xf7\x25\xdc\x97\x70\x5f\xca\xe9\xbe\xec\x8b\x4e\xd2\xea\x1f\xf2\x88\xd8\x03\xbb\x2f\xf7\x74\xfc\x00\x1c\x98\xa5\x3a\x30\xfb\xa2\xa3\xb7\xfa\x38\x4d\x16\x0e\xcc\x5a\x9c\x62\x00\x17\x26\x5c\x98\x52\xda\xa8\xe1\xc2\x94\x62\x18\xe0\xc2\x94\x64\x20\xe0\xc2\x94\x63\x1c\xe0\xc2\x94\x62\x18\xe0\xc2\x84\x0b\x13\x2e\x4c\xb8\x30\xe1\xc2\x6c\x9c\xbb\x0b\x2e\xcc\xfa\x8d\x29\x5c\x98\xe5\xca\x13\x2e\xcc\xba\x8d\x28\x5c\x98\x70\x61\xc2\x85\x29\xa7\x0b\x73\x20\x3c\x41\x3d\xfe\xdc\x98\x13\x27\x7f\xa2\xfe\x17\xdb\xbd\x6e\xd8\x49\xe8\xfd\xcd\xe7\x8c\xe8\x24\xf4\xc4\x13\x0d\xd7\x62\xda\xb5\xf8\x90\x4b\x71\x5d\xd7\x61\x3b\xdd\xbe\x89\xeb\xb0\x2a\xcb\x51\xca\xd3\xc9\x0d\x57\xf3\x29\x87\x2a\xad\x70\xab\x18\xbb\x74\x42\x8d\x9b\x08\x58\x8e\xd9\x7c\x2b\x07\xc9\x7d\xea\xcd\x3e\x3f\x3a\xf0\x09\xe5\x8e\xad\x17\x9c\x4f\xce\x99\x03\x4a\x06\x6c\x5b\x9e\x35\xfe\x21\x94\x39\x79\xa1\x59\x7a\xb8\x22\x77\x82\x73\x4d\x3d\x70\x9c\x33\xd4\xe5\xd0\x4a\x32\xc1\x5e\x38\x7b\x3d\x8a\x5c\x69\x29\x35\x84\x73\xfc\x10\xec\xe7\x30\xf2\x3a\xc3\x8b\xa1\xa8\x80\xfc\x10\xf0\x02\xf0\xa2\x51\xf0\x82\x19\x0d\x66\x86\x5f\x0f\x7c\x41\x1c\x5b\x27\x55\x40\x18\x1f\x23\xb1\x03\x62\x94\x65\x15\x02\x8c\x90\x12\x46\x1c\xc4\xc4\x34\x14\x86\x22\xf7\x13\x75\xb3\xf7\x28\x79\xce\xa0\x55\xb5\xb3\x46\x60\xa0\xda\x74\xc6\x09\xcb\xca\xc1\x42\x25\x31\x84\xdc\x7a\x69\x0f\x32\x97\xea\x72\xc6\x50\x95\xa0\x6a\xad\x2c\x61\x95\x41\xaa\xb0\x85\x95\x02\x54\x61\x0b\xab\x30\x88\xad\x33\x92\x19\x89\xdc\xb3\x7d\x18\xc3\xea\x88\x64\x86\x40\x32\x47\x47\x32\x30\xba\xc1\xe8\x56\x61\x2c\x03\xbc\x92\xee\xa6\x74\x78\xe5\x10\x46\xb7\x6e\x4b\x78\x48\xf8\x28\x51\x6b\xb5\x33\xba\x95\x5b\x5a\x1f\xe6\xb6\x35\xe7\x9a\xc8\x61\x3b\x68\x01\xa4\xd6\x10\xa4\xca\x64\x6e\x2b\xa7\x1a\x7e\x95\xe0\x29\x0c\x6d\xc7\x40\xa7\x30\xb4\x95\x02\x4e\x61\x68\xab\x30\x70\xad\x35\x86\x11\x39\xa9\x07\x2a\x30\x4c\x0d\x31\x8c\x4c\x86\xb6\xe6\x61\x18\x98\xd8\x60\x62\xab\x30\x8a\x01\x52\x49\x77\x53\x3a\xa4\x72\x10\x13\x9b\xda\x11\x41\x86\x43\x1e\x5e\x79\x68\x13\x5b\xd9\xe5\x5f\x61\x64\x5b\x73\xb6\x89\x3c\xc1\x03\x9c\x48\x59\x47\x80\x2a\x93\x91\xad\xac\x8a\xad\x55\x82\xa8\x30\xb3\x1d\x03\xa1\xc2\xcc\x56\x0a\x40\x85\x99\xad\xc2\xe0\xb5\xce\x28\xa6\x2d\x72\x4b\x0f\xba\x40\x31\x35\x44\x31\x32\x99\xd9\x9a\x88\x62\xea\x65\x68\xab\x48\x81\x0a\x98\xd9\xca\x41\x31\x40\x2a\xe9\x6e\x1e\x1f\xa9\x7c\x13\x3d\x36\x58\x73\xc1\x72\x52\xd8\x11\x36\xe1\xd8\x28\xde\xe4\x8a\xce\xb4\x5f\xa9\xeb\x45\x2a\x62\x18\x36\xfb\x8b\x70\x59\xe8\x9a\x7b\x1d\xde\xe9\x6b\xd3\x64\xd8\x95\xb0\x1a\x73\x24\xb8\xe5\x46\xac\xc4\x3f\xe5\xd3\x99\x63\x6a\xbe\x61\x25\xbb\xbf\x62\x1a\x9e\xcf\xcd\x9b\xbb\x22\x48\x70\x57\xa4\x0b\xce\x5d\x7b\x46\xfd\x2b\x3a\x17\xf9\x5b\xb8\x1b\x8a\x86\x5e\xb9\x32\xd8\x92\xe0\x81\x8d\x61\x4d\xcc\xb9\x4e\x9f\x9b\x45\x70\xa1\x78\x06\x28\xb3\xb9\xe9\x1b\x05\xb7\x47\x2b\x4b\x29\xc0\x65\x29\xfd\xcf\x97\x48\x56\xfe\x98\x53\x37\x40\x57\x8a\x53\xf4\x76\xdc\x58\xa9\xa9\xd6\x29\x0d\xe6\x1a\x5f\xb4\x51\xf1\xae\x0d\xe7\x17\xd7\xfc\x79\x61\x4d\x0a\x3a\xb7\xdc\x51\xb8\xce\x65\x97\x74\x6a\x8a\x99\xbf\x46\x52\xcd\xbc\xbc\x68\x8c\xa2\x59\xf1\xb9\x50\x8b\x67\x8b\x78\x0b\xc6\x2f\x77\x5b\xa9\xa3\xa8\xc4\xda\x52\xd9\x60\x30\x0b\xbf\xc4\x8d\x25\xf7\x22\xbc\x40\xd2\xf8\x26\x5b\x9b\x73\x5d\xd1\xac\x27\x9c\x64\xdb\xe0\x36\x0e\x7e\x6a\xad\xf8\x8d\x35\x67\xcd\x64\xee\xf9\xf6\xac\xdc\x19\x13\x09\xe0\x79\xb6\x04\x7b\xfc\xc6\xdf\x8d\xc7\x9a\x69\x16\x4f\x82\x87\xa9\x10\x43\xa0\x97\x86\x65\x44\xa0\x37\x9c\x07\xe3\x50\xa5\x25\x15\x5e\x0d\xeb\xd2\xe6\x40\x55\x01\xa6\x5a\xa2\xcb\xc7\x3f\x7c\x7f\x7f\x4a\x32\x78\xe9\xc1\xd9\x98\xd1\xa8\xf1\x64\x14\x96\x5e\x5c\x67\x83\x59\xf1\xdd\x87\xb6\x99\xfd\x48\xe1\x81\x6d\xea\xc9\xc9\xe3\x1f\x1e\x15\x41\xd1\x27\xeb\x4f\xc5\x08\x4f\xf1\xcf\xf7\xb5\x29\x9b\x73\xde\x3f\x96\x6f\xa7\xa4\xaf\xe6\xc4\x10\xb4\x15\xdf\x1c\x4d\xf4\x50\x4e\xdc\x85\xb9\x47\x3f\x86\x0f\x4a\x71\x46\xf6\x6f\x00\x8a\xee\x43\x3d\x67\xb0\x91\x89\x34\xdc\x65\x08\x49\x15\xcb\xfe\x72\xa6\x2e\xe1\x9b\xe2\xdb\x51\x9b\x92\xfa\x9a\x63\x4c\xae\x19\x6d\x8b\xbe\x1c\x89\x72\xbc\xc4\xd8\xfc\x2e\xa3\xf4\x12\xe5\x10\xeb\x6e\xf6\xa1\xc3\x7f\x50\x67\xc9\xdf\x3d\xee\x6f\x95\xff\xd0\x69\xf1\x57\x38\x90\xd9\xe6\xfe\x56\xf5\x70\xf9\x7d\x5e\xbe\x43\xc0\x08\xf2\xbb\x9f\xf8\x57\xf8\x07\xf7\xf9\x07\xf3\xbf\xd2\xee\xf2\x1f\x92\x53\x17\x94\x81\xce\xf7\x77\xd9\x97\x94\xf8\xbe\xda\x8c\x7e\x29\x11\x38\x59\xc2\xf8\x10\x97\x90\x27\xe4\x45\x04\x49\xd8\xe5\x9b\x04\xdf\x7c\x73\xff\xcd\xff\x07\x00\x00\xff\xff\x7f\x05\xdb\x96\xee\xd8\x03\x00") +var _monitoringBackendGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\x6b\x93\x9b\x46\xba\xfe\x9e\x5f\xd1\x87\x64\x37\xe3\x9c\x19\x5b\xe8\x2e\x57\xb9\x4e\xf9\xba\xf1\xae\x9d\x9d\x75\x9c\x9c\xcd\x71\x5c\x5a\x46\xf4\x68\xd8\x41\x40\x00\x8d\x47\x9e\x9a\xfd\xed\xa7\x68\x10\x34\x97\xd6\x15\x69\x1a\x78\xf2\x21\x1e\x35\x08\x35\x6f\x5f\xde\xe7\x79\x6f\x7d\xf7\x0d\x21\x8a\x66\x59\xb6\xaf\xf9\x86\x6d\x79\xca\x53\x12\x34\x11\xa2\x98\x86\xe7\x2b\x4f\xc9\x27\xf6\x89\x44\xad\xec\xca\xc5\xdc\x30\xfd\xb7\x96\xf2\x94\xa8\xa7\x49\xab\xae\xf9\x9a\x67\xcf\xdd\x09\x55\x9e\x12\xe5\xec\x8c\xfc\xc5\xd5\x2e\x35\x4b\x23\x67\x67\x0a\x77\x1b\xb5\xb4\x0b\x33\xb8\xc5\x77\xe7\x94\x6b\xbf\x32\xf4\x82\x56\x63\x62\x5b\x2f\x6d\xd3\x76\x83\x67\xba\xd3\x0b\xed\xa4\x75\x4a\xda\xaa\x7a\x4a\xda\xbd\xde\x29\x51\x1f\xf1\x8f\xb6\xb4\x19\xfb\xed\xe7\xc9\xeb\x90\x3f\x93\xe7\x26\x75\x7d\x8f\xbf\xcf\x5f\x38\xec\x3e\x5d\xf3\xae\x2e\x6c\xcd\xd5\x95\xe8\xda\x3d\xfb\xf7\xf3\x37\x84\xdc\x07\xb7\x2b\x54\x37\xfc\x4c\x6f\x95\xa9\x45\xfd\xb7\xba\xf2\x94\x58\x73\xd3\x0c\x5b\x5c\xcd\xb9\xfa\x68\xdb\xa6\x6f\x38\xca\x53\xd2\x62\x8d\x46\x70\xcb\x90\xfd\x69\x1a\xd6\x75\x20\xd7\x4f\x9f\xd9\x47\x47\xb3\xa8\xe9\xc5\x92\x5d\xca\x55\x99\xd8\xa6\xa9\x39\x1e\x0d\xbe\x78\xa9\x99\x5e\x2c\x06\x65\xea\x1a\xfa\xb9\x9d\x0c\x4d\x28\xaf\x8c\xf8\xbf\x28\x4f\x49\xbb\xcb\x35\xdc\x2e\xfb\x12\x7d\x5e\x04\x9f\x97\x2f\x1a\x3f\x9b\xf5\xb3\x1d\xdf\xc7\xf5\xee\x73\xdc\xe6\x1b\x3e\x93\x81\xf2\xce\xf0\x7c\x6a\x51\x37\x91\x66\x2c\x4b\xd7\xfe\x12\x4a\x31\x7a\x74\xfc\x5a\x9a\x69\x68\x1e\x1b\x42\xf6\x02\xc9\x2f\x5f\x68\xac\x25\xfd\xaa\xc1\x90\xbc\xa3\xd6\xd4\x67\xaf\xd7\x4a\xb5\xd3\xa2\xdb\xf9\x39\xf7\x1d\xf7\x31\xbe\xe5\xd2\x30\x4d\x5e\x54\x62\x69\x0e\x33\xd2\x54\xdb\x6b\xa4\xa9\x16\x4b\xb3\x33\x8a\x3f\x9b\x74\x4a\x2d\x3d\xfd\x53\xda\xcd\x34\xfb\x1e\xc1\xe8\xcf\x5d\x97\x5a\x7e\xc1\x95\x99\x76\x5b\xd4\x6a\x58\x05\xad\xde\x95\xfd\x25\xbf\x88\x7c\xdb\xd7\xcc\x82\xbb\x6f\x34\x73\x9e\x08\x35\xf7\x32\xa6\x61\xb1\xab\xfc\xd3\x58\xe3\x17\x43\xf7\x53\xd3\x2f\x33\xc5\x59\x53\xb0\x3c\xce\x6d\xc3\xf2\xdf\xdb\x6c\x61\xb3\x86\x64\x58\x6c\x27\xde\x6e\x92\x5f\x74\xa8\x3b\xa1\x96\xaf\x4d\x69\x6e\xa4\x9d\xe0\x51\xae\xa6\x1b\xf3\xe0\x3b\xed\x74\x7b\x7e\x62\xb8\xd4\xd2\xa9\x4b\xd9\xb6\x71\x69\xda\x7e\xf2\xc3\x1e\x75\x0d\xea\xfd\xfd\x86\xba\xae\xa1\xd3\x4c\xa7\x3d\x47\x9b\xd0\xa2\xf9\xe7\xf9\xda\xe4\x3a\xf7\x2b\x9e\x4f\x1d\x87\xea\xef\x0c\x2b\xdf\x61\x5f\x73\xa7\xd4\xf7\xb8\x1d\x94\xdf\x43\x83\xcd\xe5\xd6\x61\xdd\xf3\xe6\xb3\x13\x57\xf3\xe9\x89\xe6\x18\x9e\x6d\x69\xbe\xed\x8e\xcd\x68\xa1\x8d\x5d\xea\x39\xb6\xe5\xd1\xf1\xc4\xd6\xa9\x77\x17\xec\x70\xac\x8f\xcf\x7e\x57\xbe\x8b\x3f\xfc\xae\x9c\xba\xf4\x8f\x39\xf5\xfc\x71\xb0\x1c\x9f\xfd\xae\x68\x73\xff\xca\x76\x8d\xaf\xf4\x77\xe5\xfe\x93\x3a\xfb\xfc\x88\xdf\x27\x83\x45\x61\xbb\x33\x2d\x98\x6c\x8a\x6f\xcc\xe8\x38\x94\x49\xfa\x16\xc3\xf2\xa9\x7b\xc3\xe6\x8d\xa2\xce\x8a\xaf\xbd\xd1\x26\x3e\xdb\x9a\xd5\x56\xea\x7a\x38\xed\xdf\xc4\x3f\x12\x77\x27\xfd\x18\x97\x5e\xb2\x9d\x54\x79\xae\xc4\xcd\xf7\xa7\x0f\x23\x2d\x97\x3a\xf2\xc8\xca\xa5\x8e\x40\x52\x2f\x1e\x56\x52\x2e\x75\x6c\xd7\x97\x43\x50\x61\x5f\x04\x72\x7a\xf9\xf0\x33\x8a\x4d\xf8\xb1\x1d\xfc\x29\x87\xc0\x32\x9d\x12\x48\xee\xd5\xc3\x4b\xce\xa5\x8e\x6c\x72\x8b\xbb\x24\x90\xda\x6b\x4e\x6a\xd1\x5f\x1c\x7e\xba\x72\xa9\x77\x65\x9b\x7a\x0e\x57\xcd\xe8\x1b\xd7\x9e\x71\x60\x32\x6e\xff\x40\xa7\x91\x7e\xcc\x7c\xe1\xe7\x2b\xe3\xd2\xcf\x7f\x23\x83\xd0\x48\x24\x56\x8f\x38\xd4\x25\x1e\x9d\xd8\x96\x4e\x4e\x2e\x16\xcb\x76\x12\x88\xfb\x11\x07\xe3\x62\xf8\x7a\xc7\xa3\x09\xcd\x65\x70\x34\x83\x27\xbc\x60\xdd\x71\x4a\x78\x09\x25\xc6\x4b\x30\x68\x58\xba\x71\x63\xe8\x73\xcd\x54\x72\xa8\x62\x79\x0f\x43\xcd\x49\x07\x6e\xb5\x5b\x23\x83\xc9\x2e\xe6\x93\xeb\x50\x83\xf2\xef\x1a\x60\x9f\x08\x51\x04\xe2\x28\xc0\xff\x99\xbb\x8b\x31\x51\x8c\x7d\x3e\x7d\xce\x75\x71\xa1\xdd\xd2\x15\x8a\x5b\xa7\x13\x63\xa6\x31\x90\xdc\x12\xcc\x4b\x97\xfe\xe1\x64\x66\xa4\xa9\x5d\x50\x33\xd7\xbb\xe0\x82\x3d\x7d\xa1\x79\x34\x0d\xe7\x63\xdc\x97\xbb\x3d\x04\x7e\xe9\x1f\xe6\x5e\x71\xed\xe2\x4d\x3a\xe9\x5d\xe5\xf6\xcf\x72\x3b\x99\x6b\x2e\xec\x67\x6e\xb9\x2c\xf2\x53\x41\x33\x8d\x69\x11\xde\x65\xed\xef\xe8\x4d\xdc\xe9\x6f\xf8\x87\xd6\x98\x8a\xa4\x1a\x56\x70\x91\xae\x5a\x2a\x17\x09\x48\xfa\xeb\x99\xe3\x2f\x8a\xf9\xfb\xff\x51\xd7\xce\x5f\x01\x81\x01\x81\xd9\x00\x06\x78\x0e\xbb\xe7\xd9\xef\x4a\xfb\xf6\x56\x0e\xcd\xdf\xfe\xe7\x3f\xa5\x61\x2c\x89\x78\xba\xad\x8e\x1c\xe2\xe9\xb6\x3a\xd2\xd0\x14\x5e\x3c\x5d\x59\xc4\xd3\x95\x86\x9d\xf0\xe2\x19\xc9\x22\x9e\x91\x34\x14\x24\x11\x4f\x4f\x96\xbd\xa7\x27\xdc\x7b\x2a\xc4\x35\x42\xf9\x93\x40\xb6\x20\x1b\x20\x1b\x20\x1b\x95\x24\x1b\x59\xbf\xc7\x48\xc0\x35\x3a\xa5\x72\x0d\xd0\x06\xd0\x06\x3a\x0e\xb6\x5d\x6f\x1c\x2a\x15\x6f\x1c\xee\xd2\x7b\xb8\x41\xc8\xc5\x82\x9c\x98\x54\xa8\xda\xaf\xa8\xe6\xcf\x34\xe7\x70\x6a\xfd\xee\xee\x5f\x77\x77\xc4\xa4\xe4\xfe\xfe\x5f\xf7\xf7\x1b\xb0\x8b\x87\xd4\xf0\xcf\x97\xf2\x5b\xaf\xe2\x83\x87\x91\x70\x78\x88\x61\x45\xb7\x78\x50\xfa\x50\xfa\x4d\x52\xfa\x13\xcd\xd5\x33\x0f\x0e\x9a\xce\x35\x5d\x37\xac\x69\x7e\xe6\x04\x17\x3f\xd8\x73\x4b\xcf\x3c\x3c\xee\xe9\x24\x8a\x7d\xc9\x3c\x30\x0e\x89\xf9\xf6\xcd\xf3\x57\xaf\xdb\xcf\xf9\x39\xca\xbe\xf2\xf3\x44\x0b\x97\xb0\xf7\x47\x6a\x04\x96\x57\xaf\xe8\x2c\x5a\x47\x3e\x75\x1d\xdb\xd4\x7c\xfa\x77\x57\xb3\xa6\x29\x46\x13\x6c\xd5\xb6\x15\x6a\xe7\xd6\xe3\x5e\xc1\xfa\xb0\x1d\x6d\x62\xf8\x8b\xfc\x1a\x0c\x10\x49\xb2\xe9\xf9\xde\x72\xa5\x6d\x83\x60\x4a\xb4\x8e\xe6\x11\xcb\x72\xa3\x4f\xe9\xe1\xa5\x0d\xf3\x45\xbc\x2f\xa4\x75\xdb\x95\x31\xbd\x32\x8d\xe9\x95\xff\x32\x1a\xe7\x14\x44\x08\x41\x50\x6f\x25\x08\x8a\xe6\xa7\x10\x78\x64\xd1\x44\x21\x5c\x70\xe9\x0d\x75\x3d\xfa\x9b\xa8\x9b\x50\xc1\x87\x56\xc1\x2b\x54\xed\x3e\x1a\x35\x7a\xf5\x72\x34\x6b\x91\x4e\x0a\x5a\x7f\x34\x3c\xdf\x9e\xba\xda\x4c\x38\x0f\x97\x0a\x34\x3b\x12\xca\xed\xf3\xdc\xbe\x99\xdf\x70\x93\xe7\xdc\x86\x13\xf4\xa7\xf9\xec\x82\x21\xd2\x94\x48\xa2\x8b\x3f\x1b\x5f\xb3\x1a\x55\x59\xe4\x7f\xa6\x58\x23\xf2\xaa\x86\xdf\xb7\x0a\x75\x49\xa1\x26\xc9\x2b\x3b\x91\xe4\x1c\xd3\xf0\xe3\x19\x56\xb8\x57\x2f\xc2\x37\x7a\x11\xed\xe7\xc1\xd4\xb7\x95\xec\xd5\x62\x61\x2c\x72\xc2\x28\x54\x2f\x35\xe6\x94\xea\x20\x27\xcf\x70\x3f\x1d\x80\x54\x82\x54\xca\xa6\xd1\xf8\x68\xb1\x3a\xe8\xb3\xe3\x52\x4a\x97\x3a\x20\x94\x61\x3b\x08\x25\x08\x25\xff\xd0\x87\x27\x94\xbd\xc1\xa8\xfb\xa6\x0d\x42\x19\x37\x14\x87\xdb\xe4\xe1\xca\xa1\x18\x65\x6f\x75\x3a\x41\x23\x19\x65\xdd\xf4\xef\x5e\x7c\x52\xa4\x4e\xc1\x26\x09\xd8\x64\x33\xd9\x64\xbb\x97\x93\x67\xb8\x97\xb6\xc0\x26\xc1\x26\x65\xd2\x66\x99\x8c\x9a\x3a\x28\xb3\xe3\x91\xc9\x0f\x4c\x78\xe0\x92\x61\x3b\xb8\x24\xb8\x24\xff\xd0\x87\xe7\x92\x6f\xda\xdd\x51\xef\x25\xb8\x64\xdc\x50\xc8\x25\x0b\xc0\xca\xc1\xb8\xe4\x10\x5c\xb2\xe6\xda\x77\x67\x2a\xb9\x42\x99\x82\x49\x12\x30\xc9\x66\x32\xc9\x4e\x27\x27\x4f\xb6\x93\xf6\xc1\x24\xc1\x24\xa5\xd2\x65\xa2\x9a\x03\x75\x50\x6a\x0f\x11\xf2\x7a\xc2\xc4\xf8\x08\xec\x32\x6c\x07\xbb\x04\xbb\xe4\x1f\x2a\x01\xbb\x7c\x33\x1a\x76\x5a\x60\x97\x71\x43\x21\xbb\x2c\x00\x30\x87\x62\x97\xfd\xd5\xc5\x06\x1a\xc7\x2e\xeb\xac\x91\x4b\x88\x80\x5d\xa5\x60\xc1\x38\x09\x18\x67\x33\x19\x67\x57\x50\xca\xa5\xdf\x06\xe3\x04\xe3\x94\x4d\xbf\xe5\x6b\xb5\xd5\x41\xbb\x1d\x3f\x1e\x16\x6c\x13\x6c\x13\x6c\x53\x62\xb6\xf9\x62\x38\x18\xbc\x1a\x81\x6d\xc6\x0d\x85\x6c\xb3\x00\xbc\x1c\x8c\x6d\xae\x2e\x37\xd1\x48\xb6\x59\x4f\x6d\xbc\x77\x74\x2c\x98\x66\xea\x22\x98\xe6\x16\x4c\xf3\x18\xc7\x32\x74\x45\xe1\xaa\xbd\x24\x9b\x7c\xb3\x83\x19\xc8\xdb\x60\x79\x5a\x9a\x49\x9e\x9f\xbf\x55\x72\xb3\xab\x89\xa7\x34\x74\x05\xe5\x8a\x54\x15\xe7\x34\x80\x50\x97\xa4\xc2\x8d\x68\xd9\x8d\x35\xc7\xd8\xb9\xf4\x39\x3b\xa8\x46\x8e\xea\x83\x5a\xf6\xd0\x9c\x43\x14\x3f\x2d\x45\x68\x8e\x63\x1a\x13\x76\xd8\xcf\xf8\x9a\x2e\x64\x11\x5f\xa6\x57\x02\x41\x96\x55\x26\xb5\x6c\x41\xba\xf4\x92\xba\x2e\x75\xc7\x97\x86\xe9\x53\x57\x42\xa1\x66\x7b\x28\x10\x70\x59\x85\x56\x4b\x16\xb0\x7c\x02\x15\x09\xb0\xac\x52\xac\x65\x08\x90\xba\xae\x2d\xcb\x5c\x0c\xfb\x22\x10\xda\x6b\x99\x84\x76\x43\x2d\x59\x94\x4a\xd8\x17\x81\xd0\xde\x48\x24\xb4\x19\xf5\x5d\x63\x22\x89\xd4\xa2\xce\x08\xc4\xf6\x17\x89\xc4\xe6\x51\xf7\xc6\x98\xd0\xb1\x6f\x5f\x53\x59\xf6\xb8\x74\x9f\x04\x42\xfc\x51\x3e\x21\xca\x25\x3e\x91\xe0\xde\xca\x24\x38\x5f\x93\x65\xa3\x63\x5d\x11\x88\xec\xaf\x12\x89\x6c\xee\x69\x53\x3a\x36\x8d\x99\x21\x8b\xe4\xf8\x1e\x09\x04\xf8\x37\x99\x04\xe8\x1b\xa6\xf1\x95\x21\x28\x49\xe4\x97\x74\x48\x20\xbe\x77\x92\x38\x3e\xe3\xea\xf1\xbc\xc5\x0a\xc7\x56\x71\x6f\x01\x5f\x67\x61\x27\x1b\xe7\xeb\xac\x91\x6d\x36\xef\x2f\x14\x19\x67\xdb\xe5\xe6\xd7\xe0\xe0\xaa\xf8\x32\x2c\xba\x87\x84\x07\x8d\x3c\xc5\x6a\x6f\x59\x35\xe8\x48\xab\x12\x64\xd5\x98\xf3\xad\x4a\x90\x55\x63\x0e\xbb\xda\x5b\x56\x38\xf9\x8a\xbb\x50\x2e\x77\xc1\x31\x58\x04\xe4\x05\xe4\xa5\xe2\xe4\x25\x1b\x58\xd2\x13\x94\x2c\x57\x07\xab\xcf\x80\x40\x60\x09\x68\xc8\x9e\x6a\x7d\xbf\x40\xd1\x74\x9c\x49\x1d\x22\x44\x8f\x98\xaf\xc1\x84\x87\x34\x8d\xb0\x1d\xda\x1f\xda\x9f\x7f\xe8\xc3\xa7\x69\xe0\x3c\xac\x4d\xcc\xae\x05\xd0\xe5\x50\x69\x1a\xea\xa0\xbf\x12\x0d\x55\x28\x4f\x03\xba\xf8\xb0\xd9\x1a\x62\xd5\x8a\x24\x0d\x76\x09\x49\x1a\xcd\x2b\x07\xd0\x17\xe4\x86\xa8\x03\x9c\x8c\x05\x96\x29\xb1\x66\x13\x05\xe6\xd7\x41\xc7\x1d\x91\x6f\x26\x62\x24\x7f\xa3\x0b\x30\xcf\xa8\x1d\xcc\x13\xcc\x93\x7f\x28\x98\x67\x35\x98\x67\x01\x9c\x39\x1c\xf3\xac\x4d\xb5\x73\xe8\xe7\x58\xdc\x07\xe6\xa0\x9b\xa8\x5b\xb0\x51\x02\x36\xda\x4c\x36\x3a\x10\x94\x43\x57\x07\x48\xa6\x07\x1b\xad\x86\xb6\x13\x66\x37\xd7\x41\xf3\x1d\xf3\xf0\xad\x50\x8c\xe4\x4d\x28\x46\x30\xd3\xb0\x1d\xcc\x14\xcc\x94\x7f\x28\x98\x69\x35\x98\x69\x01\xb4\x39\x18\x33\x1d\xae\xce\x6e\x01\x33\x6d\x84\xae\xde\xe3\xa8\xae\x0d\x54\x2f\x58\x2a\x01\x4b\x6d\x26\x4b\x1d\x0a\x4a\xa8\xab\xc3\xd5\x27\x54\x80\xa5\x82\xa5\x4a\xa2\xf9\xea\xa5\xe9\x1e\xc4\x5f\x0a\x46\x1a\xb5\x83\x91\x82\x91\xf2\x0f\x05\x23\xad\x06\x23\x2d\x80\x31\x87\x63\xa4\xab\x4f\x97\x01\x23\xad\xa5\x5e\x2e\xc3\x4f\x0a\xf6\x09\xf6\x09\xf6\xb9\x7c\xfb\xe0\x0b\xa2\x9a\x36\xc3\xd5\x27\x56\x80\x7d\x82\x7d\x3e\xa4\x96\xcb\xd4\xd7\xad\x83\x7e\x3b\x1e\xef\x7c\x6b\xf9\x74\xea\x86\x81\x43\xaf\x99\x20\x37\xab\x09\x01\x06\x0a\x06\x9a\xeb\x24\x18\x28\x18\xa8\x1c\x0c\x34\x0f\x65\x0e\xc7\x40\xbb\x60\xa0\x4d\xd2\xcd\x3b\x73\xcf\x2d\x54\x2d\x58\x28\x01\x0b\x6d\x26\x0b\x1d\x89\xaa\x13\x0d\x51\x9d\x08\x2c\x54\x62\x4d\x97\x3e\xb0\xa4\x0e\x9a\xee\x78\x2c\xf4\x35\x13\x1e\x58\x67\xd8\x0e\xd6\x09\xd6\xc9\x3f\x14\xac\xb3\x1a\xac\xb3\x00\xba\x1c\x8e\x75\xa2\x3a\x51\xa3\x74\xf1\xce\xac\x73\x85\x6a\x05\xcb\x24\x60\x99\xcd\x64\x99\x6a\x4b\x54\x9e\x68\x88\xf2\x44\xa0\x99\xf2\xaa\xb6\xec\x11\x8f\x75\xd0\x6d\xc7\xe3\x99\xef\x43\xe9\x81\x68\x86\xed\x20\x9a\x20\x9a\xfc\x43\x41\x34\xab\x41\x34\x8b\xd0\xcb\xe1\x98\x26\xaa\x11\x35\x4b\x1d\xef\x4c\x35\x57\x69\x57\x70\x4d\x02\xae\xd9\x50\xae\xa9\x8a\x8a\x0f\x0d\x51\x7c\x08\x5c\x53\x5e\xe5\x26\x38\x17\xbf\x0e\x3a\xee\x78\x94\xf3\xe7\x50\x88\xe4\x23\x13\x22\x98\x67\xd8\x0e\xe6\x09\xe6\xc9\x3f\x14\xcc\xb3\x22\xcc\xb3\x00\xcb\x1c\x8c\x79\x8e\x50\x6d\xa8\x91\xca\x79\x67\x02\xba\x81\xae\x05\x0f\x25\xe0\xa1\x0d\xe5\xa1\x6d\x51\x79\xa1\x11\xca\x0b\x81\x87\x4a\xaf\xea\xea\xa5\xe4\x8e\xce\x40\xc1\x3d\xa3\x76\x70\x4f\x70\x4f\xfe\xa1\xe0\x9e\x15\xe1\x9e\x05\xf8\xe5\x70\xdc\x13\x75\x85\x1a\xa6\x90\xf7\x65\x9d\xe0\x9b\xe0\x9b\xe0\x9b\xcb\xb7\x67\x37\x88\x0a\x0a\x8d\x50\x50\x08\x7c\x53\x62\xf5\xe6\x6b\x35\xcb\x1e\x39\x22\xd9\x0c\x64\x07\xa6\x19\xb6\x83\x69\x82\x69\xf2\x0f\x05\xd3\xac\x0a\xd3\x3c\x62\xfd\xa0\x11\xea\x07\x35\x49\x15\xef\x4e\x33\x85\x9a\x15\x1c\x93\x80\x63\x36\x94\x63\x76\x44\xe5\x82\x46\x28\x17\x04\x8e\x29\xaf\x62\x9b\x7b\xda\x94\x8e\x4d\x63\x66\xd4\x4c\xbf\x1d\x8f\x6a\xfe\x12\x88\x90\xbc\x63\x22\x04\xe3\x0c\xdb\xc1\x38\xc1\x38\xf9\x87\x82\x71\x56\x84\x71\x16\xe0\x98\xc3\x31\x4e\xd4\x0e\x6a\xa0\x62\xde\x99\x78\xae\xd5\xb3\xe0\x9f\x04\xfc\xb3\xa1\xfc\xb3\x2b\xaa\x23\x34\x42\x1d\x21\xf0\x4f\x89\xd5\x9c\x6f\x98\xc6\x57\x56\x8c\xbc\x56\x5a\xee\x88\xf4\x33\x91\x20\xd8\x67\xd8\x0e\xf6\x09\xf6\xc9\x3f\x14\xec\xb3\x22\xec\xb3\x00\xc5\x1c\x8e\x7d\xa2\x9e\x50\xf3\xd4\xf2\xee\xe4\x73\x8d\x96\x05\xf7\x24\xe0\x9e\x2b\xb8\xe7\xc4\x36\x4d\xcd\xf1\x18\x8a\x4a\xef\x01\xc2\xfd\x53\xcd\xec\x9f\xed\xee\x3a\x12\xd8\x13\x14\xf8\xe9\x71\x0c\x45\xb3\xa8\x99\xc3\x98\xd1\x24\xff\x5f\xdb\xbd\xa6\x2e\xb7\xfd\x2f\xa7\x93\x6b\x7f\x51\x0a\x5f\xab\xba\x94\x7a\x03\x69\x76\x8b\xa5\xd9\x45\xb5\x24\x30\xea\xfd\x54\xf7\x17\xb6\xd0\xc6\xff\xb6\x2f\xc6\x13\x7b\x6e\x89\xf5\xf2\x3d\xe1\x74\x6f\xb0\x1a\x85\xda\x37\xd0\x35\xe3\xf0\xe5\x0f\xad\x81\x83\x7e\x54\x82\x1a\xff\xd5\xbe\xc8\x51\xe1\x7f\xdb\x17\x24\x2d\x48\x70\xde\x72\x39\xaf\x0d\xc6\x2b\x1d\xe3\xad\xae\x9e\x5e\x6f\xfa\xee\xb7\x05\xa8\x67\x75\x40\x2b\xf4\x34\xf4\xf4\x16\x7a\xda\x9d\x5b\x91\x8a\xdd\x8c\x50\x47\x44\xfa\x03\x75\x6c\xd7\xff\xab\x7d\x51\x2b\x1a\x7d\x3c\x15\x1e\xca\x2f\xd0\xda\x79\xa3\x76\x38\x22\x30\x67\x43\xb5\x37\x59\xb5\x1f\xdf\x98\xdd\x1b\x8c\xba\x6f\xda\x30\x66\xc7\x0d\xc5\xc6\xec\x02\x5c\x72\x28\x63\x76\x6f\x75\x90\xb9\xfc\xb6\x6c\x28\xda\x92\xed\xd5\x02\xbd\x99\x32\x53\xef\xae\x3f\x61\xa0\xae\x9d\x81\xba\xce\x0c\x71\xd0\xca\x09\x34\xdc\x36\x57\x07\xa0\x82\x21\x82\x21\x1e\x5e\x71\xfd\x64\xfb\xc6\xe5\xa2\x6e\x8a\xeb\x78\x0c\x31\x94\x1f\x18\x22\x18\x62\xfa\x65\xc0\x10\x97\x0f\x3e\x3e\x43\xbc\xe8\x5e\x5e\xb6\x5a\x60\x88\x71\x43\x31\x43\x2c\xc0\x25\x07\x63\x88\xab\xc3\xc0\x9b\xc0\x10\xeb\xa9\x68\x77\x66\x88\x02\xbd\x09\x86\x08\x86\xf8\x80\x21\x4c\x83\x61\xee\xcd\xc2\x70\xcd\xa4\x54\x5f\x51\x08\x93\x4b\x1d\x1a\x0e\x85\x4e\x1d\xd3\x5e\xcc\x02\xfd\x90\xa0\xff\x89\xed\x50\xfd\xd7\x90\xc8\xa6\xc6\x35\xbe\xf7\x69\x7a\x27\xf2\xa8\x49\x27\x7e\xfe\x55\xd9\x45\x9f\xde\xb2\x9f\xba\xd0\x26\xd7\xd4\xd2\xcf\x26\xae\x6d\xa5\xd7\x2e\xc3\x58\xb9\x5b\x72\xcb\xf8\x3e\xbf\x2a\xcf\x6d\xdd\x23\x27\xdf\x25\x3d\x7b\xb4\x45\x34\xd6\x44\x9b\x5c\xd1\x8f\xc6\x8c\xda\xf3\xdc\xba\x67\x2a\xf6\x85\x36\xb9\x9e\xba\xd1\x14\x49\x29\x09\x76\xf9\xd7\xa8\xdb\xe9\xd1\x9d\x2c\x8d\x02\xc9\xae\xad\x7c\xfb\xa6\xdd\x1d\xf5\x5e\xf2\xd3\xde\x9d\x5e\x68\x27\xed\xce\xe0\x94\xa8\xed\xd1\x29\xe9\xb6\x4e\x49\xeb\xf1\x70\xc4\x6f\xae\xca\xb7\xed\xd1\x68\xd2\xed\x2b\xb9\x6d\x6c\x03\xc5\x5b\xb4\x08\xb9\x25\x68\xd9\x16\xaf\xa5\xb5\x39\xa3\xa6\x77\xa9\x05\xb8\x7c\x3f\xb5\xd5\x4a\x2f\xc2\xe5\x85\x82\x95\x98\x85\x5e\x31\xb3\x79\x17\x60\xc6\x9c\x92\xe3\xef\x78\xaf\x85\xe1\x74\x82\x0d\x4a\xb8\x6a\x3a\x99\x55\xd3\x5f\xbb\x68\x0a\xca\x44\x19\x3a\x9b\x09\x4b\xfa\x5c\x08\x0a\x3a\x89\x20\x79\xad\xa6\xac\xd2\xfc\x33\xcd\x71\x0c\x6b\xfa\x31\x9c\x8b\x6a\x51\xfb\x0a\x0d\x1f\x11\x95\x70\x81\x10\xdf\x26\x6c\x2d\x15\xae\x1d\x35\x59\x2e\x02\x48\xbf\x7c\x18\x83\x88\xab\x1f\xd6\x5e\xa1\x42\x67\xda\xed\x2b\xcd\xd7\xce\x97\xf6\x09\x6e\x76\xe4\x6d\x23\x13\xdb\xb2\xc2\x9d\x21\x75\xcf\xc7\x70\x4b\x48\xad\xb8\x62\xc3\x89\x39\x9f\x1a\xd6\xaf\xd4\xf5\x0c\x3b\xd8\xfb\x95\xfe\xe3\xf6\xe3\x6e\xf2\x30\xc7\xf6\xfc\x4b\xe3\x36\x3d\x0c\x51\xe3\x1b\xdb\x5a\x6e\xd2\x4a\xaf\xf5\x27\xee\xba\x4b\xf3\xdf\x61\x6d\xc2\xaf\x30\x99\xbd\xd7\x9c\x15\x63\x75\x19\xc2\x8a\xb4\x39\x88\xf0\x1b\xe0\x4f\x4f\x9e\x67\x2e\xd8\xf1\x17\x56\x08\x5c\xfe\xed\xd8\x73\x34\xf7\xda\x0c\x8d\x45\x5c\x07\x2f\x0d\xd3\x8c\x49\x0e\xdb\xef\x3a\xea\x29\x51\xd5\xe1\x29\x3b\x43\x93\xb4\x1e\xab\xc3\xd4\x7e\x77\x39\x37\x8b\xcc\x79\xc1\x93\xf9\xe7\x84\x8f\x69\xb7\x4e\x89\x3a\xea\xa4\x1e\xb0\x12\x97\xfb\xda\x85\x19\x3c\x67\x3e\xb3\xd2\x63\xbf\x15\xd0\xbe\x9e\x5f\xd0\xb1\x4b\x1d\xd3\x98\xb0\xf0\xf6\x89\x6d\xf9\xae\x6d\x9a\xd4\x1d\x7b\xbe\xe6\xcf\xbd\xb1\x4b\x35\x7d\xb1\xbc\xc5\xe3\x70\xf6\xf7\x09\xcc\xfe\xfe\xb4\xf0\x11\xcf\xfe\xf3\x3d\xa7\xcb\x1e\xff\xf0\xfd\xfd\xee\x01\x8c\x09\xa8\x5e\x89\xa9\xf7\x36\x57\x29\xea\x69\x5b\x29\x02\xd8\x4a\xa7\xe5\x29\x85\x08\x3b\x7b\x65\xe9\x84\x99\x5b\x96\x61\x4d\x89\x63\xeb\x05\x41\xd5\x9e\x61\x4d\x4d\x1a\xc8\x38\xb9\xc6\x66\x2a\xbf\x68\x87\xfc\xa2\x65\x57\x57\x2f\x5a\x3b\x40\xe2\xca\xb3\xe2\xf5\xda\x12\x2c\x8b\x75\x0b\x96\xdd\xf8\x53\xb4\xdb\x6a\x37\xd3\xc3\xa0\x8f\xf3\xe5\x2e\x56\x00\x3f\xb6\x40\x26\x11\xc4\xd8\x16\x99\x44\x80\x06\xc8\x64\x1f\x64\xd2\x2f\x0d\x99\xb4\x8b\x90\x49\x6a\x4a\x01\x9b\x94\x8e\x4d\x80\x3d\x80\x3d\xa4\xc1\x1e\x0e\x9d\x94\x86\x39\xc8\x19\xa9\x03\xd4\x79\x50\x30\xf3\x8b\xa5\xdd\x68\x86\x19\x8c\x3b\x00\x0d\xcc\x29\xf5\x01\x2d\x05\x7e\x99\x5d\x51\xcb\x00\xf6\x14\xd8\x53\x80\x69\x1a\x8b\x69\x58\x32\xe7\xc9\xf2\xff\x96\xaf\x19\x16\x75\xc7\x33\x3a\xb3\xdd\xc5\x38\xac\xe8\x77\xb1\xf0\xa9\x10\x58\x38\xb6\x9e\x86\x11\x67\x9f\xb4\xb3\xaf\xad\xb3\xd1\xe7\xff\x4e\xfe\x0a\xf0\xcc\xc5\x82\x9c\x58\xb6\x4e\x1f\x35\xc9\x9c\xc2\x7c\x23\xba\xe1\xf9\xae\x71\x31\xf7\xa9\x4e\x6c\x8b\x5c\xd9\x9e\x0f\x28\x52\x26\x14\xd9\xd1\x7e\xa2\x77\xbb\x5a\x47\x03\x14\xd9\x0f\x8a\x0c\x4b\x83\x22\x7d\x18\x50\x8e\x0f\x46\x00\x36\x00\x36\x8e\x06\x36\x66\xda\xed\x89\x37\x9f\x9d\xe8\xd4\xf4\xb5\xd0\xb6\xe1\xd8\xfa\x38\xc1\x1d\xb1\x4d\xc3\xf3\x35\xd7\xf7\xc6\x2c\xba\x7c\x4f\xe8\xf1\xa9\x17\x07\x4e\x39\xb6\xde\x28\xf8\xf1\x5e\xbb\x65\x86\x0f\xb2\x14\x28\x39\x31\x35\xcf\x27\x3d\x32\x33\xac\xb9\x4f\xbd\x82\x50\x8d\xba\xe1\x90\xea\x64\x89\x0c\x32\xaa\x76\x7d\xe8\xd1\x50\x50\x47\x40\xe5\xb2\x27\x4a\xc8\x12\x09\xd4\xf7\xeb\x99\xe3\x2f\xf2\x61\x5e\xcb\x38\xcf\xfc\x95\x3a\xa5\x96\x10\xcd\x23\x5f\x83\xb7\x5c\xa3\x4c\x37\x4d\x31\xe9\xa5\xdb\xb7\x4a\x31\x91\x5f\x99\xc9\x98\x04\x73\x2c\xb3\xfd\xfe\xca\xa5\xbd\x52\xb9\xb0\x35\x71\x96\x32\x66\x73\xf7\xbd\x33\xac\xeb\xa2\x68\x7e\x4e\x09\xa5\xda\x03\xf1\xb2\x61\x58\x8b\x6d\xeb\x1c\x7e\xc1\x9c\x04\x05\x32\x8d\xa5\xf6\x42\x29\x4d\x3e\x70\x11\xad\x1d\x8e\xb9\xb5\x76\x40\x5e\x6e\x3e\x20\x35\xb4\x6f\xcd\x3d\xaa\x9f\xa5\xad\x48\x69\xf1\xbc\x92\x24\xcf\xed\xdc\xd6\x09\x93\x3c\x39\x61\xfb\xd6\x29\x61\x23\x7b\x4a\xe6\x56\xf0\xef\x23\xa2\x59\x7a\x88\x4f\xd9\xdb\x24\x86\x32\x83\x53\x42\x48\x77\x2b\x37\xdd\xed\xe0\xb9\x64\x95\x4e\x78\xe3\x21\x25\xca\x9d\x15\xd2\x94\xfe\xf6\x34\x25\x6f\x11\x64\x34\xa5\xdc\xaa\xa4\x8d\x67\x29\x65\xb1\x93\x3d\x12\xe0\xc3\xa4\x16\xaa\xbf\x58\x7c\xc8\x1b\xac\xc1\x5d\x76\xe3\x2e\x0f\x69\x2d\x3b\x34\x96\x09\xd3\x09\x1d\x5b\xaf\x44\xe2\xfe\xf9\xa6\xb6\xb4\x1d\x20\x4b\xab\xa1\x90\x05\xc8\x24\xdd\x49\xe9\x52\xf1\x8f\x91\x43\x39\x12\x9d\x05\xd6\x4e\xd4\xe0\xca\x24\xca\xd4\x7a\x0d\x5b\xdf\xfa\xd4\x65\x6c\x96\x55\x36\x6b\xb5\x06\x23\x55\x1d\x74\x78\x6f\x5a\x78\xdf\x79\xf0\xdc\xb7\x99\x8c\xcd\x03\xab\xaa\xe5\x41\x18\x6b\xd4\x55\x7c\xdb\x0a\x95\x85\x3c\x4c\x78\xeb\x09\xd9\x35\x0f\x73\xd4\xcf\x3d\x7c\x23\x67\xbd\xda\x2e\x4c\x77\x40\xe0\x20\x02\x07\xc9\x21\x7d\xf9\xbb\x6e\xed\x9d\xec\xa5\x84\xa3\xa4\x26\x77\x75\xf6\xfd\xa6\xc4\x0c\xd4\xd1\xc4\x2d\x51\x88\x00\x12\x3e\xb7\x81\x39\x48\xf8\xac\x3c\x04\xca\x25\x7c\xee\x0e\x81\x3a\x45\x10\x08\x01\x8b\x08\x58\x7c\x28\x90\xd3\x06\xc8\xa9\x27\xc8\x41\xd8\x00\x32\x4b\xab\x8d\x9c\x60\x20\xaa\x04\x3a\xca\x67\x96\xee\x0e\x8f\xba\x45\xf0\x08\x16\x22\x58\x88\xe4\x04\x4f\x03\x80\xa7\x6a\x82\xa7\x1a\x86\xf8\x49\x64\x20\x42\x0a\x2b\x52\x58\x6b\x8e\x79\x72\x29\xac\xbb\x63\x9e\x1e\x4c\x42\xc7\x47\x3d\x40\x35\x62\x54\xd3\x07\xaa\xa9\x26\xaa\x41\xae\xac\x60\x29\x20\x57\x36\x69\x46\xae\xec\xe6\x21\x66\x82\x20\x74\xb5\x5d\xee\x91\x6a\x8d\x0f\x43\x97\x28\x59\x76\xe7\x40\x40\xb5\x76\x5a\x53\xc6\xc0\x76\x24\xe5\x22\x29\x57\x24\x53\x24\xe5\x4a\x35\x1c\x48\xca\x45\x52\x2e\x92\x72\x91\x94\x8b\xa4\x5c\x22\x25\x1f\xda\x3a\x29\xb7\xdd\xca\xdb\x38\x23\x3e\xb4\xfa\xdc\x45\xf0\xa1\xa4\xb5\x5a\x69\xb9\xdb\xf3\xa0\x11\x68\xd0\x11\x68\x10\xf2\x7b\x91\xdf\x5b\x5f\xf4\x23\x01\x7e\xa8\x0a\xce\xa9\x73\x8a\x6f\x5b\x15\xd5\x2a\x6c\x0f\x13\xc5\x56\xa3\x14\x5f\x97\xea\x86\xb7\x46\x65\x85\xf7\xac\xd0\x57\x48\xee\x45\x1c\x03\x21\x3b\x26\xf7\xb6\xd5\x4e\xee\xe1\x1b\x86\x31\x24\xc0\x0f\xa1\x9b\x08\xdd\x94\x3e\xc8\x41\x8e\xe4\xde\x52\x76\xfc\xa6\x84\x37\xd4\xd1\x48\x2e\x51\x34\x03\xd2\x7a\xb7\x01\x38\x48\xeb\xad\x3c\xf8\xc9\xa6\xf5\xee\x0e\x7e\x8a\x8f\x98\x47\x0c\x27\x62\x38\x9b\x9d\xd6\x0b\x78\x23\x69\x78\x4b\x4d\x42\x0e\x90\xd0\x9b\x59\x78\x72\x63\x26\x18\x85\x2a\x81\x8b\x72\x09\xbd\x7b\x00\x23\xb5\x08\x18\xc1\x2a\x04\xab\x90\x9c\xb0\x49\x8e\x84\x5e\xc0\x26\xa4\xf2\x26\x57\x91\xca\x5b\x7b\xb4\x83\x54\x5e\x89\x52\x79\xf7\x40\x3b\x85\x05\x6e\x61\x06\x82\x19\xa8\xd9\xa9\xbc\xc0\x33\x48\xe2\x95\x16\xe1\x20\x89\xb7\x4a\x41\xeb\x5b\x27\xf1\xb6\x55\x51\xd0\x7a\x27\x89\xeb\x42\xd0\x7a\xdc\x88\x24\x5e\xfe\x52\x95\xf5\xa5\x8c\x71\xeb\x48\xdf\x45\xfa\xae\x48\xa6\x48\xdf\x95\x6a\x38\x90\xbe\x8b\xf4\x5d\xa4\xef\x22\x7d\x17\xe9\xbb\x44\x4a\x26\xb4\x7d\xfa\x6e\x3b\x6f\xd7\x8c\x98\x50\x17\x4c\x08\xe9\xbb\x12\xa5\xef\xd6\x97\x00\x21\x71\x17\x89\xbb\xf5\xc5\x3d\x12\x20\x87\xaa\x20\x9c\x5a\x27\xee\xb6\x45\x85\x13\x3b\x9c\x51\xaf\x46\x89\xbb\x5f\x6c\xf7\x7a\x6d\xb1\x89\xe8\xa6\x15\x1a\x0b\xa9\xbb\x88\x5b\x20\x64\xd7\xd4\xdd\x4e\x2b\xf7\xf0\x0d\xc3\x16\x12\x2f\x31\x82\x34\x11\xa4\x29\x7d\x50\x83\x1c\xa9\xbb\x25\xed\xf9\x4d\x09\x6b\xa8\xa3\x89\x5c\xa2\x28\x06\x24\xef\x6e\x03\x71\x90\xbc\x5b\x79\xf8\x93\x4b\xde\xdd\x1d\xfe\x0c\x8a\xe0\x0f\xa2\x36\x11\xb5\xd9\xec\xe4\x5d\x00\x9c\x52\x01\x0e\x42\x0e\x90\xbe\x5b\x6d\xd4\x04\xc3\x50\x25\x90\x51\x3e\x7d\x77\x77\x68\x34\x2c\x82\x46\xb0\x0c\xc1\x32\x24\x27\x70\x92\x23\x7d\x17\xc0\x69\x2b\xe0\x54\xc3\xd0\x40\x89\x0c\x43\x48\xe0\x45\x02\x6f\xcd\xf1\x4e\x2e\x81\x77\x77\xbc\x53\x58\xc4\x16\xa6\x20\x98\x82\x9a\x9d\xc0\x0b\x44\xb3\x15\xa2\x41\x0a\xaf\x60\x19\x20\x85\x37\x69\x46\x0a\xef\xc6\xe1\x64\x1d\x51\xe0\x7a\x37\xe9\x39\x02\xd7\xe3\x46\xa4\xf0\xf2\x97\xaa\xad\x31\x65\x8c\x61\x47\x12\x2f\x92\x78\x45\x32\x45\x12\xaf\x54\xc3\x81\x24\x5e\x24\xf1\x22\x89\x17\x49\xbc\x48\xe2\x25\x52\x72\xa1\xed\x93\x78\xbb\x79\xdb\x66\xc4\x85\x54\x70\x21\x24\xf1\x4a\x94\xc4\x5b\x67\x0a\x54\xfb\x34\x5e\xc7\xd6\x91\xc4\xdb\x4c\xdc\x23\x01\x72\xa8\x0a\xc2\xa9\x75\x12\x6f\x57\x50\x38\x31\x29\x16\xb2\x32\x85\x97\x57\x23\xc7\x52\x39\x13\x97\x23\x3b\xa4\x50\xe1\xb0\x5b\x56\xa8\x9b\x78\x6b\x79\x79\xfe\x0b\xf9\x25\xe0\xb4\x3b\x27\xe2\x56\x07\x86\x6e\x6f\x92\xef\x0e\x8a\x27\x47\x3f\x99\x1d\x82\x88\xc1\x12\xb0\x69\x9d\x60\xa6\x44\x26\x77\xf9\x17\xa7\x94\x58\xd0\xb2\x75\x3a\x8e\x51\x5d\x1a\x0f\x3e\x4d\x90\xe1\xc4\x99\x47\x36\x32\x8f\x4e\x6c\x4b\x8f\xb0\xe1\xd3\xbb\x3b\xf2\xf8\xe7\xf9\xec\x83\xe6\x53\x72\x7f\xcf\x41\xc5\xff\xec\x6e\x3b\x2b\x07\x26\xae\xb6\x9e\xaf\x80\x89\xa5\x19\xd0\x1f\x12\x5a\xc6\xfb\x3f\xac\x67\x92\xa3\x48\x18\xc9\xb8\x76\xb9\x30\x64\xaf\x5b\x0c\x13\xd4\x6e\x7b\x33\x14\x59\x4a\x21\x98\xee\xb1\x54\x9c\x69\x78\x3e\xb5\xd6\x1a\x3d\xe2\xdb\x80\x43\xf7\xc5\xa1\xbd\x9e\x68\x82\x75\x00\x44\x49\x45\x81\xe8\xae\xeb\x9c\x23\x1f\x0f\x69\xf8\x2c\x71\x17\x00\xe0\x95\x0c\xf0\x8a\xca\x1b\x02\xf2\x02\xf2\x02\xf2\xa6\x3b\xd9\x4c\xc8\xdb\x6f\x8b\x10\xc9\x86\x86\xd3\x6a\x41\xde\x52\x4a\xf5\x02\xec\x6e\x36\xb5\x84\x71\xd0\x3d\x80\x5d\x02\xb0\x8b\x52\xdd\x80\xb9\x80\xb9\x80\xb9\x80\xb9\x80\xb9\x87\x86\xb9\x03\x61\x1c\x62\xbf\x8e\x30\xb7\xec\x0a\xdf\x00\xba\x2b\x27\x97\x2a\x9a\x5c\x85\xb5\x1a\x01\x74\x01\x74\xab\xb2\x03\x00\xea\x02\xea\x02\xea\x02\xea\x72\xcd\x80\xba\x92\x43\xdd\x61\x31\x1a\xd9\xf0\x2c\x9b\x3a\x04\xc2\xfe\x63\x6e\xfb\xda\x91\xa0\xea\x84\x95\x5e\xc9\x48\xf3\x08\xf8\xf5\x92\x2b\xfc\xa1\xb6\xf8\xca\x1f\x65\x22\x5b\xd1\xc9\x48\x40\xb0\x7b\x22\x58\x6d\x4a\xa3\xd1\x4b\xa9\x42\x04\xce\x86\x3d\x74\x6d\x36\xdd\xd3\x88\x79\x25\x16\xbd\xb2\xbf\xfc\x48\x35\x9d\xbd\x70\xfa\x6b\x21\x28\xe0\x5e\x74\x62\x9b\x99\x9d\x55\xa7\xde\x44\x38\x4b\x4a\x84\xb9\x9e\xbf\x30\x57\x69\x70\xb6\xf5\x04\x92\xfa\x98\x46\x10\xe1\x1e\x41\x13\x2c\xf8\xdb\x6f\xbf\xfd\x76\xf6\xfe\xfd\xd9\xab\x57\xe4\xc7\x1f\x9f\xce\x66\x4f\xbd\x0c\xb6\x74\x34\xdf\xa7\xae\x55\xfc\xac\xe5\x16\x78\x65\xe8\x3a\xb5\xd6\x67\xd4\xc7\xdd\xca\x03\xb4\xa5\x40\x6d\x37\x5a\x05\x39\xa5\x9b\x54\x20\xfc\xbc\xcf\x0b\x71\xf9\xcd\x19\x98\x1c\x62\xde\xfc\xc4\x0c\x2e\x7c\x8c\xe1\xa3\xf2\xca\x35\x4c\x93\xe8\xf6\x97\xcc\xec\x0c\x6e\xfb\xc5\x4d\x13\xf3\xac\x08\x59\x89\x41\xf2\x6d\xb6\x98\x5a\x31\x32\x4e\x89\xd8\x9a\xcf\x2e\xb2\x3c\x6c\x6e\x19\x1c\x00\xda\x4e\xfa\x1f\xe8\x1f\x73\x9a\xcb\xff\x6f\xca\x00\xbc\x90\x67\x00\xc8\x9f\x9a\x39\x04\x2f\xcb\x1d\x82\x48\xe1\xb1\x8f\xdb\x0d\xc4\x3b\x63\x66\x34\x75\x1d\xbc\x7a\xf8\x75\x10\x8a\xbf\xa9\xab\xe0\xb5\x0c\xab\xe0\xdc\xd6\xe5\x93\x7e\x1a\x99\xef\x24\xfc\x27\xfa\x93\xbb\x3b\xf2\xf8\xa7\xa5\x2d\x8e\xdc\xdf\xe7\x1a\xce\x3a\xde\x44\x33\xe9\xd9\xf5\xfc\x82\xba\x16\xf5\xa9\x77\x36\xb1\x67\xce\xdc\xa7\x67\x2e\x0d\xc9\x93\x77\xe6\xd8\xfa\xff\xdc\x68\xee\x59\x62\xe2\x4b\x0c\x7c\x7f\x0e\x2e\x38\xb6\xfe\xec\xbb\xf1\x78\x42\xb3\x05\x54\xb9\x11\x77\xb2\x52\x3e\xf6\x6a\x93\x68\x8c\x39\xb1\x3c\x79\xfc\xc3\x93\xed\xe5\xe2\xf9\xae\x61\x4d\x37\x93\x4b\xf4\x17\x67\x40\xac\xb3\x91\x58\xbb\x30\x69\xd6\x3c\xec\xf9\x1a\x23\x63\xb9\x35\xb5\xa5\xe5\x58\x58\x16\x21\xd5\xbe\x47\x49\xb8\x74\x55\x89\xe5\x02\x1c\xbb\x11\x56\x62\x52\x9e\xd8\x6e\xaa\xd6\x56\x8d\x04\xf9\xa2\x0c\x41\x4a\x3a\x61\xc9\x13\x82\x61\x5e\xd6\x9e\x3b\xf0\x7a\x31\x19\xa6\xaa\xbb\x18\x5f\x35\x7b\xb5\x34\x63\x90\x5f\x1f\xc7\xd1\xb8\xde\x9f\xc8\x7c\x01\xfb\xf9\x13\x77\x2c\xad\xe3\x6a\x96\x17\x0c\x42\x7e\x08\x62\x38\x94\x69\x86\xaf\xb1\x29\xbe\xc6\x6f\xc8\x43\xb9\x07\x87\x82\x3a\x39\x6a\x37\x39\x3d\xed\xf0\x91\x70\x89\xfb\xa2\x3a\xd9\x8d\x70\x31\xae\x9f\x8f\xdb\xbb\x18\x87\x82\xd2\x3c\x6a\x37\xa9\x0b\x08\x27\x63\xe5\x9c\x8c\x3b\x17\xc4\xcf\x5c\xa9\x7e\x4e\x34\x7c\x99\xf0\x65\x4a\x68\xc1\x86\x2f\x13\xbe\x4c\xf8\x32\x4b\x1d\x02\xf8\x32\x77\x19\x04\xf8\x32\xe1\xcb\x84\x2f\x13\xbe\x4c\xf8\x32\xab\x66\xec\x96\xc9\xde\x0c\x5f\x26\x7c\x99\xf0\x65\xc2\x97\x09\x5f\x26\x7c\x99\x65\x0f\x32\x7c\x99\xf0\x65\xc2\x97\x29\xa1\x2f\x73\x24\xaa\xd7\xdc\x4b\x0c\xdb\x35\xf2\x65\x96\x5f\xbc\x0e\x5e\xcc\x92\xbc\x98\x23\x51\x61\xe7\x1e\x52\x25\xe1\xc5\xac\x74\xb1\x3b\xf8\x2f\xe1\xbf\x94\xd0\x6a\x0d\xff\x25\xfc\x97\xf0\x5f\x96\x3a\x04\xf0\x5f\xee\x32\x08\xf0\x5f\xc2\x7f\x09\xff\x25\xfc\x97\xf0\x5f\x56\xcd\xc0\x2d\x93\x8d\x19\xfe\x4b\xf8\x2f\xe1\xbf\x84\xff\x12\xfe\x4b\xf8\x2f\xcb\x1e\x64\xf8\x2f\xe1\xbf\x84\xff\x52\x3e\xff\x65\xa7\x25\x3a\x7c\xab\x77\xcc\xf3\x66\x8f\xe6\xbf\x3c\xc4\xa9\x04\xf0\x60\x96\xe3\xc1\xec\xb4\x44\xa7\x75\xf5\x92\xa3\x69\xe1\xc1\x84\x07\xb3\x92\xa7\x18\xc0\x87\x09\x1f\xa6\x84\x96\x6b\xf8\x30\xe1\xc3\x84\x0f\xb3\xd4\x21\x80\x0f\x73\x97\x41\x80\x0f\x13\x3e\x4c\xf8\x30\xe1\xc3\x84\x0f\xb3\x6a\x46\x6e\x99\xec\xcc\xf0\x61\xc2\x87\x09\x1f\x26\x7c\x98\xf0\x61\xc2\x87\x59\xf6\x20\xc3\x87\x09\x1f\x26\x7c\x98\x12\xfa\x30\x55\xc1\xc9\xea\x1b\x9e\xab\x5e\xf5\xe3\x26\xdf\xd3\x99\xed\x2e\x70\x38\xba\x78\x7e\x08\x0e\x47\x4f\x3c\xdc\xf0\x2a\x4a\x7b\x08\xba\xfc\x0b\x51\xca\x03\xca\x13\x94\x36\x63\xdb\x43\x04\x19\x2f\x16\xfe\x0a\x78\x46\x36\xc3\x67\xa7\x24\x7e\xfa\x7f\x3d\xfb\x1e\xe7\x8f\x97\x79\xfe\x38\xbf\x99\xef\x07\xf1\x70\x04\x39\x9b\xec\x80\x71\x4d\x3c\x82\xbc\xa3\x0a\x8e\x20\x57\x7b\xdd\xcd\x50\x61\x29\x71\x6d\xfd\x63\xa9\xb1\x43\x9c\x31\x00\x5c\xb9\x6e\x8e\x89\x8e\x26\xef\xf5\x80\x2c\xe5\x47\x96\xbb\x2e\xea\x76\xe6\x4a\xf5\xcf\x07\x00\x80\x05\x80\x05\x80\x8d\x2f\x00\xc0\x02\xc0\xf2\x0f\x7d\x10\x00\xdb\x16\x1d\x92\xd5\xdb\xd0\xac\x59\x2d\x00\x5b\x6e\x61\x39\x40\xd7\x75\xb3\x4b\x74\xe4\x55\x6f\x00\xe8\x0a\xe8\x5a\x81\xb5\x0e\xd0\x7a\x5c\xd0\xea\xd8\x3a\x20\x2b\x20\x2b\x20\xeb\x26\x9d\x6c\x26\x64\xed\x08\x6b\x21\x1f\xf3\x5c\xd7\xa3\x41\xd6\xb2\x73\x89\x01\x5a\xd7\xcd\x2f\x61\x85\x63\x9c\xd3\x0a\xd0\x5a\x8d\xd5\x0e\xd8\x0a\x5b\x2b\x80\x2b\x80\x2b\x80\xab\x44\xc0\xb5\x2b\x28\x82\x33\xd8\x0c\xb6\xd6\x24\x80\x14\x75\x6c\x4a\xc2\xa9\x5d\x41\x1d\x1b\x54\xb1\xa9\x70\x15\x1b\xf9\xd7\xf3\x43\x95\x98\xe1\x74\x18\x2a\xcc\x14\x74\xab\x10\xa5\x2d\x45\xfa\xe0\xc9\xed\x15\x2c\x32\xa3\xd3\x49\x88\xd6\xb6\x1e\x03\x94\x9a\x91\x6a\x18\x9a\x5a\x6a\x43\x8a\x82\x33\xd1\x58\xa0\xe6\x4c\x69\xe3\xb0\xfb\x82\x40\xe5\x99\x12\x87\x01\x95\x67\x50\x79\x06\x95\x67\x4a\xad\x3c\xb3\x93\x81\xf8\x60\xf6\xe1\x5a\xd7\x95\x89\x04\x5c\x86\x68\x25\x95\x65\x29\xa5\x65\x1e\x64\x46\x6e\x53\x38\xa6\xfe\xe3\x78\xac\xda\x31\xf5\x97\x64\x29\xe5\x63\x64\x5d\x11\x8d\x19\x45\x29\xea\xc3\xf0\xb6\xfc\xfd\x5c\x82\x28\x11\xc3\xfe\x43\x89\x98\x3a\x94\x88\xe9\x09\x4a\xc4\xa8\xfd\x63\x1e\xd3\x9f\xf8\x13\xab\x93\x1c\x08\x2f\xe1\x81\xbc\x84\x3d\x41\x55\x1a\xb5\x8f\xf3\xfa\x2b\xec\x27\xdc\x75\x6f\xe8\x64\xae\x54\x3f\xab\x18\xee\x48\xb8\x23\xa5\xb4\x37\xc3\x1d\x29\xc5\x30\xc0\x1d\x29\xc9\x40\xc0\x1d\x29\xc7\x38\xc0\x1d\x29\xc5\x30\xc0\x1d\x09\x77\x24\xdc\x91\x70\x47\xc2\x1d\xd9\x08\xb3\x3d\xdc\x91\xf5\x18\x47\xb8\x23\xe1\x8e\x84\x3b\x12\xee\x48\xb8\x23\xa3\x69\x02\x77\xa4\xbc\xee\x48\x51\x75\xe2\xfe\x31\x4f\xdd\x3f\x9a\x3b\xf2\x20\xc5\xdd\xe0\x88\x2c\xcb\x11\x29\x2a\x63\xdc\x47\xc2\x22\x1c\x91\x95\xae\x11\x07\x17\x24\x5c\x90\x52\xda\x98\xe1\x82\x94\x62\x18\xe0\x82\x94\x64\x20\xe0\x82\x94\x63\x1c\xe0\x82\x94\x62\x18\xe0\x82\x84\x0b\x12\x2e\x48\xb8\x20\xe1\x82\x6c\x84\xa9\x1e\x2e\xc8\x7a\x8c\x23\x5c\x90\x70\x41\xc2\x05\x09\x17\x24\x5c\x90\xd1\x34\x81\x0b\x52\x5a\x17\x64\x5f\x74\xbe\x54\xff\x98\x07\xa4\x1e\xcd\x05\x79\xa0\x62\xfd\x70\x42\x96\xe4\x84\xec\x8b\x0e\xa4\xea\xe3\x2c\x55\x38\x21\x2b\x5e\xf3\x1f\x6e\x48\xb8\x21\xa5\xb4\x33\xc3\x0d\x29\xc5\x30\xc0\x0d\x29\xc9\x40\xc0\x0d\x29\xc7\x38\xc0\x0d\x29\xc5\x30\xc0\x0d\x09\x37\x24\xdc\x90\x70\x43\xc2\x0d\xd9\x08\x73\x3d\xdc\x90\xf5\x18\x47\xb8\x21\xe1\x86\x84\x1b\x12\x6e\x48\xb8\x21\xa3\x69\x02\x37\xa4\xb4\x6e\xc8\x81\xf0\xcc\xf0\xf8\x73\xad\xcf\x5e\xfc\x89\xfa\x5f\x6c\xf7\xba\x31\xa7\x7e\xf7\xb7\x9f\x21\xa2\x53\xbf\x13\xdf\x31\x1c\x83\x69\xc7\xe0\x3a\x87\xe0\xa6\x8e\xbf\x76\xba\x7d\x1b\xc7\x9f\xfc\x4b\x50\xca\x93\xb8\x0d\x57\xf3\x29\x87\x18\xad\x70\x7b\x18\xbb\x74\x42\x8d\x9b\x08\x34\x8e\xd9\x4c\xdb\x17\xab\x7d\xea\xcd\x3e\x3f\x3a\xf2\x69\xdc\x8e\xad\x17\x9c\xc5\xcd\x91\xf9\x92\x21\xd9\x8e\xe7\x6a\x7f\x08\xa5\x4d\x5e\x68\x96\x1e\xae\xc2\xbd\x00\x5b\x53\x0f\xd7\xe6\x0c\x6c\x39\x3c\x92\x4c\xb0\x17\xce\x41\x8f\xdd\x56\x5a\x4a\x0d\x01\x1b\x3f\x04\x87\x39\x78\xbb\xce\x90\x62\x28\x2a\xbd\x3e\x04\xa4\x00\xa4\x48\x4b\xb9\xb6\x90\x82\x19\x04\x66\x86\x5f\x75\x4c\x41\x1c\x5b\x27\x55\x40\x15\x1f\x23\x81\x03\x56\x94\x65\xeb\x01\x74\x90\x12\x3a\x1c\xc5\x70\x34\x14\x06\x0c\xf7\x13\x15\x73\xf0\xf8\x75\xce\x4c\x55\x9d\x93\x39\x60\x7c\xda\x7c\x96\x09\x0b\xb5\xc1\xfa\x24\x31\x54\xdc\x79\x39\x0f\x32\x97\xaa\x7f\x0a\x4f\x95\x20\x69\x4d\xac\x5c\x95\x41\xa4\xb0\x73\x95\x02\x48\x61\xe7\xaa\x30\x58\xad\x33\x7a\x19\x89\x9c\xab\x7d\x18\xba\xea\x88\x5e\x86\x40\x2f\x0f\x88\x5e\x60\x50\x3b\x32\x7c\x81\x41\xad\x1c\xfc\x02\x8c\x92\xee\xa6\x74\x18\xe5\x18\x06\xb5\x6e\x4b\x78\x44\xf6\x28\x51\x65\x35\x32\xa8\x95\x5b\x94\x1e\xa6\xb4\xb5\xf3\x4b\xe4\x74\x1d\xb4\x00\x46\x6b\x08\x46\xe5\x30\xa5\x95\x53\x47\xbe\x4a\x30\x14\x46\x34\x18\xd1\x2a\x08\x42\x61\x44\xab\x30\x40\xad\x35\x6e\x11\x39\x9a\x07\x2a\x70\x4b\x0d\x71\x8b\x1c\x46\xb4\xe6\xe1\x16\x98\xcf\x8e\x0c\x5c\x60\x3e\x2b\x07\xb9\x00\x9d\xa4\xbb\x29\x1d\x3a\x39\x8a\xf9\x4c\xed\x88\x60\xc2\x31\x8f\x74\x3c\x9e\xf9\xac\xec\x82\xaa\x30\xa0\xad\x9d\x61\x22\x6f\xee\x00\xe7\x34\xd6\x11\x88\xca\x61\x40\x2b\xab\x06\x6a\x95\xa0\x28\x4c\x68\x30\xa1\x55\x10\x88\xc2\x84\x56\x61\x90\x5a\x67\xe4\xd2\x16\xb9\x96\x07\x5d\x20\x97\x1a\x22\x17\x39\x4c\x68\x4d\x44\x2e\x75\x31\xa2\x55\xa4\x50\x04\x4c\x68\xe5\x20\x17\xa0\x93\x74\x37\x1f\x1e\x9d\x7c\x13\x3d\x36\x58\x73\xc1\x72\x52\xd8\xe1\x2f\xe1\xd8\x28\xde\xe4\x8a\xce\xb4\x5f\xa9\xeb\x45\x6a\x61\x18\x36\xfb\x8b\x70\x59\xe8\x9a\x7b\x1d\xde\xe9\x6b\xd3\x64\xd8\x95\xb0\x9a\x71\x24\xb8\xe5\x16\xac\xc4\x3f\xe5\xd3\x99\x63\x6a\xbe\x61\x4d\xe3\x57\x50\x4c\xc3\xf3\xb9\x79\x73\x57\x04\x03\xee\x8a\xb4\xc0\xb9\x6b\xcf\xa8\x7f\x45\xe7\x22\x2f\x0a\x77\x43\xd1\xd0\x2b\x57\x06\x5b\x12\x3c\x98\x31\xac\x89\x39\xd7\xe9\x73\xb3\x08\x22\x14\xcf\x00\x65\x36\x37\x7d\xa3\xe0\xf6\x68\x65\x29\x05\x58\x2c\xa5\xf3\xf9\x12\xc3\xca\x1f\x73\xea\x06\x88\x4a\x71\x8a\xde\x8e\x1b\x2b\x35\xd5\x3a\xa5\xc1\x5c\xe3\xcb\x23\x2a\xde\xb5\xe1\xfc\xe2\x9a\x3f\x2f\xac\x49\x41\xe7\x96\x3b\x0a\xd7\xb9\xec\x92\x4e\x4d\x31\xf3\xd7\x48\xaa\x99\x97\x17\x8d\x51\x34\x2b\x3e\x17\xea\xef\x6c\x11\x6c\xc1\xf8\xe5\x6e\x2b\x75\x14\x95\x58\x4f\x2a\x5b\x0c\x66\xe1\x97\xb8\xb1\xe4\x5e\x84\x17\x48\x1a\xd9\x64\xab\x60\x6e\x2a\x9a\xcd\x84\x93\x6c\x1b\xdc\xc6\xc1\x4f\xad\x15\xbf\xb1\xe1\xac\x99\xcc\x3d\xdf\x9e\x95\x3b\x63\x22\x01\x3c\xcf\x96\x30\x8f\xdf\xf8\xbb\xf1\x58\x33\xcd\xe2\x49\xb0\x9e\xfe\x30\xd4\x79\x69\x58\x46\x04\x74\xc3\x79\x30\x0e\x55\x5a\x52\x45\xd5\xb0\x2e\x6d\x0e\x4e\x15\xa0\xa9\x25\xae\x7c\xfc\xc3\xf7\xf7\xa7\x24\x83\x97\xd6\xce\xc6\x8c\x46\x8d\x27\x63\x41\x91\xc3\x4d\xb6\x96\xc2\x6f\xad\xdb\x5a\x0e\xf3\xe6\x6b\xb6\xa6\x27\x27\x8f\x7f\x78\x54\x04\x3f\x9f\x6c\x3e\xfd\x22\x0c\xc5\x3f\xdf\xd7\xa6\x6c\x9e\x79\xff\x58\xbe\x9d\x92\xbe\x9a\x13\x43\xd0\x56\x7c\x73\x34\xb9\x43\x39\x71\x17\xe6\x1e\xfd\x18\x3e\x28\xc5\x0d\xd9\xbf\x01\x10\xba\x0f\x75\x9b\xc1\xc6\x24\xd2\x6a\x97\x21\x0c\x55\x2c\xfb\xcb\x99\xba\x84\x6c\x8a\x6f\x47\x6d\x4a\xea\x6b\x8e\x31\xb9\x66\xf4\x2c\xfa\x72\x24\xca\xf1\x12\x57\xf3\x3b\x8b\xd2\x4b\x14\x42\xac\xaf\xd9\x87\x0e\xff\x41\x9d\x25\x7f\xf7\xb8\xbf\x55\xfe\x43\xa7\xc5\x5f\xe1\x80\x65\x9b\xfb\x5b\xd5\xc3\x25\xf7\x79\xf9\x0e\x01\x0b\xc8\xef\x78\xe2\x5f\xe1\x1f\xdc\xe7\x1f\xcc\xff\x4a\xbb\xcb\x7f\x48\x4e\x2a\x50\x06\x3a\xdf\xdf\x65\x5f\x52\xe2\xfb\x6a\x33\xb2\xa5\x44\x80\x64\x09\xdd\x43\x2c\x42\x9e\x90\x17\x11\x0c\x61\x97\x6f\x12\x4c\xf3\xcd\xfd\x37\xff\x1f\x00\x00\xff\xff\x45\xa5\x5c\x18\xc6\xd3\x03\x00") func monitoringBackendGrafanaDashboard1JsonTplBytes() ([]byte, error) { return bindataRead( @@ -184,7 +184,7 @@ func monitoringKubernetesResourcesByPodGrafanaDashboard1JsonTpl() (*asset, error return a, nil } -var _monitoringSystemGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\xeb\x72\xdb\x38\xb2\xfe\x9f\xa7\xc0\xe1\xcc\xac\xed\x59\xc9\xd1\xd5\xb6\x5c\x95\x3a\x15\x27\x93\x33\xbb\x95\xcc\x7a\x62\xcf\x6e\xe5\xa4\x5c\x5a\x88\x84\x25\xac\x49\x80\x01\x40\x5b\x8a\xcb\xf3\xec\x5b\x04\x49\x11\xbc\xe9\x4a\xdd\xf1\x27\xb1\x00\x12\x04\xbb\x1b\xfd\x7d\x68\x34\xc0\xe7\x57\x00\x18\x90\x10\x2a\xa0\xc0\x94\x70\xe3\x12\xf8\x45\x00\x18\x36\xe6\xc2\xb8\x04\x5f\xe5\x2f\x10\x96\xca\x9a\x9e\x87\x6d\xf1\x37\x62\x5c\x82\x7a\x25\x2e\xb5\xa0\x80\x9c\x7a\xcc\x44\xc6\x25\x30\xaa\x55\xf0\x7f\x0c\xde\x43\x02\x41\xb5\x6a\x28\x97\x21\x02\x7b\xb6\x7f\x89\x60\x1e\x52\xca\x07\xd8\xca\x29\xc5\x26\x25\xef\xa8\x4d\x99\xdf\x26\xeb\xf7\xe0\x71\xad\x02\x1a\xf5\x7a\x05\x34\xda\xed\x0a\xa8\x9f\xa8\x4d\x13\xe8\xc8\x67\xbf\x8d\x5f\x07\xfc\x05\xbc\xb5\x11\x13\x5c\xbd\x4e\x8c\x5c\x79\x9d\x05\xf9\xa0\x47\x21\xb3\x8c\xb0\xee\x45\xfe\x7f\xf7\x0a\x80\x17\xff\x72\xc3\x42\xdc\x64\xd8\xf5\x5b\xf2\xaf\x97\x6d\x18\xc8\xc2\x22\xf5\x0a\x46\x9f\x20\xf1\x37\xcb\xb8\x04\xc4\xb3\xed\xa0\x84\x41\x77\x70\x4b\xa9\x2d\xb0\x6b\x5c\x82\x9a\x2c\xc4\xfe\x25\xe7\xc1\x9f\x02\x31\x18\x36\x5c\x3f\xab\xd5\x3b\xe7\xed\x7a\xbb\x5e\x6f\x76\x64\xad\x8d\xc9\x83\xaf\x8a\xaf\x77\xf2\xa7\x0b\x09\xb2\xf9\x58\x19\x91\x2a\x0c\x93\xda\x36\x74\x39\xf2\x9b\xbd\x87\x36\x1f\x4b\xce\xe8\x33\x6c\x5d\xd3\x58\x9b\x81\x88\x53\x1a\x7b\x32\x2e\x41\xa3\xa5\x14\x0c\xa3\x9e\x86\xbf\x47\xfe\xef\x48\x36\xe3\xb6\xe5\x5b\x5c\x8c\x7f\xc6\x9d\xbb\x1b\x97\x09\x2c\xa4\x80\x8c\x7f\xa1\x1e\x78\xeb\xba\x46\x5c\x13\xca\x9e\xd1\xa7\x40\xea\x61\xbb\xe3\x77\x82\x36\x86\x5c\xaa\x5c\xf6\x3e\x7e\x6c\x0f\xca\x92\xe4\x7b\xfa\x2a\xfc\x88\x48\x5f\xc8\x77\xab\x25\xca\x51\xde\xe5\xaa\x8d\xfe\xa8\xfc\x1c\x5f\x72\x8f\x6d\x5b\x95\x53\xb1\x28\x2f\x52\xa2\xac\x37\xa6\x88\xb2\x9e\x2f\xca\xce\xf8\xa7\x8d\xfa\x88\x58\xc9\x27\xc1\xc7\x7e\xfa\x35\x7c\xcd\x7b\x8c\x21\x22\x72\x6a\x1c\x38\xcc\x2b\xc5\x24\xa7\x94\x0f\xe8\x53\x76\xcc\x09\x2a\xa0\x9d\x73\xf5\x23\xb4\xbd\x58\xa6\x99\x77\xb1\x31\x91\xb5\x6a\x6b\xb2\xf0\x09\x5b\x22\x61\x7a\x29\xf3\x96\x45\xfe\xc0\xb9\xa6\x98\x88\x4f\x54\xfa\x01\x59\x10\x6b\x85\xba\x63\xef\x14\x3f\xd1\x45\xcc\x44\x44\xc0\x3e\xca\x28\xda\xf5\x9b\x62\xd0\xc2\x9e\x7f\x4f\x23\x59\x9e\xb5\x0b\x86\x88\x85\x18\x92\x5e\xe6\xde\xa6\x22\x7e\x30\x47\x0c\x23\xfe\x8f\x47\xc4\x18\xb6\x50\xaa\xd3\xdc\x85\x26\xca\x33\x3f\x2e\xa0\xf9\x90\x96\x05\x17\xc8\x75\x91\xf5\x11\x93\x6c\x7f\x05\x64\x7d\x24\xb8\xe2\x6f\x55\x8f\xeb\x7b\x9d\xa1\x2b\x7b\xc7\x3d\xe7\x98\x41\x81\x8e\x19\xc4\x36\xef\x32\xf4\xcd\x43\x5c\xf0\xae\x54\xda\xb3\xef\x01\x65\xa7\xde\x1c\xfd\x38\xfe\xfb\xa8\xe2\x52\xeb\xcd\x9f\x47\x7c\xc4\x05\x72\xaa\xd0\x75\xab\x5f\x61\xf5\x7b\xad\xda\xb9\xfb\x6b\xfc\xd7\xd1\xcb\xd7\xba\x73\x77\x72\x02\x7a\x23\x70\xcc\x05\x14\x1e\x57\x9d\xab\x3f\x32\x28\x73\xa0\x6f\x72\x86\xc0\x0e\xea\x06\x92\x49\x5e\x82\x89\x40\xec\x51\x5a\x8f\x51\x77\xf2\xeb\x3e\x40\x53\x48\x7f\x5e\x4f\x54\x07\xb6\xff\x61\xfc\x8c\xe7\xe7\x7f\x3f\x3f\x07\xfd\x78\x79\xf9\xf7\xcb\x4b\xb2\x31\x86\xee\xa5\xbf\x35\xde\x1a\xe3\xe2\x97\xf0\x2f\xc5\x0d\x0d\x18\xe2\x03\x6a\x5b\x19\xf7\xe4\xa0\x0f\x8c\x3a\x8a\xc3\x1e\x97\x7f\x46\xfd\xd0\xd2\x52\x37\xdc\x0c\xf0\xbd\xc8\xde\x11\x3a\xba\x1b\x29\x5c\x10\xe9\x03\xb8\x88\x01\x8e\x4c\x4a\x2c\x70\xdc\x1b\x81\xb4\x40\x0d\x31\x86\x86\x67\x75\x3c\x42\x26\x9d\x79\x6a\x44\x72\xca\x44\xca\x9f\xc8\xc1\xd8\x8d\xbc\x29\x26\x16\x7e\xc4\x96\x07\x6d\x23\x33\x2e\xa3\x6b\x24\x22\xc5\x1d\x18\xc2\x21\x4e\x39\xb5\x9e\x67\x3e\x04\x46\xa8\xbe\xa3\xef\x3d\xc2\x31\xe9\x8b\x21\x07\x70\x53\x57\xe7\x7b\x95\xb1\xf7\xf8\x7a\x97\xe9\xe2\x08\x0e\xd1\x04\xdb\xb7\x90\x89\x1d\x28\x31\xa6\x5e\x60\x91\x0c\x7d\x73\x53\xb6\x68\xc3\x1e\xb2\xc7\xb0\x1d\x17\xd3\xfe\x15\xe4\x28\xd3\x56\xe0\x37\x93\xaf\x32\x76\x9c\x99\x62\xe5\x1d\x63\xfb\xab\xe4\x77\x3f\xee\x25\x1f\xf8\x8a\xcc\xed\x65\xe6\x09\x2b\xec\x67\x66\x9c\x8c\xb2\xb6\x00\x6d\xdc\xcf\x83\x0c\x59\xfe\x11\x3d\x8e\x3b\x9d\xa0\x4e\x7b\x0c\xe6\x89\x82\x09\x68\xae\xf4\x5b\xc3\xb9\x86\xf3\xad\x80\x73\x93\x12\xc1\xa8\x6d\x23\xb6\x79\x48\x8f\xfb\xb2\x07\xb0\x9e\x27\x58\x0d\xed\x1a\xda\x81\x86\xf6\x1d\x82\xf6\xf4\x3c\xbd\x53\x80\xec\x75\x8d\xec\x1a\xd9\x37\x8f\xec\x95\x60\x32\xf9\xe6\xa8\x31\x1c\x6a\x98\x2f\x13\xe6\x1b\xc3\xa1\x86\xfa\xf1\x5b\x68\xa8\xd7\x50\x0f\x76\x1b\xea\xcf\x52\x48\x9f\x99\xc4\x17\x41\x7d\x43\x43\x7d\x09\x50\x9f\x78\xec\x96\x20\x7d\xf2\x29\xbb\x02\xf5\x2d\x0d\xf5\xe5\x42\x7d\x4b\x43\xbd\xf2\x16\x7b\x07\xf5\x46\xcd\xd0\x48\x7f\xd0\x48\x7f\x31\x23\xd2\x37\x35\xd2\xef\xeb\xa4\x7e\x37\xa1\xbe\xad\xa1\xbe\x5c\xa8\x6f\x6b\xa8\x57\xde\x42\x43\x7d\x6e\x27\x35\xd4\xef\x0c\xd4\x4f\xcf\xb3\x3b\x2f\xc0\xfa\x96\xc6\x7a\x8d\xf5\x4b\x62\x7d\xd7\xf2\x82\xc4\xdd\x6e\x00\x23\xbc\xcb\x3d\xa7\x94\x95\xfa\xd7\x60\xd6\x27\x9a\xd4\x23\xa2\x8c\x67\x2e\x4c\x28\x66\x24\x0d\xb7\xd2\xc0\xb7\x9b\x25\x7c\xf6\x65\x0d\xe0\x23\x62\xb0\x8f\x00\x43\xdc\xa5\x84\x23\x90\xc0\x53\x4d\x09\x16\xa3\x04\x19\xd0\x53\x00\x57\xc7\xf9\xb7\x8d\x12\x98\x90\x59\xa9\x86\xfd\xa2\x6b\x68\x59\x98\xf4\xb3\x96\xe3\x57\x7e\xa6\x1e\xb1\x52\x8d\x8f\x7b\x6a\x86\x3b\x37\x52\x0d\x8e\x37\x74\xfc\xd0\x3e\xef\xb4\x3e\x34\x54\x1b\x95\xb7\xdc\x98\x30\x18\x9b\xfc\x5b\x42\x03\x51\xed\x00\x39\xe1\x38\x12\x88\xb9\xd4\x86\x02\x5d\x49\x73\x55\x2e\x45\x43\x97\x92\x00\xb8\x6b\xa7\xed\x9c\xd1\x41\x5d\x68\x62\x31\xca\x8e\x40\x9f\xad\xc4\x2e\x4c\xf0\x68\x9c\xcd\xc3\x6e\xf2\x76\x8e\x80\xe5\x58\x4e\x36\x01\x31\x4b\x73\x06\x08\x0a\x07\xba\x49\xf0\x1e\x60\x0b\xfd\x3f\x62\xf4\x6a\xec\x2f\x92\x80\x38\xc0\xfd\x81\x8d\xfb\x03\xf1\x2e\xd4\x7f\x82\x57\x04\xcc\xa9\x3d\x91\x39\x85\x76\x5b\xc8\x56\xd2\x14\x24\x97\x63\x30\xf4\x88\x18\x47\x5f\x8a\xba\x39\x27\x6e\x4f\x04\xd0\x40\xa5\x33\x23\xe8\xe9\xcf\x47\x2f\xc1\x64\xdc\x46\x85\x98\x19\xc9\x7e\x89\x09\x78\x6d\xfa\x0c\xdc\x46\x8b\xcf\xbc\x27\x60\xe7\x14\x88\xcc\x99\x3d\x87\xaf\x2b\x67\xd1\x09\xd0\x04\x81\x70\x01\x26\xe1\xa5\x33\xa4\xbd\xe7\x21\x90\x5f\xfa\x2b\xe6\x82\xf6\x19\x74\x0a\xad\x2b\x82\xcb\xb4\xf4\x8d\xe1\xdb\x8c\x97\xcc\xba\xd7\xb8\x9d\x61\x60\x76\xbf\x79\x4e\x4f\x92\xd3\x84\x20\xc2\xca\x1b\xfc\x3d\x8d\x9f\xc6\x28\xfb\x18\x05\xff\x54\x0a\x90\x0f\x7d\xf9\xc8\x91\x8b\x1b\xb9\xa8\x51\x24\x3c\xd7\xc6\x62\x6c\x58\xb9\xce\x79\x14\xbc\xd4\x55\xe8\xc0\x0d\xe8\x09\x6a\xa4\x6b\xf3\xe5\x31\xca\xc8\x23\x1f\x4f\xd6\xb0\x3d\xad\x59\x10\xd7\x6d\xc4\x53\x84\x89\x1b\xd4\x6e\xb0\x85\x1e\xf0\xb7\x83\xd8\xa0\x36\x55\x98\xad\x82\xcd\x7e\x8d\x29\x99\x6f\x3e\xcb\x78\xcb\x6f\xf3\x37\x76\x06\xd3\xea\x54\x61\x3c\xab\x4e\x55\xcc\x33\xa9\x66\x3e\x68\xdd\xe4\xee\x1b\x5d\x70\xbe\x9d\xeb\x1a\xf4\x74\x7b\xbd\xd3\x6d\x1e\x0c\xc9\xee\x7f\x68\xcf\x9f\x5c\x9b\x26\xe2\x93\x03\xec\x8b\x4e\x66\xc3\x2d\xc7\x29\x6b\xc8\x43\xe6\xc6\x44\x60\xbe\x09\xfa\x78\xef\xd9\xc0\xef\xf3\x0c\xd8\x5c\x30\xf7\x98\x2c\x8a\x7b\x88\x6d\x64\xad\x44\x12\x73\xbe\xf0\x07\xd9\x93\x49\x2f\x7b\xb5\x25\x93\xfb\xbf\xd3\x1e\x07\x2e\xa3\xbe\x7e\x90\x05\x7c\x8a\x39\x0a\x69\xc9\x72\x93\xfb\x46\xda\x7d\x1c\xc8\xe4\x3e\x36\x26\x5a\x14\xd4\x2f\x67\x72\xac\xc3\xfa\x6a\xb9\x0e\xeb\x5f\x82\xd6\x45\x01\x3b\x99\x92\xac\x57\x22\x3b\xf1\x11\xe3\x17\xc7\x15\xa3\xfc\x2a\x7f\x96\x7d\xa8\x94\x06\x40\x0e\xbe\xfb\xef\xbf\xeb\xd4\x66\x85\x5b\x01\x16\x65\x36\x32\xf4\xf0\xcd\x43\x5e\x71\xf4\x61\xe5\x24\xe7\x7a\x0c\xa2\x98\x00\x19\x8a\x90\x1d\x9a\x35\x1a\xb1\x62\xc6\xb3\x8c\x84\x16\x23\x3f\x33\x4b\x61\x4b\xa9\x50\x6f\x04\x64\xd7\x35\x0d\xd2\x34\x48\xd3\xa0\x2d\xa6\x41\x99\xb8\x7f\x21\x0f\x9a\x92\xca\xa8\x79\xd0\x3a\x78\x90\xe6\x3f\x63\xd9\x97\xcb\x7f\x9e\x28\x7b\x58\x36\x07\x72\x29\xe0\x97\x70\x1f\xf4\x62\x17\xb2\x1f\xb3\x78\xff\x1f\xda\x03\x12\x7a\x35\xe4\x6b\xc8\xd7\x90\xbf\xbd\x90\x9f\x8e\x7c\xb4\xcf\x0a\x10\x7f\x4a\x42\xe3\x0c\x88\x5f\x9c\xee\x38\x3b\xe4\x6b\x60\x3f\x50\x60\x77\xe0\x30\x89\xe9\x4f\x10\x0b\x4c\xfa\x13\x13\x14\x5f\xd6\x39\x59\x9f\x3f\x50\xb1\x49\xc8\xfe\xdd\xef\x28\xe0\xf8\xbb\xf2\xd6\x1a\xa3\x17\xc2\xe8\x95\xc3\x9f\x46\x69\xb5\x7c\x67\x50\x3a\x95\x98\x17\x8c\x38\x1b\x0a\x44\xcc\x11\xc0\x1c\x88\x01\x02\x16\xbe\xbf\x47\x0c\x11\x13\x29\x49\x4c\x80\x63\xbf\xc0\xaf\xa7\xb6\x85\xb8\x90\x74\x1a\x13\x59\x22\x3d\x0c\x78\x82\x1c\x20\x22\xff\xb6\x4e\xc1\x3f\x18\x18\xd0\x27\x60\x53\xd2\x0f\x2e\xe5\x20\x74\x8f\x00\x0b\x0e\x84\xc7\x08\x10\x14\xf4\x10\x40\x43\x64\x7a\x02\x59\xa7\xeb\x89\x1f\x14\xb2\x89\xc9\x49\x7e\x9a\x4d\x68\x36\xb1\xda\x0c\x10\x95\x4e\xc8\x51\xd4\x0d\x07\xa6\x66\x12\x8b\x31\x89\x40\x7c\x58\xb3\x89\xa5\xd9\x84\x66\x12\x9a\x49\xac\x6f\xbe\x7f\xd6\x2a\x40\xe8\xb3\xc9\x08\xad\x37\x30\x6a\x3c\x55\xf0\x74\x10\x65\xcf\x77\xbf\x79\x90\x08\x6c\xa3\xe3\xda\x69\xa7\x5d\x01\x79\x01\xf9\x2e\xf3\x48\x88\x95\x33\xec\x91\x50\x03\xf3\x36\x3a\x39\x01\xab\x86\xdf\x4e\xfb\x27\x10\xbd\x05\xa0\xf7\x20\xec\x39\xd8\x72\x1c\xfe\x3b\xed\x45\x31\x78\x9f\xf8\x47\xbb\x0c\xf5\xe6\x42\x8d\xc9\x1a\x93\xb7\x1a\x93\x33\xd3\xe6\x42\x50\x3e\xd7\xa0\xac\x41\x79\xdb\x40\x39\x58\xa7\xae\x80\xb5\x80\xf3\xee\xad\x8c\x17\x81\xb2\x5e\x26\x07\x1a\xa0\x35\x40\x87\x85\x5b\x0d\xd0\xe9\x49\xf3\x79\xa3\x00\x9f\x2f\x26\xe3\xf3\x7a\xd2\xe2\x74\x54\x7b\x27\x00\x7f\x5d\xc9\x6f\xd1\x32\xd1\x96\xe5\xb6\xef\x56\x90\x3b\xdc\xbb\x2d\xf7\x21\x02\xd7\xe3\x03\x9d\xd6\x5e\x1e\x78\xeb\xf5\x73\x0d\xe0\x6b\x9e\x61\x17\x22\xf8\x94\x2f\xe4\xee\xe4\xc2\x74\xfa\x0e\x8d\xe1\xdb\x8b\xe1\x99\x3c\x37\x86\x04\x1b\x4d\xc9\x72\xd3\x58\x9d\x3c\x42\x48\xc8\x9d\xf7\x42\x87\xbd\x35\x30\xef\x23\x30\xaf\xe3\xb0\xa3\x8b\x82\xf3\x79\x9a\x93\x0f\x3b\x62\xc8\x45\x81\x56\x2c\xe4\xda\x74\xe4\x20\x22\xde\x51\x72\x8f\xfb\x46\x76\xa0\x5e\x53\x8b\x83\xe3\x1f\xd3\x57\x9e\xcc\x71\x42\x92\x09\xcd\x01\xba\xc5\x0e\xa2\x5e\xc6\x21\xc8\x13\xfc\xae\xa0\xf9\xd0\x67\xe1\xc1\x53\x09\x77\x2d\xab\xff\xe9\x8f\x93\x8c\x18\xcd\x88\xf4\xc4\x43\xc4\xf8\xe1\x43\xa3\xd5\x69\xbf\x53\x07\x24\xeb\xf7\xe0\x71\xa3\x79\x5e\x01\xf5\x46\xa7\x02\x5a\xb5\x0a\xa8\x9d\x5e\x74\x54\x77\x6c\xfc\xd0\xe8\x74\xcc\xd6\x99\x91\xb1\x8e\x99\x12\x08\xb3\xa7\x7b\x29\x76\x4f\x28\x51\x2e\xee\x43\x4f\xe2\xe4\x73\x82\x45\x44\xef\x57\xaf\xd5\x92\x44\x22\xaa\xa8\xcd\x40\x17\x22\xd7\xfd\xd1\x1f\x55\x19\x70\x55\xaf\xf8\x04\xd9\x03\x62\x85\x14\xa2\xd0\x3c\x9b\x29\xf3\x4c\x7f\x62\x21\x63\x9d\xd9\x2f\x22\xfb\x6c\xca\x37\x84\x08\xca\x73\x0f\x17\x6c\xc6\x1c\x25\x77\xaf\x56\x1e\x73\x71\xa0\xeb\x62\xd2\xbf\x0d\x4c\xb1\x9e\x57\x3e\xc1\x9b\x86\x4e\x3b\x70\xc8\x40\x50\x20\xd0\x30\xe5\xac\x1e\x23\x1d\x4d\xf5\x79\x51\x63\x0c\x92\xfe\x94\xc6\x1a\x13\x1c\x93\x03\x87\xef\xa1\x80\xd7\x11\x57\x52\x8c\x23\xcb\xd3\x4c\x4a\x08\x32\x05\x52\x0e\xb1\xf1\xaf\xb9\xf5\x9f\x9c\x1a\x70\xf9\x24\xce\xf6\xfa\x98\xfc\x13\x31\x1e\xe6\xc1\x9e\x9d\x36\x4e\x5b\x86\xc2\xd8\xb8\xb8\xc7\xc3\xa4\x1a\xc2\xc2\x0f\x94\x44\x27\xbf\x19\xed\xda\x4f\x4a\x3d\x43\xd9\x7b\x64\x59\xe1\x2d\x52\x66\x9f\xa0\x3b\x09\xf9\x02\x1e\x92\xa4\xa6\xb2\x46\x04\x6f\x6b\xfc\xf6\xfa\x6d\xaa\x82\x8e\x6f\x98\x20\x70\xee\x42\xf6\x60\x07\xbc\x50\xb1\x7c\x7f\x82\x33\x3e\xa0\x54\x3a\x93\x66\xbd\x02\xea\xf5\x8b\x0a\xa8\x5f\x74\x7c\x67\x52\xbf\x48\x38\x93\x7b\xcf\xce\xe3\xed\x7e\xcb\x6a\x3b\x41\x33\x8d\x5a\x05\xd4\x3b\xcd\x44\x03\x13\x4f\xcf\x14\xfe\xac\xe6\x1d\xb5\x3d\x27\x75\x90\xe8\x5c\x01\xa8\x07\xaf\x87\xba\x0c\xb9\x36\x36\xe5\x41\x98\xf1\x67\x0f\xba\xc1\x67\x27\xba\x0c\x41\x6b\x14\x5d\xc2\x8b\x4e\xc6\xcc\x6d\xe2\xcd\x9f\x47\x19\xb8\xa8\x7e\x0d\xcf\x99\x2e\xfd\x88\xe9\x05\x98\xad\x51\xaf\x34\x8c\x3c\x72\x6b\x34\x6b\xdc\xc8\x25\xb1\xe9\x9a\x88\xc5\x7a\x84\x60\xd2\x07\x2e\xb5\x78\x16\x10\x39\x26\x7d\x1b\xf9\x02\x8d\xeb\xe4\xd0\x57\xed\xff\x42\xb5\x7f\x59\x3b\xd9\xfe\xa9\x4f\x90\x8d\x37\xf9\xa6\x5f\xcb\xf7\x33\x53\x6d\x5f\x5e\xf8\x5b\xe8\xb8\xfc\x19\xf2\x4a\x70\xfc\x3a\x72\x08\x39\x40\x3e\x07\xc6\x87\x60\x3d\x2f\xc6\x87\xd4\x40\x63\xfc\x32\x18\x7f\x56\x16\xc6\x37\xf2\x30\x3e\x61\x51\x1a\xe5\x4b\x47\x79\x8d\xe2\x07\x83\xe2\x2e\x32\x57\x81\xde\xa0\x0a\x34\x7f\x28\x8f\x3f\xfc\x41\xe0\x23\xc4\xb6\x6f\x0d\x9a\x43\xe8\x58\xc0\xfe\xf0\x84\xcc\x5a\xce\xc2\x44\xa1\xa9\x83\x01\x3a\x18\xa0\x69\xc4\x8a\x68\x84\x5c\xb2\x3a\x8e\xfe\x25\x02\x62\x82\x58\xd7\x41\x0e\x65\xa3\xae\xc7\x61\x1f\x75\x7b\x23\x81\x0a\x01\x3c\xf8\x34\x46\x0e\x5c\xe7\x7d\x63\x2a\xc8\x5c\x21\xd4\x42\xe5\x7f\x69\x6a\xa3\x30\x2e\x63\xe4\x16\xe6\x82\xe1\x9e\x27\x90\x05\x28\x01\x03\xca\x85\xc6\xf3\x32\xf1\x7c\xc1\x79\xbf\xd5\x6a\xc1\x26\xd4\x78\xbe\x1c\x9e\xa7\x3f\x9f\xbc\x30\x9e\xb7\xf4\xc4\x7f\xfd\x88\xae\x11\x7b\x6f\x10\x5b\xe6\x9e\x78\xce\xb1\x85\x6c\x01\x83\x89\xb8\x4b\xad\x6e\x0c\xde\xe3\x09\x38\x17\x90\xcd\xf6\x29\xea\x19\xf1\xfb\x6b\xdb\xb9\x3b\x09\xf3\x4f\x5d\x6a\xad\xfc\x6b\x91\x05\x29\x2c\xb3\x1f\xa0\x5c\x1e\xc2\x7f\x82\x43\x39\x41\x07\x91\x58\xc1\xb1\x0d\xb9\x00\x6d\xe0\x60\xe2\x09\xc4\x73\xd6\xc3\xf7\x0d\xea\x77\x27\xd5\xf0\x3c\x85\x66\xd3\x13\x29\x0a\xf6\xf2\x35\xa7\x1c\x81\x33\xe7\x5e\xbe\x95\x1f\x93\xbb\x45\x1b\x00\x1d\x38\xbc\x46\xec\xb3\xec\x4f\xb3\x18\xda\x56\xf1\xa9\x80\x76\xb2\x7c\x9e\x4c\xc3\x71\x6a\x4c\x02\x54\x83\xd2\xf7\x98\x21\x33\x3c\x28\x2a\x51\xbd\x75\x7b\x0a\x37\x14\x1c\x5e\x75\x86\xa3\xb4\xe3\x6a\x22\x4e\xaa\x5c\xf7\x11\x93\x87\xbc\x3c\x30\x65\x66\x98\x28\xf7\x25\x2d\x35\x32\x95\xf2\xed\xe9\xca\x79\x5a\xbc\x32\x14\x9d\x23\xde\x5c\xe0\x5d\x52\x54\x7a\x79\x62\x41\x25\x79\x64\xaa\x9a\xde\xcd\xae\xa6\xbd\x09\xff\x64\xc4\xc4\x91\x55\x4d\x86\x60\x92\x42\x7a\xbf\x25\x79\xd0\xd7\xd4\x02\x52\xfe\xe0\x58\xba\xb7\x0a\x90\xfa\xad\x00\x8f\xf8\xff\x9f\x00\x48\xac\x80\x79\xca\xb7\x89\xa3\x4c\x3e\x0e\xc5\xcd\xe9\x3d\x4d\x8b\x7c\x8e\xbc\x56\x60\x85\x2b\x4f\x56\x4e\x3e\x78\xc7\x32\xaa\x55\xb6\xa8\xf7\x3a\xe5\x4e\x40\xce\xe6\x9d\x80\x74\xb2\xe1\xb4\x60\x02\x52\xee\x09\x5f\x7a\x02\xb2\xad\x5b\x9c\xf6\x67\xe2\xb1\xb1\xc0\xd4\xda\xe2\x52\x72\xa7\x95\x4b\xad\x5d\xd8\x67\x75\x3d\x6b\xd0\x4a\x6f\xbe\xd2\x9b\xaf\x4a\xa7\x0a\xfb\xbc\xf9\xaa\x73\x5e\x00\xd9\xf1\xf9\x5f\x4b\x6f\xbe\x7a\x77\xfd\x07\xf8\xc3\x9f\x86\x2d\xb9\x03\x6b\x77\x98\xd3\xdc\xa1\xdb\x4e\xc1\xd7\xcf\x9a\x53\x8e\x79\x39\x8c\x63\xd8\xf6\x3b\x0a\x6b\x28\xf8\xb1\x95\x5c\x88\x50\x0b\x75\xc7\xac\x26\xc9\x87\x2e\x63\x66\x64\xba\x5e\x18\x6d\x89\xce\x75\x93\xc6\x72\xf9\xfc\x0c\x4e\x6f\x3c\xe7\x33\x14\x08\xbc\xbc\x28\x54\xe9\xcf\x65\xa3\x30\xe5\xd0\xa4\x19\x36\xa4\xe7\xd3\xa4\xd2\x22\xb6\x9b\xa4\x56\x63\xe7\xac\xe3\x30\x5b\xce\xa2\x74\xb8\x45\x29\xdf\x2a\x12\x55\xaf\xb5\x0b\xe0\xbb\x53\x2e\x8b\xfa\xdd\xa3\x02\xae\x95\x45\x99\x32\xcb\x23\xd5\xef\x35\x50\xab\x7b\x25\xbb\xa0\x5e\x53\xd3\x0b\xca\x23\x5d\xf5\x5a\xc1\x37\x63\x5a\x35\xcd\xba\x56\xc2\xba\x60\x1f\x85\x4a\x4d\x80\xd1\xd6\xb2\x31\x93\x51\x69\x97\xc9\xc3\x77\x26\x72\xb4\x01\x7d\xfa\x15\x41\x4b\x76\x21\x79\x5b\x00\x94\x8a\x1d\x99\xd4\x4e\x79\x1b\x0b\x71\xb3\x50\x9b\x25\xd2\x3f\x2e\x46\xf6\x24\x54\x93\x3e\xc2\x17\xc6\x6d\x12\x55\x83\xc1\x8c\x62\x76\xf4\xe5\xcb\x97\x2f\xd5\x4f\x9f\xaa\xef\xdf\x83\x5f\x7f\xbd\x74\x9c\x4b\x9e\x62\x5b\x2e\x14\x02\x31\x92\xdf\x56\xe4\xab\x06\xd8\xb2\x10\x99\xbe\xe0\x37\xee\x56\x96\xb4\x44\x02\xa5\x2c\xb4\xcb\x0c\x10\xc5\xd9\xb8\x77\xcb\xbc\x90\xb2\xf0\x92\x22\x8e\x01\x0f\x4c\x0d\xd1\xb0\xe2\x76\x4c\xa9\x8c\xf7\x0c\xdb\x36\xb0\xe8\x13\x31\x32\x97\xfd\xc1\x72\x3e\xe8\xab\x88\x50\xa6\xdb\x82\x1f\xd2\xa9\x85\xf9\x6c\x31\x21\x62\xe2\x39\x3d\xc4\x92\xf7\x79\x04\x2b\xa4\x60\x3e\xe9\x7f\x46\xdf\x3c\x94\x59\x98\x3c\x14\x05\x5c\x6d\x8f\x02\xc0\x4f\x87\xa9\x82\x77\xe5\xaa\x20\x84\x20\xf9\x73\x3e\x45\x7c\xc4\x0e\x3e\xd4\x71\xf0\x7e\xf3\xe3\x20\x10\xff\xa1\x8e\x82\x5f\xb6\x61\x14\x5c\x53\x6b\xfb\xa4\x9f\x64\xd0\x0b\x09\xff\xb5\xf5\xfa\xf9\x19\x9c\xfe\x16\xc5\xa8\xc0\xcb\x4b\xa6\xa0\xda\xe4\x26\xb4\x51\xf5\xc1\xeb\x21\x46\x90\x40\xbc\x6a\x52\xc7\xf5\x04\xaa\x32\x14\xcc\x72\x78\xd5\xa5\xd6\xff\x3e\x42\x56\x8d\x43\x5f\x71\xe0\xeb\x2f\x7e\x85\x4b\xad\x37\x3f\x76\xbb\x26\x4a\x6f\x27\x50\x34\xee\xa6\xa5\xbc\xee\xd1\xb6\x45\x3a\x56\xc4\xf2\xfa\xf4\xe7\xd7\xf3\xcb\x85\x0b\x86\x49\x7f\x36\xb9\x84\x7f\x29\x41\xb5\xfd\x0f\x9e\xca\x53\x67\x13\x95\x98\x70\x01\x73\xce\x96\x9d\x3f\xa2\x5a\xb8\xd6\x9c\x28\x5f\x22\xb1\x35\xb9\x60\x1f\x0d\xc3\x2e\x0b\x19\x93\x94\xb5\x49\x59\x22\x21\x70\xef\xc4\x79\x55\x86\x38\xb7\xda\x78\xc1\x6b\xa0\x55\x9e\x4c\x99\x5d\xf1\x08\xb2\x25\xd7\x3a\x0c\x61\xbe\xd7\xe3\xe7\xb0\x14\xfe\xcb\x7a\x96\xe9\xa6\xaf\xc6\xc9\x20\xff\x46\x72\x9a\x18\x24\xdc\x57\x42\x56\x05\x63\xe2\x94\x2a\xd6\x2b\x75\x87\xb2\x52\xf7\x0a\x6c\x6a\x71\xad\xde\xcc\xa8\x3a\x58\xa5\x89\x57\x2f\x96\x5e\x5c\xfb\x24\x37\x8b\xe8\x2c\xa5\x69\xaa\x28\xd8\x61\xda\x8a\x73\x86\x0f\x78\xc1\x6c\xdb\x13\x92\xb6\x32\xb9\x68\xc6\x0d\x5b\x8b\xd3\x8d\x0a\x18\x3f\xe2\x7f\xde\x1c\xe9\xd4\xa1\xf2\x52\x87\x54\xa7\xa9\xb3\x87\x96\xe3\x24\xd2\xde\x35\x27\x39\xc8\xec\xa1\x46\xc1\xb6\xa9\x56\xb3\x74\x82\xa3\x13\x88\x4a\xe5\x43\x8d\x82\x6f\x7b\xb5\x5a\x9a\x0f\xed\x6e\xaa\xd0\xa6\xd2\x7e\x72\x0f\x3b\xd3\x59\x3f\xe3\x6e\xe5\xf2\x8d\x48\xa4\x1b\x5f\x70\xdc\xc1\xc4\x1f\x0b\x99\x01\xef\x98\x5b\x07\x3a\xfd\x67\xab\xd4\x70\xa8\xe9\x0f\x5b\x91\x04\x14\xea\x42\xe7\x01\x95\xa6\x87\xc5\x07\x84\xce\x06\x2a\x51\x0d\x3a\x1b\x48\x67\x03\xe9\x6c\xa0\x52\xb3\x81\xb6\x31\xda\xb9\xd7\xb9\x3e\xa1\x98\xcb\x13\xf0\x96\x4a\xb4\x94\x74\x9f\x0d\x5a\xe7\x3c\xc9\x3c\x87\xa2\xd3\x75\xe5\xf3\x1c\x8a\x3c\x4b\x49\xe9\xd9\xee\x31\x72\x60\x1a\xdd\x8a\x9c\x1d\x35\xb4\xae\xd3\x76\x74\xda\xce\x56\x2d\x91\xbd\x02\x1b\x5b\xd5\xea\x64\x54\x1d\xac\x8d\x28\x91\xf9\x65\x57\xb5\x7e\x43\xe2\x89\xb2\x07\x9d\xb7\x33\x45\x17\xcd\x82\x4f\xec\xb7\xca\x3d\x98\x71\x9f\xd6\xa9\xca\xca\xd7\x59\xe2\x34\xc5\xad\xcc\xd7\xc1\x0c\x0a\xa4\xb0\x20\x12\x0c\xc1\x2e\x43\x26\xc2\x8f\x21\x11\xca\x9c\x8a\xb8\x04\xf3\x28\xfd\x58\xc4\x85\x93\x76\xb6\xee\x58\xc4\xcf\x81\xcc\xc1\x15\x24\x56\x60\xe9\x4b\xd1\x8f\x43\xcd\xc2\x51\x02\x4b\x19\x74\x8d\x0d\xec\xca\x5d\x69\x7e\xce\x8e\x9f\x91\xa8\x8f\x53\x5e\x05\x6c\x17\x1c\xce\xd8\x3a\xd7\xb0\xad\x61\xbb\x14\xd8\x96\x93\x47\x07\x0b\x8d\xdb\x6b\xc4\xed\xdb\x50\xe8\x1a\xb8\xcb\x8a\x0d\x68\x70\xde\x4a\x70\x5e\x4b\xa0\xa1\x55\x70\xf8\x9e\xfa\x31\xca\xa5\x23\x0d\x6f\x4d\xe1\x33\xed\xcf\xc8\xa4\xcc\x5a\x6f\x02\xed\x84\x6f\xd3\x6e\x92\xce\x5c\xcc\xaf\xa8\xa2\xe3\xf6\xf4\x21\xc7\x2b\xa0\x33\x53\xbf\x7f\xbe\xcf\x74\x47\xfa\x55\xf9\x65\x2c\x68\xa2\x63\x06\xb1\xcd\xbb\xe1\xf7\x67\x31\x25\x5d\x97\x52\x5b\xf9\x5d\xd2\x1a\x49\xe5\x28\x6e\xf2\xa8\x02\x8e\x7e\xac\xfb\xff\x72\x01\x05\xf2\xff\x38\x3e\xfd\xf9\xe4\x28\x64\x44\xf1\x85\x8b\x13\x23\x99\xd9\xee\x18\x15\x30\x91\x35\xcd\xf0\x11\x88\xb8\x33\xbb\x40\x9e\x3e\xfb\xca\x04\xd7\xd4\x02\xef\x62\x05\x6e\x64\xdd\x65\xe7\xf9\xd3\x0c\x1f\x91\xb2\xa9\x09\xd3\xcb\x7a\x11\x69\x49\xad\xca\x1d\x22\xb3\xda\xd4\xa7\x21\x76\x27\xea\x91\xa6\x09\xf5\xc6\x34\x9a\xd0\x2e\xd8\x64\xdc\xde\xf8\x26\xe3\x74\xf1\x5e\xf0\x84\x7d\xe6\x01\xa0\x00\xfa\x39\xfe\x8e\x4a\xca\x8b\x58\x00\xbe\xa3\xa4\x88\x6c\x4a\x70\x1a\xda\x97\x40\xf6\x9d\x40\xf1\x18\xc1\xc1\x35\xa5\x36\x90\x3b\xbe\x34\x94\x2f\x07\xe5\xf5\x0d\x41\xf9\x6c\x18\xa9\xb1\x7c\x8f\xb1\x3c\x51\x30\x19\xcc\xeb\x1a\xcc\x35\x98\x97\x02\xe6\x4f\x10\x0b\x4c\xfa\x9b\xc3\xf3\xc3\xc6\xec\x7f\x05\xe2\xd7\xb0\xad\x67\xe0\x7b\x84\xda\xed\x24\x64\xbf\x0a\xdb\xf4\x07\xa5\x3f\xde\xe4\x60\xaf\x05\x9e\xc0\xe0\xe6\x00\x39\x30\x0e\xb5\xd6\x2f\x82\x62\x31\x0a\xc6\x8d\x05\xd9\x43\x70\xa5\x80\xfd\xd8\x18\x8c\x60\xb3\x58\x28\x34\x83\x8f\xb8\x40\x8e\x31\x7e\x92\x40\x8e\x6b\x43\x39\xb4\xa2\xee\x1b\x36\xe6\x42\x31\x26\x15\x9e\xb1\xb4\x63\x75\x1c\x61\x62\xda\x9e\x85\xde\xda\x79\xa8\x96\xaf\x1c\xc3\xf1\x6c\x81\x73\x2e\x0f\x87\x83\x91\xc3\x1f\x12\x30\xa5\x6e\xb0\x32\xbe\x79\x88\x8d\xe4\x66\x35\x46\x1d\x24\x06\xc8\x53\x5d\xa7\x22\xca\x7a\xa2\xb4\x8f\x86\x29\xd3\x36\xf8\x03\x76\xff\x60\xf6\xcd\x88\x98\x79\xc8\x1d\xba\x01\xa5\x73\xe9\x71\x98\x50\xbf\x2d\xb7\x4b\x66\x5f\x3e\xa6\x17\x09\x23\x8f\x94\x96\xda\x3b\x86\x86\x61\x04\x35\xb9\x05\x30\x39\x06\x1e\xc3\x27\x65\x2f\xcb\x1b\x5b\x0b\x6b\xd1\x18\xc3\x9d\x31\x87\x32\x73\x6f\x52\x74\xa9\xbc\x88\x2a\x10\x49\x0f\x6c\x64\x8a\x1c\x7f\x3e\xbb\x68\x66\x13\x4e\x3c\xa4\x95\x41\xad\x9a\xd6\x84\x67\xcc\x68\x35\xa6\xc7\x05\x75\x66\xb4\x98\x84\x55\x4e\xe7\xd6\xd2\xc3\xdf\x63\x82\xa3\x8f\x0b\x05\xeb\x12\x01\x60\xc4\x1b\x39\x30\xb9\xa7\x93\x3f\x21\x1d\x38\x87\xea\xe9\xcf\x47\x2f\x15\x90\x4a\xa7\x98\x6a\x35\x29\xb8\x1a\x1b\x4d\xe1\xb2\xe7\x2c\x8e\x60\xc2\xbd\xd3\xdc\xc1\x4a\x84\x30\xc5\x9b\xbc\x3e\xf6\x49\x5e\xc8\xf5\xee\xfe\x7a\x52\xfd\xea\x13\xbe\xc4\x66\xd1\x29\xf6\x12\x32\x15\xb5\x75\x01\xfb\xd2\x30\xf8\xef\xd1\xab\x19\xc9\xda\x8c\x0c\xfc\xb2\xfc\x8b\x43\x6b\x0c\x84\xa4\x54\x78\x1c\xdd\x06\x0d\x25\xe6\x1f\xf2\x7f\x9f\x6e\xbc\x04\x60\x81\xa5\x5a\x42\x98\xb8\x0f\x78\x9f\x41\xe8\x53\xb5\x1e\x11\x23\x43\xd0\xb0\xcc\x48\xdc\xe6\x62\xf3\x41\x4e\x01\xc2\x9b\x43\x41\x76\x23\xee\xaa\xba\x02\xa3\xad\xac\xb4\x44\xf8\x27\x7f\x34\xd5\x1f\x0a\x0d\x36\xda\xca\xdf\x75\xf5\x47\xb3\xa6\xd6\x28\xf4\xad\xa1\xfc\x5d\xb7\x82\x81\x79\x17\xbd\x83\xcf\xc2\xb3\x2e\xaa\xf8\x29\x6a\xc3\x67\x6a\xc3\xea\x53\x1a\x2d\xf5\x47\xbc\xb1\xda\x38\xb7\xd4\xfe\x46\x7d\x49\x88\xef\x3b\x25\xb1\x63\x88\xb9\x72\x80\xed\xe0\x35\xb8\x09\x60\x5d\xd6\x3e\xc6\x14\xe1\xd5\xcb\xab\xff\x06\x00\x00\xff\xff\xe4\x64\x6c\xc0\xa5\x30\x01\x00") +var _monitoringSystemGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\x6d\x73\xdb\xb8\xb5\xfe\x9e\x5f\x81\xcb\xdd\xad\x9d\xad\xe4\xe8\x35\xb6\x3c\x93\xb9\x13\x27\x9b\xbb\xed\x24\x5b\x6f\xec\x6d\x27\x37\xe3\x51\x21\x12\x96\x50\x93\x00\x03\x80\xb6\x14\x8f\xf7\xb7\x77\x08\x92\x22\xf8\xa6\x57\x4a\xa6\x24\x7c\x49\x2c\x00\x04\x81\x73\x0e\xce\xf3\x00\x38\x00\x1f\x5f\x00\x60\x40\x42\xa8\x80\x02\x53\xc2\x8d\x73\xe0\x27\x01\x60\xd8\x98\x0b\xe3\x1c\x7c\x95\xbf\x40\x98\x2a\x73\x06\x1e\xb6\xc5\xdf\x88\x71\x0e\x9a\xb5\x38\xd5\x82\x02\x72\xea\x31\x13\x19\xe7\xc0\xa8\xd7\xc1\xff\x31\x78\x0b\x09\x04\xf5\xba\xa1\x14\x43\x04\x0e\x6c\xbf\x88\x60\x1e\x52\xd2\x47\xd8\xca\x49\xc5\x26\x25\xef\xa8\x4d\x99\x5f\x27\x1b\x0e\xe0\x71\xa3\x06\x5a\xcd\x66\x0d\xb4\xba\xdd\x1a\x68\xbe\x54\xab\x26\xd0\x91\xef\x7e\x1b\x77\x07\xfc\x05\xbc\xb5\x11\x13\x5c\x2d\x27\x26\xae\x2c\x67\x41\x3e\x1a\x50\xc8\x2c\x23\xcc\x7b\x92\xff\xdf\xbc\x00\xe0\xc9\x2f\x6e\x58\x88\x9b\x0c\xbb\x7e\x4d\x7e\x79\x59\x87\x81\x2c\x2c\x52\x5d\x30\x86\x04\x89\xbf\x59\xc6\x39\x20\x9e\x6d\x07\x29\x0c\xba\xa3\x6b\x4a\x6d\x81\x5d\xe3\x1c\x34\x64\x22\xf6\x8b\x9c\x06\x7f\x0a\xc4\x60\x58\x71\xf3\x75\xa3\xd9\x3b\xed\x36\xbb\xcd\x66\xbb\x27\x73\x6d\x4c\xee\x7c\x55\x7c\xbd\x91\x3f\x5d\x48\x90\xcd\xa7\xca\x88\x54\x61\x98\xd4\xb6\xa1\xcb\x91\x5f\xed\x2d\xb4\xf9\x54\x72\xc6\x90\x61\xeb\x92\xc6\xda\x0c\x44\x9c\xd2\xd8\x83\x71\x0e\x5a\x1d\x25\x61\x1c\xb5\x34\xfc\x3d\xf1\x7f\x47\xb2\x99\xd6\x2d\x7b\x71\x36\xfd\x19\x37\xee\x66\x9a\x26\xb0\x90\x02\x32\xfe\x85\x06\xe0\xad\xeb\x1a\x71\x4e\x28\x7b\x46\x1f\x02\xa9\x87\xf5\x4e\xfb\x04\x6d\x0c\xb9\x54\xb9\x6c\x7d\xfc\xda\x01\x94\x29\xc9\x7e\xfa\x2a\xfc\x88\xc8\x50\xc8\xbe\x35\x12\xe9\x28\xaf\xb8\x6a\xa3\x3f\x2a\x3f\xa7\x45\x6e\xb1\x6d\xab\x72\x2a\x16\xe5\x59\x4a\x94\xcd\xd6\x1c\x51\x36\xf3\x45\xd9\x9b\xfe\xb4\xd1\x10\x11\x2b\xf9\x26\x78\x3f\x4c\x77\xc3\xd7\xbc\xc7\x18\x22\x22\x27\xc7\x81\xe3\xbc\x54\x4c\x72\x52\xf9\x88\x3e\x64\xc7\x9c\xa0\x02\xda\x39\xa5\xef\xa1\xed\xc5\x32\xcd\xf4\xc5\xc6\x44\xe6\xaa\xb5\xc9\xc4\x07\x6c\x89\x84\xe9\xa5\xcc\x5b\x26\xf9\x03\xe7\x92\x62\x22\x3e\x51\xe9\x07\x64\x42\xac\x15\xea\x4e\xbd\x53\xfc\x46\x17\x31\x13\x11\x01\x87\x28\xa3\x68\xd7\xaf\x8a\x41\x0b\x7b\xfe\x33\xad\x64\x7a\xd6\x2e\x18\x22\x16\x62\x48\x7a\x99\x5b\x9b\x8a\xf8\xc5\x1c\x31\x8c\xf8\x3f\xee\x11\x63\xd8\x42\xa9\x46\x73\x17\x9a\x28\xcf\xfc\xb8\x80\xe6\x5d\x5a\x16\x5c\x20\xd7\x45\xd6\x47\x4c\xb2\xed\x15\x90\x0d\x91\xe0\x8a\xbf\x55\x3d\xae\xef\x75\xc6\xae\x6c\x1d\xf7\x9c\x63\x06\x05\x3a\x66\x10\xdb\xbc\xcf\xd0\x37\x0f\x71\xc1\xfb\x52\x69\x8f\xbe\x07\x94\x8d\x7a\x73\xf4\xe3\xf4\xef\xa3\x9a\x4b\xad\x37\x7f\x1e\xf1\x09\x17\xc8\xa9\x43\xd7\xad\x7f\x85\xf5\xef\x8d\x7a\xef\xe6\xaf\xf1\x5f\x47\x4f\x5f\x9b\xce\xcd\xcb\x97\x60\x30\x01\xc7\x5c\x40\xe1\x71\xd5\xb9\xfa\x23\x83\x32\x07\xfa\x26\x67\x08\xec\xa0\x7e\x20\x99\x64\x11\x4c\x04\x62\xf7\xd2\x7a\x8c\xa6\x93\x9f\xf7\x01\x9a\x42\xfa\xf3\x66\x22\x3b\xb0\xfd\x0f\xd3\x77\x3c\x3e\xfe\xfb\xf1\x31\x68\xc7\xd3\xd3\xbf\x9f\x9e\x92\x95\x31\x74\x2b\xfd\xad\xf1\xd6\x98\x26\x3f\x85\x7f\x29\x6e\x68\xc4\x10\x1f\x51\xdb\xca\xb8\x27\x07\x7d\x60\xd4\x51\x1c\xf6\x34\xfd\x33\x1a\x86\x96\x96\x7a\xe0\x6a\x84\x6f\x45\xf6\x89\xd0\xd1\x5d\x49\xe1\x82\x48\x1f\xc0\x45\x0c\x70\x64\x52\x62\x81\xe3\xc1\x04\xa4\x05\x6a\x88\x29\x34\x3c\xaa\xe3\x11\x32\xe9\xcc\x53\x23\x92\x53\x26\x52\xfe\x44\x0e\xc6\x7e\xe4\x4d\x31\xb1\xf0\x3d\xb6\x3c\x68\x1b\x99\x71\x19\x95\x91\x88\x14\x37\x60\x0c\xc7\x38\xe5\xd4\x06\x9e\x79\x17\x18\xa1\xda\x47\xdf\x7b\x84\x63\xd2\x17\x43\x0e\xe0\xa6\x4a\xe7\x7b\x95\xa9\xf7\xf8\x7a\x93\x69\xe2\x04\x8e\xd1\x0c\xdb\xb7\x90\x89\x1d\x28\x31\xa6\x59\x60\x91\x0c\x7d\x73\x53\xb6\x68\xc3\x01\xb2\xa7\xb0\x1d\x27\xd3\xe1\x05\xe4\x28\x53\x57\xe0\x37\x93\x5d\x99\x3a\xce\x4c\xb2\xd2\xc7\xd8\xfe\x6a\xf9\xcd\x8f\x5b\xc9\x47\xbe\x22\x73\x5b\x99\x79\xc3\x06\xdb\x99\x19\x27\x93\xac\x2d\x40\x1b\x0f\xf3\x20\x43\xa6\x7f\x44\xf7\xd3\x46\x27\xa8\xd3\x1e\x83\x79\x22\x61\x06\x9a\x2b\xed\xd6\x70\xae\xe1\xbc\x12\x70\x6e\x52\x22\x18\xb5\x6d\xc4\x9e\x1f\xd2\xe3\xb6\xec\x01\xac\xe7\x09\x56\x43\xbb\x86\x76\xa0\xa1\x7d\x87\xa0\x3d\x3d\x4f\xef\x15\x20\x7b\x53\x23\xbb\x46\xf6\xe7\x47\xf6\x5a\x30\x99\x7c\x73\xd4\x1a\x8f\x35\xcc\x97\x09\xf3\xad\xf1\x58\x43\xfd\xb4\x17\x1a\xea\x35\xd4\x83\xdd\x86\xfa\xd7\x29\xa4\xcf\x4c\xe2\x8b\xa0\xbe\xa5\xa1\xbe\x04\xa8\x4f\xbc\xb6\x22\x48\x9f\x7c\xcb\xae\x40\x7d\x47\x43\x7d\xb9\x50\xdf\xd1\x50\xaf\xf4\x62\xef\xa0\xde\x68\x18\x1a\xe9\x0f\x1a\xe9\xcf\x16\x44\xfa\xb6\x46\xfa\x7d\x9d\xd4\xef\x26\xd4\x77\x35\xd4\x97\x0b\xf5\x5d\x0d\xf5\x4a\x2f\x34\xd4\xe7\x36\x52\x43\xfd\xce\x40\xfd\xfc\x38\xbb\xd3\x02\xac\xef\x68\xac\xd7\x58\xbf\x26\xd6\xf7\x2d\x2f\x08\xdc\xed\x07\x30\xc2\xfb\xdc\x73\x4a\xd9\xa9\x7f\x05\x16\x7d\xa3\x49\x3d\x22\xca\x78\xe7\xca\x84\x62\x41\xd2\x70\x2d\x0d\xbc\xda\x2c\xe1\xb3\x2f\x6b\x00\xef\x11\x83\x43\x04\x18\xe2\x2e\x25\x1c\x81\x04\x9e\x6a\x4a\xb0\x1a\x25\xc8\x80\x9e\x02\xb8\x7a\x9d\xbf\x6a\x94\xc0\x84\xcc\x4a\x55\xec\x27\x5d\x42\xcb\xc2\x64\x98\xb5\x1c\x3f\xf3\x33\xf5\x88\x95\xaa\x7c\xda\x52\x33\x3c\xb9\x91\xaa\x70\x7a\xa0\xe3\x87\xee\x69\xaf\xf3\xa1\xa5\xda\xa8\x7c\xe4\xca\x84\xc1\xd8\xe4\xdf\x12\x1a\x88\x72\x47\xc8\x09\xc7\x91\x40\xcc\xa5\x36\x14\xe8\x42\x9a\xab\x52\x14\x8d\x5d\x4a\x02\xe0\x6e\x9c\x74\x73\x46\x07\x75\xa1\x89\xc5\x24\x3b\x02\x7d\xb6\x12\xbb\x30\xc1\xa3\x71\xb6\x0c\xbb\xc9\x3b\x39\x02\xd6\x63\x39\xd9\x00\xc4\x2c\xcd\x19\x21\x28\x1c\xe8\x26\xc1\x7b\x84\x2d\xf4\xff\x88\xd1\x8b\xa9\xbf\x48\x02\xe2\x08\x0f\x47\x36\x1e\x8e\xc4\xbb\x50\xff\x09\x5e\x11\x30\xa7\xee\x4c\xe6\x14\xda\x6d\x21\x5b\x49\x53\x90\x5c\x8e\xc1\xd0\x3d\x62\x1c\x7d\x29\x6a\xe6\x92\xb8\x3d\x13\x40\x03\x95\x2e\x8c\xa0\x27\x3f\x1f\x3d\x05\x93\x71\x1b\x15\x62\x66\x24\xfb\x35\x26\xe0\x8d\xf9\x33\x70\x1b\xad\x3e\xf3\x9e\x81\x9d\x73\x20\x32\x67\xf6\x1c\x76\x57\xce\xa2\x13\xa0\x09\x02\xe1\x02\x4c\xc2\xa2\x0b\x84\xbd\xe7\x21\x90\x9f\xfa\x2b\xe6\x82\x0e\x19\x74\x0a\xad\x2b\x82\xcb\xb4\xf4\x8d\xf1\xdb\x8c\x97\xcc\xba\xd7\xb8\x9e\x71\x60\x76\xbf\x79\xce\x40\x92\xd3\x84\x20\xc2\xcc\x2b\xfc\x3d\x8d\x9f\xc6\x24\xfb\x1a\x05\xff\x54\x0a\x90\x0f\x7d\xf9\xc8\x91\x8b\x1b\xb9\xa8\x51\x24\x3c\xd7\xc6\x62\x6a\x58\xb9\xce\x79\x12\x74\xea\x22\x74\xe0\x06\xf4\x04\x35\xd2\xb9\xf9\xf2\x98\x64\xe4\x91\x8f\x27\x5b\x38\x9e\xd6\x2e\x58\xd7\x6d\xc5\x53\x84\x99\x07\xd4\xae\xb0\x85\xee\xf0\xb7\x83\x38\xa0\x36\x57\x98\x9d\x82\xc3\x7e\xad\x39\x91\x6f\x3e\xcb\x78\xcb\xaf\xf3\x0f\x76\x06\xd3\xea\x54\x62\x3c\xab\x4e\x65\x2c\x33\xa9\x66\x3e\x68\x5d\xe5\x9e\x1b\x5d\x71\xbe\x9d\xeb\x1a\xf4\x74\x7b\xbb\xd3\x6d\x1e\x0c\xc9\xfe\x7f\xe8\xc0\x9f\x5c\x9b\x26\xe2\xb3\x17\xd8\x57\x9d\xcc\x86\x47\x8e\x53\xd6\x90\x87\xcc\xad\x99\xc0\x7c\x15\xb4\xf1\xd6\xb3\x81\xdf\xe6\x05\xb0\xb9\x60\xee\x31\x5b\x14\xb7\x10\xdb\xc8\xda\x88\x24\x96\xec\xf0\x07\xd9\x92\x59\x9d\xbd\xa8\xc8\xe4\xfe\xef\x74\xc0\x81\xcb\xa8\xaf\x1f\x64\x01\x9f\x62\x4e\x42\x5a\xb2\xde\xe4\xbe\x95\x76\x1f\x07\x32\xb9\x8f\x8d\x89\x16\x2d\xea\x97\x33\x39\xd6\xcb\xfa\x6a\xba\x5e\xd6\x3f\x07\x9d\xb3\x02\x76\x32\x27\x58\xaf\x44\x76\xe2\x23\xc6\x2f\x8e\x2b\x26\xf9\x59\xfe\x2c\xfb\x50\x29\x0d\x80\x1c\x7c\xf7\xfb\xbf\xeb\xd4\x66\x83\x47\x01\x56\x65\x36\x72\xe9\xe1\x9b\x87\xbc\xe2\xd5\x87\x8d\x93\x9c\xcb\x29\x88\x62\x02\xe4\x52\x84\x6c\xd0\xa2\xab\x11\x1b\x66\x3c\xeb\x48\x68\x35\xf2\xb3\xb0\x14\x2a\x4a\x85\x06\x13\x20\x9b\xae\x69\x90\xa6\x41\x9a\x06\x55\x98\x06\x65\xd6\xfd\x0b\x79\xd0\x9c\x50\x46\xcd\x83\xb6\xc1\x83\x34\xff\x99\xca\xbe\x5c\xfe\xf3\x40\xd9\xdd\xba\x31\x90\x6b\x01\xbf\x84\xfb\xa0\x15\xbb\x10\xfd\x98\xc5\xfb\xff\xd0\x01\x90\xd0\xab\x21\x5f\x43\xbe\x86\xfc\xea\x42\x7e\x7a\xe5\xa3\xfb\xba\x00\xf1\xe7\x04\x34\x2e\x80\xf8\xc5\xe1\x8e\x8b\x43\xbe\x06\xf6\x03\x05\x76\x07\x8e\x93\x98\xfe\x00\xb1\xc0\x64\x38\x33\x40\xf1\x69\x9b\x93\xf5\xe5\x17\x2a\x9e\x13\xb2\x7f\xf7\x1b\x0a\x38\xfe\xae\xf4\x5a\x63\xf4\x4a\x18\xbd\x71\xf8\xd3\x28\xad\xa6\xef\x0c\x4a\xa7\x02\xf3\x82\x11\x67\x43\x81\x88\x39\x01\x98\x03\x31\x42\xc0\xc2\xb7\xb7\x88\x21\x62\x22\x25\x88\x09\x70\xec\x27\xf8\xf9\xd4\xb6\x10\x17\x92\x4e\x63\x22\x53\xa4\x87\x01\x0f\x90\x03\x44\xe4\xdf\xd6\x09\xf8\x07\x03\x23\xfa\x00\x6c\x4a\x86\x41\x51\x0e\x42\xf7\x08\xb0\xe0\x40\x78\x8c\x00\x41\xc1\x00\x01\x34\x46\xa6\x27\x90\x75\xb2\x9d\xf5\x83\x42\x36\x31\x3b\xc8\x4f\xb3\x09\xcd\x26\x36\x1b\x01\xa2\xd2\x09\x39\x8a\xfa\xe1\xc0\xd4\x4c\x62\x35\x26\x11\x88\x0f\x6b\x36\xb1\x36\x9b\xd0\x4c\x42\x33\x89\xed\xcd\xf7\x5f\x77\x0a\x10\xfa\xf5\x6c\x84\xd6\x07\x18\x35\x9e\x2a\x78\x3a\x8a\xa2\xe7\xfb\xdf\x3c\x48\x04\xb6\xd1\x71\xe3\xa4\xd7\xad\x81\xbc\x05\xf9\x3e\xf3\x48\x88\x95\x0b\x9c\x91\x50\x17\xe6\x6d\xf4\xf2\x25\xd8\x34\xfc\xf6\xba\x3f\x81\xa8\x17\x80\xde\x82\xb0\xe5\xa0\xe2\x38\xfc\x77\x3a\x88\xd6\xe0\x7d\xe2\x1f\x9d\x32\xd4\x87\x0b\x35\x26\x6b\x4c\xae\x34\x26\x67\xa6\xcd\x85\xa0\x7c\xaa\x41\x59\x83\x72\xd5\x40\x39\xd8\xa7\xae\x81\xad\x80\xf3\xee\xed\x8c\x17\x81\xb2\xde\x26\x07\x1a\xa0\x35\x40\x87\x89\x95\x06\xe8\xf4\xa4\xf9\xb4\x55\x80\xcf\x67\xb3\xf1\x79\x3b\x61\x71\x7a\x55\x7b\x27\x00\x7f\x5b\xc1\x6f\xd1\x36\x51\xc5\x62\xdb\x77\x6b\x91\x3b\x3c\xbb\x2d\xcf\x21\x02\xd7\xe3\x23\x1d\xd6\x5e\x1e\x78\xeb\xfd\x73\x0d\xe0\x5b\x9e\x61\x17\x22\xf8\x9c\x2f\xe4\xee\xe4\xc6\x74\xfa\x09\x8d\xe1\xd5\xc5\xf0\x4c\x9c\x1b\x43\x82\x4d\xe6\x44\xb9\x69\xac\x4e\x5e\x21\x24\xe4\xc9\x7b\xa1\x97\xbd\x35\x30\xef\x23\x30\x6f\xe3\xb2\xa3\xb3\x82\xfb\x79\xda\xb3\x2f\x3b\x62\xc8\x45\x81\x56\x2c\xe4\xda\x74\xe2\xf8\xc8\x97\x1d\xa2\x97\xd4\xe2\xe0\xf8\xc7\xb8\xcc\xcb\x25\x6e\x45\x32\xa1\x39\x42\xd7\xd8\x41\xd4\xcb\x38\x01\x79\x6b\xdf\x05\x34\xef\x86\x2c\xbc\x6c\x2a\xe1\xa2\x65\xf6\x3f\xfd\xb1\x91\x11\x9d\x19\x11\x9d\x78\x58\x18\x3f\x7c\x68\x75\x7a\xdd\x77\xea\x20\x64\xc3\x01\x3c\x6e\xb5\x4f\x6b\xa0\xd9\xea\xd5\x40\xa7\x51\x03\x8d\x93\xb3\x9e\xea\x82\x8d\x1f\x5a\xbd\x9e\xd9\x79\x6d\x64\x2c\x62\xa1\xa0\xc1\xec\x8d\x5e\x8a\xad\x13\x4a\x94\xc2\x43\xe8\x49\x6c\x7c\x4c\x30\x87\xa8\x7f\xcd\x46\x23\x49\x1e\xa2\x8c\xc6\x02\x14\x21\x72\xd7\x1f\xfd\x91\x94\x01\x54\xb5\xc4\x27\xc8\xee\x10\x2b\xa4\x0d\x85\x26\xd9\x4e\x99\x64\xfa\xb3\x0a\x19\x8b\xcc\x7e\x05\xd9\x67\x50\xbe\x21\x44\xf0\x9d\x7b\xa1\x60\x3b\xe6\x25\xb9\xe7\xb3\xf2\xd8\x8a\x03\x5d\x17\x93\xe1\x75\x60\x8a\xcd\xbc\xf4\x19\x1e\x34\x74\xd4\x81\x13\x06\x82\x02\x81\xc6\x29\x07\x75\x1f\xe9\x68\xae\x9f\x8b\x2a\x63\x90\x0c\xe7\x54\xd6\x9a\xe1\x8c\x1c\x38\x7e\x0f\x05\xbc\x8c\xf8\x91\x62\x1c\x59\x6e\x66\x52\x42\x90\x29\x90\x72\x71\x8d\x5f\xe6\xda\x7f\x73\x6a\xc0\xe5\x13\x37\xdb\x1b\x62\xf2\x4f\xc4\x78\x18\xfb\xfa\xfa\xa4\x75\xd2\x31\x14\x96\xc6\xc5\x2d\x1e\x27\xd5\x10\x26\x7e\xa0\x24\xba\xed\xcd\xe8\x36\x7e\x52\xf2\x19\xca\x3e\x23\xd3\x0a\x1f\x91\x32\xfb\x04\xdd\x59\x68\x17\x70\x8f\x24\x1d\x95\x39\x22\xe8\xad\xf1\xdb\xab\xb7\xa9\x0c\x3a\x7d\x60\x86\xc0\xb9\x0b\xd9\x9d\x1d\x70\x41\xc5\xf2\xfd\x49\xcd\xf4\x52\x52\xe9\x4c\xda\xcd\x1a\x68\x36\xcf\x6a\xa0\x79\xd6\xf3\x9d\x49\xf3\x2c\xe1\x4c\x6e\x3d\x3b\x8f\xab\xfb\x35\xab\xf5\x04\xd5\xb4\x1a\x35\xd0\xec\xb5\x13\x15\xcc\xbc\x31\x53\xf8\x33\x99\x77\xd4\xf6\x9c\xd4\xe5\xa1\x4b\x2d\x3a\xdd\x79\x03\xd4\x67\xc8\xb5\xb1\x29\x2f\xbf\x8c\x3f\x75\xd0\x0f\x3e\x35\xd1\x67\x08\x5a\x93\xa8\x08\x2f\xba\x0d\x33\xb7\x8a\x37\x7f\x1e\x29\x40\x51\xff\x1a\xde\x2a\x5d\xfa\x85\xd2\x2b\xf0\x58\xa3\x59\x6b\x19\x79\x54\xd6\x68\x37\xb8\x91\x4b\x59\xd3\x39\x11\x67\xf5\x08\xc1\x64\x08\x5c\x6a\xf1\x2c\x14\x72\x4c\x86\x36\xf2\x45\x19\xe7\xc9\x41\xaf\x5a\xfe\x99\x6a\xf9\x32\x77\xb6\xe5\x53\x9f\x0e\x1b\x6f\xf2\x8d\xbe\x91\xef\x61\xe6\x5a\xbd\x2c\xf8\x5b\xe8\xb2\xfc\xf9\xf0\x46\x10\xfc\x32\x72\x05\x39\x10\xbe\x04\xba\x87\x30\xbd\x2c\xba\x87\xa4\x40\xa3\xfb\x3a\xe8\xfe\xba\x2c\x74\x6f\xe5\xa1\x7b\xc2\xa2\x34\xbe\x97\x8e\xef\x1a\xbf\x0f\x06\xbf\x5d\x64\x96\x8b\xdb\xa0\x0e\x34\x67\x28\x83\x33\xfc\x41\xe0\x3d\xc4\xb6\x6f\x01\x9a\x37\xe8\x99\xff\xfe\x70\x83\xcc\x6e\xcd\xca\xe4\xa0\xad\xa7\xfe\x7a\xea\xaf\xa9\xc3\x86\xa8\x83\xdc\x94\x3a\x8e\xfe\x25\x02\x62\x82\x58\xdf\x41\x0e\x65\x93\xbe\xc7\xe1\x10\xf5\x07\x13\x81\x0a\xa1\x3b\xf8\xf8\x45\x02\xa8\xf3\xbe\x1f\x15\x44\xa5\x10\x6a\xa1\xf2\xbf\x22\xf5\xac\x00\x2e\x57\xc1\x2d\xcc\x05\xc3\x03\x4f\x20\x0b\x50\x02\x46\x94\x0b\x8d\xe4\x65\x22\xf9\x8a\xb3\x7c\xab\xd3\x81\x6d\xa8\x91\x7c\x3d\x24\x4f\x7f\x1a\x79\x65\x24\xef\xe8\x69\xfe\xf6\xb1\x5c\x63\xf5\xde\x60\xb5\x8c\x2b\xf1\x9c\x63\x0b\xd9\x02\x06\x93\x6f\x97\x5a\xfd\x18\xb6\xa7\x93\x6e\x2e\x20\x5b\xec\x33\xd3\x73\x91\xfb\x6b\xd7\xb9\x79\x19\x46\x95\xba\xd4\xda\xf8\x37\x20\x0b\x02\x53\x16\xbf\x16\xb9\x3c\x6c\xff\x04\xc7\x72\x52\x0e\x22\x81\x82\x63\x1b\x72\x01\xba\xc0\xc1\xc4\x13\x88\xe7\xec\x78\xef\x1b\xc8\xef\x4e\x00\xe1\x69\x0a\xc7\xe6\x87\x47\x14\x9c\xd0\x6b\xcf\xb9\xd8\x66\xc9\x13\x7a\x1b\xbf\xfc\xb6\x42\xc7\xfa\x1c\x38\xbe\x44\xec\xb3\x6c\x4f\xbb\x18\xd4\x36\xf1\x01\x80\x6e\x32\x7d\x99\xf8\xc1\x69\xc0\x4b\x02\x4e\x83\xd4\xf7\x98\x21\x33\xbc\xfe\x29\x91\x5d\xb9\x93\x82\x5b\x5f\x04\xde\x74\xc4\xa2\xb4\xe0\x7a\x62\x55\x54\x29\xf7\x11\x93\xbb\xbc\xb8\x2e\x65\x36\x98\x48\xf7\x65\x2c\x75\x31\x97\xe6\xed\xdd\xae\x78\x5a\xb0\x72\xc9\x39\x47\xb0\xb9\x60\xbb\xa6\x90\xf4\xd6\xc3\x12\x8a\xf1\xc8\x5c\xd5\xbc\x5b\x5c\x35\x7b\xb0\xb4\x93\x11\x10\x47\x56\x3d\xb9\xbc\x92\x14\xcf\xfb\x8a\xc4\x2f\x5f\x52\x0b\x48\xc9\x83\x63\xe9\xc6\x6a\x40\x6a\xb6\x06\x3c\xe2\xff\xff\x12\x40\x62\x05\xdc\x52\xf6\x26\x5e\x41\xf2\x91\x26\xae\x4e\x9f\x45\x5a\xe5\x33\xe2\x8d\x02\x2b\xdc\x78\x90\x71\xf2\xc5\x3b\x16\x09\xad\xf2\x41\x7d\x46\x29\x77\x8a\xf1\x7a\xd9\x29\x46\x2f\xbb\x54\x16\x4c\x31\xca\xbd\x99\x4b\x4f\x31\xaa\x7a\x34\x69\x7f\xa6\x16\xcf\xb0\xe8\xb4\xb5\x35\x27\x79\x36\xca\xa5\xd6\x2e\x9c\x8c\xba\x5c\x74\x41\x4a\x1f\x97\xd2\xc7\xa5\x4a\x27\x09\xfb\x7c\x5c\xaa\x77\x5a\x00\xd6\xf1\x8d\x5d\x6b\x1c\x97\x7a\x77\xf9\x07\xf8\xc3\x9f\x74\xad\x7c\x66\x6a\x77\x78\xd2\xd2\x4b\xb1\xbd\x82\x6f\x94\xb5\xe7\x5c\xc6\x72\x18\x97\xa5\xed\xf7\xaa\xaa\xa1\x60\x46\x25\x99\x0f\xa1\x16\xea\x4f\x39\x4c\x92\xfd\x9c\xc7\x3c\xc8\x74\xbd\x70\x55\x25\xba\x7d\x4d\x1a\xcb\xf9\xe3\x23\x38\xb9\xf2\x9c\xcf\x50\x20\xf0\xf4\xa4\x10\xa3\x3f\x57\x5f\x6d\x29\x87\x14\x2d\x70\x60\x3c\x9f\x14\x95\xb6\x02\xfb\x9c\x44\x6a\xea\x90\xf5\x7a\x4b\xc5\x39\x93\x5e\x56\x51\xd2\x2b\x45\x99\x9a\x8d\x6e\x01\x70\xf7\xca\xe2\x4c\xbf\x7b\x54\xc0\x2d\x71\x26\x53\x46\x67\xa4\xda\xba\x05\x22\x75\xab\xc4\x06\x34\x1b\x6a\x70\x40\x79\x14\xab\xd9\x28\xf8\x8e\x4b\xa7\xa1\x39\xd6\x46\x38\x16\x1c\xa2\x50\xa9\x09\x00\xaa\x2c\xf7\x32\x19\x95\x76\x99\xbc\x10\x67\x26\x23\x1b\xd1\x87\x5f\x11\xb4\x64\x13\x92\x8f\x05\xe0\xa8\xd8\x91\x49\xed\x94\x87\xb1\x10\x37\x0b\xb5\x59\x22\xd9\xe3\x62\x62\xcf\x42\x32\xe9\x23\x7c\x61\x5c\x27\x91\x34\x18\xcc\x28\x66\x44\x5f\xbe\x7c\xf9\x52\xff\xf4\xa9\xfe\xfe\x3d\xf8\xf5\xd7\x73\xc7\x39\xe7\x29\x86\xe5\x42\x21\x10\x23\xf9\x75\x45\xbe\x6a\x84\x2d\x0b\x91\xf9\xdb\x78\xd3\x66\x65\x89\x4a\x24\x50\xca\x42\xbb\xcc\x80\x4f\x1c\x45\x7b\xb3\x4e\x87\x94\x4d\x95\x14\x59\x0c\xb8\x5f\x6a\x88\x86\x19\xd7\x53\x1a\x65\xbc\x67\xd8\xb6\x81\x45\x1f\x88\x91\x29\xf6\x07\xcb\xf9\xc8\xae\x22\x42\x19\x26\x0b\x7e\x48\x87\x04\xe6\x33\xc4\x84\x88\x89\xe7\x0c\x10\x4b\x3e\xe7\x11\xac\x10\x81\xe5\xa4\xff\x19\x7d\xf3\x50\x66\xd3\xf1\x50\x14\x70\x51\x1d\x05\x80\x9f\x0e\x53\x05\xef\xca\x55\x41\x08\x41\xf2\xe7\x72\x8a\xf8\x88\x1d\x7c\xa8\xe3\xe0\xfd\xf3\x8f\x83\x40\xfc\x87\x3a\x0a\x7e\xa9\xc2\x28\xb8\xa4\x56\xf5\xa4\x9f\x64\xd0\x2b\x09\xff\x95\xf5\xea\xf1\x11\x9c\xfc\x16\xad\x48\x81\xa7\xa7\x4c\x42\xbd\xcd\x4d\x68\xa3\xfa\x9d\x37\x40\x8c\x20\x81\x78\xdd\xa4\x8e\xeb\x09\x54\x67\x28\x98\xe5\xf0\xba\x4b\xad\xff\xbd\x87\xac\x1e\x2f\x74\xc5\xcb\x5c\x7f\xf1\x33\x5c\x6a\xbd\xf9\xb1\xdf\x37\x51\xfa\x18\x80\xa2\x71\x37\x2d\xe5\x6d\x8f\xb6\x0a\xe9\x58\x11\xcb\xab\x93\x9f\x5f\x2d\x2f\x17\x2e\x18\x26\xc3\xc5\xe4\x12\xfe\xa5\x2c\xa4\xed\xf3\x52\xa9\xbc\x03\x36\x91\x89\x09\x17\x30\xe7\xa6\xd7\xe5\xd7\x4f\x0b\xf7\x91\x13\xe9\x6b\x84\xa5\x26\xb7\xe1\xa3\x01\xd8\x67\x21\x57\x92\x52\x36\x29\x4b\x04\xf8\xed\x91\x20\x2f\xca\x10\x64\x45\x0d\x16\xbc\x02\x5a\xcd\x51\xc0\xeb\x86\xc7\x8b\x2d\x39\xd5\xbe\x8b\xf1\xfd\x61\x8f\x96\xc3\x50\xf2\x2f\xdb\xd9\x70\x9b\xbf\xaf\x26\x17\xed\x9f\x25\x16\x89\x41\xc2\x7d\x25\x64\x55\x30\xa5\x43\xa9\x64\xbd\xe7\x76\x28\x7b\x6e\x2f\xc0\x73\x6d\x93\x35\xdb\x19\x55\x07\x7b\x2f\xf1\x9e\xc4\x1a\xdb\x64\x9f\xe4\x91\x0e\x1d\x5d\x54\x2c\xfe\x82\x93\x9e\x9d\x38\xb2\xf7\x80\xb7\xbe\xaa\x1e\x48\x54\xc9\xa0\xa0\x05\x0f\x54\xad\x42\x2e\x6a\x60\x5a\xf9\xff\xbc\x39\xd2\x21\x3f\xe5\x85\xfc\xa8\x8e\x52\x47\xfd\xac\xc7\x40\xa4\xa5\x6b\x06\x72\x90\x51\x3f\xad\x82\x63\x4d\x9d\x76\x89\x74\x46\x07\xfe\x94\xc4\x7e\x5a\x05\xdf\xc9\xea\x74\x34\xfb\xd9\xdd\x10\x9f\xe7\x0a\xd7\xc9\xbd\x5c\x4c\x47\xeb\x4c\x9b\x95\xcb\x31\x22\x91\x3e\xfb\x46\xe1\x0e\x06\xec\x58\xc8\x0c\xb8\xc6\xd2\x3a\xd0\x61\x3b\x95\x52\xc3\xa1\x86\x2d\x54\x22\x78\x27\xd4\x85\x8e\xdf\x29\x4d\x0f\xab\x0f\x08\x1d\xc5\x53\xa2\x1a\x74\x14\x8f\x8e\xe2\xd1\x51\x3c\xa5\x46\xf1\x54\x6b\x6d\x73\xaf\x63\x74\x42\x01\x97\x21\xda\x8a\xca\xb2\x94\x30\x9d\x67\xb1\xc8\x65\x82\x70\xf6\x5f\x8f\xdb\x8a\xc3\xd9\x7f\x49\x96\x12\x8a\x53\xd5\x11\x71\x30\x5a\xac\x44\xac\x8d\xba\x54\xae\xc3\x6d\x74\xb8\x4d\xa5\x36\xbb\x5e\x80\x67\xdb\x9f\xea\x65\x54\x1d\xec\x78\x28\xeb\xed\xab\xef\x4f\xfd\x86\xc4\x03\x65\x77\x3a\xde\xa6\x50\xfe\xed\x82\x0f\xcf\x77\xca\xbd\xf6\x70\x9f\x76\x9c\xca\x8a\xb3\x59\xe3\xae\xc2\x4a\xc6\xd9\x60\x06\x05\x52\xd8\x0e\x09\x06\x5f\x9f\x21\x13\xe1\xfb\x90\xf0\x64\xee\x1c\x5c\x89\x67\x94\x7e\xe9\xe0\xca\xc1\x36\x95\xbb\x74\xf0\x73\x20\x6d\x70\x01\x89\x15\xd8\xf8\x5a\x64\xe3\x50\xa3\x67\x94\xc5\xa1\x0c\x96\xc6\x06\x76\xe1\x6e\x34\xae\x66\xc7\x6f\x20\xd4\xd7\x14\x6f\x02\xb0\x0b\xae\x3e\xec\x9c\x6a\xc0\xd6\x80\x5d\x0a\x60\xcb\xa9\xa2\x83\x85\x46\xec\xad\x20\xf6\x75\x28\x6e\x0d\xd9\x65\xad\x01\x68\x58\xae\x24\x2c\x6f\x65\x41\xa1\x53\x70\xcd\x9d\xfa\xe9\xc6\x35\x56\x14\xde\x9a\xc2\x67\xd7\x9f\x91\x49\x99\xb5\xad\x90\xd7\x19\x5f\x6f\x7d\x4e\xf2\x72\xb6\xbc\x72\x8a\x2e\xb6\xd3\x97\x07\x6f\x80\xbc\xcc\xfd\x36\xf8\x3e\x93\x1b\xe9\x4b\xe5\x37\xa5\xa0\x89\x8e\x19\xc4\x36\xef\x87\x5f\x68\xc5\x94\xf4\x5d\x4a\x6d\xe5\xf7\xda\x3b\x1f\xb5\xa3\xb8\xb2\xa3\x1a\x38\xfa\xb1\xe9\xff\xcb\x05\x14\xc8\xff\xe3\xf8\xe4\xe7\x97\x47\x21\xff\x89\x0b\xae\x4e\x83\x64\x14\xba\x63\xd4\xc0\x4c\x8e\xb4\xc0\xa7\x14\xe2\xc6\xec\x02\x55\xfa\xec\xab\x11\x5c\x52\x0b\xbc\x8b\x55\xf7\x2c\xbb\x29\x3b\xcf\x96\x16\xf8\x08\x93\x4d\x4d\x98\xde\xac\x8b\x28\x4a\x6a\xaf\xed\x10\x79\xd4\x73\x7d\x60\x61\x77\x56\x37\xd2\x04\xa1\xd9\x9a\x47\x10\xba\x05\xc7\x7f\xbb\xcf\x7e\xfc\x37\x9d\xbc\x17\x0c\x61\x9f\x19\x00\x28\x00\x7d\x8e\xbf\xa3\xb5\xe3\x1c\x56\x00\xee\x28\xc8\x21\x1b\xb8\x9b\x06\xf5\x35\x30\x7d\x27\xf0\x3b\xc6\x6e\x70\x49\xa9\x0d\xe4\xb9\x2c\x0d\xe2\xeb\x81\x78\xf3\x99\x40\x7c\x31\x74\xd4\x28\xbe\xc7\x28\x9e\x48\x98\x0d\xe3\x4d\x0d\xe3\x1a\xc6\x4b\x81\xf1\x07\x88\x05\x26\xc3\xe7\x40\xf2\xc3\x46\xeb\x7f\x05\x82\xd7\x80\xad\x67\xdd\x7b\x84\xd7\xdd\x24\x58\xbf\x08\xeb\xf4\x07\xa5\x3f\xde\xe4\x60\x6f\x04\x9e\xc0\xe0\xe6\x08\x39\x30\x5e\x58\x6d\x9e\x05\xc9\x62\x12\x8c\x1b\x0b\xb2\xbb\xa0\xa4\x80\xc3\xd8\x18\x8c\xe0\x30\x57\x28\x34\x83\x4f\xb8\x40\x8e\x31\x7d\x93\x40\x8e\x6b\x43\x39\xb4\xa2\xe6\x1b\x36\xe6\x42\x31\x26\x15\x98\xb1\xb4\x63\x75\x1c\x61\x62\xda\x9e\x85\xde\xda\x79\x78\x96\xaf\x1c\xc3\xf1\x6c\x81\x73\x8a\x87\xc3\xc1\xc8\x61\x0e\x09\x80\x52\x0f\x40\x19\xdf\x3c\xc4\x26\xf2\x30\x19\xa3\x0e\x12\x23\xe4\xa9\xae\x53\x11\x65\x33\x91\x3a\x44\xe3\x94\x69\x1b\xfc\x0e\xbb\x7f\x30\xfb\x6a\x42\xcc\x3c\xcc\x0e\xdd\x80\xd2\xb8\xf4\x38\x4c\xa8\xdf\x96\xc7\x19\xb3\x9d\x8f\x89\x45\xc2\xc8\x23\xa5\xa5\xce\x76\xa1\x71\xb8\x6a\x9a\x3c\xa2\x97\x1c\x03\xf7\xe1\x9b\xb2\xc5\xf2\xc6\xd6\xca\x5a\x34\xa6\x40\x67\x2c\xa1\xcc\xdc\x87\x14\x5d\x2a\x1d\x51\x05\x22\x89\x81\x8d\x4c\x91\xe3\xcf\x17\x17\xcd\x62\xc2\x89\x87\xb4\x32\xa8\x55\xd3\x9a\xf1\x8e\x05\xad\xc6\xf4\xb8\xa0\xce\x82\x16\x93\xb0\xca\xf9\xac\x5a\x7a\xf8\x5b\x4c\x70\xf4\xd1\x9e\x60\x17\x22\x00\x8c\xf8\x30\x06\x26\xb7\x74\xf6\x67\x97\x03\xe7\x50\x3f\xf9\xf9\xe8\xa9\x06\x52\x01\x13\x73\xad\x26\x05\x57\x53\xa3\xc9\xd9\xd8\x5c\xc4\x05\xe4\x3e\x35\xcf\x05\x6c\xa4\xe3\x73\x3c\xc8\xab\x63\x9f\xd8\x85\xfc\xee\xe6\xaf\x2f\xeb\x5f\x7d\x92\x97\x38\xc0\x39\xc7\x46\x42\x76\xa2\xd6\x2e\xe0\x50\x1a\x03\xff\x3d\xea\x9a\x91\xcc\xcd\xc8\xc0\x4f\xcb\x2f\x1c\x5a\x60\x20\x24\x25\xc3\xe3\xe8\x3a\xa8\x28\x31\xdb\x90\xff\xfb\x14\xe3\x29\x00\x08\x2c\x15\x12\x42\xc3\x6d\xc0\xf5\x0c\x42\x1f\xea\xcd\x88\x0c\x19\x82\x86\x69\x46\xe2\x31\x17\x9b\x77\x92\xf0\x87\x0f\x87\x82\xec\x47\x7c\x55\x1d\xfe\x46\x57\xd9\x51\x89\x30\x4f\xfe\x68\xab\x3f\x14\xea\x6b\x74\x95\xbf\x9b\xea\x8f\x76\x43\xcd\x51\x28\x5b\x4b\xf9\xbb\x69\x05\x83\xf1\x26\xea\x83\xcf\xbc\xb3\x6e\xa9\xf8\x2d\x6a\xc5\xaf\xd5\x8a\xd5\xb7\xb4\x3a\xea\x8f\xf8\xb0\xb3\x71\x6a\xa9\xed\x8d\xda\x92\x10\xdf\x77\x4a\x62\x67\x10\xf3\xe3\x00\xcf\xc1\x2b\x70\x15\x40\xb9\xcc\xbd\x8f\x69\xc1\x8b\xa7\x17\xff\x0d\x00\x00\xff\xff\x64\xce\x85\x85\x85\x2f\x01\x00") func monitoringSystemGrafanaDashboard1JsonTplBytes() ([]byte, error) { return bindataRead( @@ -204,7 +204,7 @@ func monitoringSystemGrafanaDashboard1JsonTpl() (*asset, error) { return a, nil } -var _monitoringZyncGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\xeb\x72\xdb\xb8\x92\xfe\x9f\xa7\xc0\x72\xe6\x1c\x3b\x59\xc9\x96\x64\xcb\xb7\xaa\xd4\x56\x9c\x4c\x76\xce\x56\x92\xf5\x38\xce\x54\x65\x52\x29\x1d\x88\x6c\x49\x58\x93\x00\x03\x80\xb6\x14\x97\xe7\xd9\xb7\x08\xde\xc0\x9b\x24\xdb\xb4\x2d\x5b\xf8\x31\x13\x0b\x20\x41\xb0\xbb\xd1\xdf\x87\x46\x83\xb8\x7a\x81\x90\x85\x29\x65\x12\x4b\xc2\xa8\xb0\x8e\x50\x58\x84\x90\xe5\x12\x21\xad\x23\xf4\x4d\xfd\x42\x71\xa9\xaa\x19\x06\xc4\x95\xff\xa2\xd6\x11\xea\xb6\xb2\x52\x07\x4b\x2c\x58\xc0\x6d\xb0\x8e\x90\xd5\x6e\xa3\xff\xe6\x78\x84\x29\x46\xed\xb6\xa5\x5d\x06\x14\x0f\xdd\xf0\x12\xc9\x03\xd0\xca\x27\xc4\xa9\x28\x25\x36\xa3\x6f\x99\xcb\x78\xd8\x26\x1f\x0f\xf1\x66\xa7\x85\x7a\xdd\x6e\x0b\xf5\xfa\xfd\x16\xea\xbe\xd4\x9b\xa6\xd8\x53\xcf\x7e\x93\xbd\x0e\xfa\x27\x7a\xe3\x02\x97\x42\xbf\x4e\xce\x7c\x75\x9d\x83\xc5\x64\xc8\x30\x77\xac\xb8\xee\x5a\xfd\xfb\xfd\x05\x42\xd7\xe1\xe5\x16\x38\x44\x16\x7a\x6b\x8d\x29\xc8\x7f\x39\xd6\x11\xa2\x81\xeb\x46\x25\x1c\xfb\x93\x33\xc6\x5c\x49\xfc\x44\x26\x16\x91\xc0\x55\x17\xd2\x12\x97\xd0\xf3\x50\xbc\xdf\xbe\xab\x9f\x3e\xa6\xe0\x8a\x54\xc0\x89\x78\x2d\x9b\xb9\x2e\xf6\x05\x84\x8f\x18\x61\x57\xa4\xd2\xb0\xc6\x9c\x38\x27\x2c\xd3\x50\x24\xb6\x82\x16\x2e\xad\x23\xd4\xdb\xd5\x0a\xa6\xd6\x11\xea\x68\xbf\x67\xe1\xef\xe4\x7d\xd3\xb6\x49\xf8\xb8\x83\x7e\xfa\x3b\xeb\xdd\xf7\xb4\x4c\x12\xa9\x44\x61\xfd\x35\xa3\xb6\x95\x15\xc7\xd2\xe4\xec\x32\x92\x63\xdc\x6a\xfa\x46\xd8\x25\x58\x28\x25\xaa\xbe\x67\x0f\x1d\x62\x55\x92\x7f\xcb\x50\x29\x1f\x80\x8e\xa5\x7a\xb3\x4e\xae\x1c\xaa\x2e\xd7\xad\xee\x57\xed\x67\x7a\xc9\x88\xb8\xae\x2e\xa5\x7a\x41\x1e\x14\x04\xd9\xed\x2d\x10\x64\xb7\x5a\x90\xfb\xfb\xe9\x6f\x17\xc6\x40\x9d\xfc\xa3\xf0\xc5\xb8\xf8\x1e\xa1\xe2\x03\xce\x81\xca\x8a\x1a\x0f\x4f\xab\x4a\x09\xad\x28\x15\x13\x76\x59\x1e\x46\x92\x49\xec\x56\x5c\x7d\x81\xdd\x20\x13\x6a\xe9\x65\x5c\x42\x55\xad\xde\x9a\x2a\xbc\x24\x8e\xcc\x59\x5e\xc1\xba\x55\x51\x38\x40\x4e\x18\xa1\xf2\x23\x53\x43\x5b\x15\x64\x6a\x61\x7e\xea\x70\xb2\x27\xfa\xc0\x6d\xa0\x12\x8f\xa1\xa4\x69\x3f\x6c\x8a\x63\x87\x04\xe1\x3d\xbd\x7c\x79\xd9\x30\x38\x50\x07\x38\x28\xc7\x31\x72\x99\xcc\x1e\x2c\x80\x13\x10\xff\x7b\x01\x9c\x13\x07\x0a\x9d\x16\x3e\xb6\xa1\xca\xfe\x84\xc4\xf6\x79\x51\x16\x42\x82\xef\x83\xf3\x81\xd0\x72\x7f\x25\xe6\x63\x90\x42\x73\xa1\xba\x13\x0d\xbd\xcb\xd4\x57\xbd\x13\x81\xb7\xc9\xb1\x84\x4d\x8e\x89\x2b\x06\x1c\x7e\x04\x20\xa4\x18\x28\xa5\x5d\x85\x4e\x4d\x75\xea\xf5\xc6\xaf\xe9\xdf\x1b\x2d\x9f\x39\xaf\xff\xde\xf8\x39\xa3\x76\xfb\x1b\x6e\xff\xec\xb4\x0f\xbf\xff\x67\xf6\xd7\xc6\xf5\xb7\xae\xf7\xfd\xe5\x4b\x34\x9c\xa1\x4d\x21\xb1\x0c\x84\xee\x29\xc3\x41\xc1\xb8\x87\x43\x63\xb3\x24\xf1\x60\x10\xc9\x24\x7f\x09\xa1\x12\xf8\x85\xb2\x1b\xab\xeb\x55\xd7\xbd\xc7\xb6\x54\xce\xb9\x9b\xab\x8e\xac\xfe\x7d\xfa\x8c\xab\xab\x7f\x5f\x5d\x45\xfd\xb8\xbe\xfe\xf7\xf5\x75\xbe\x31\x0e\x23\xe5\x51\xad\x37\x56\x5a\x7c\x1d\xff\xa5\xb9\x9f\x09\x07\x31\x61\xae\x53\x72\x4b\x1e\xbc\xe7\xcc\xd3\x5c\x72\x5a\x7e\x0a\xe3\xd8\xc6\x0a\x37\x7c\x9e\x90\x91\x2c\xdf\xa1\x39\x38\x94\xe8\x01\xf9\xc0\x91\x00\x9b\x51\x07\x6d\x0e\x67\xa8\x28\x4e\x4b\xa6\xae\xff\x4a\x1f\x87\x98\x2b\x1f\x5e\x18\x89\x82\x71\x59\x70\x24\x6a\x10\x0e\x12\x37\x4a\xa8\x43\x2e\x88\x13\x60\xd7\x2a\x8d\xc7\xe4\x1a\x85\x38\x59\x07\xa6\x78\x4a\x0a\xde\x6c\x18\xd8\xe7\x91\xf1\xe9\x6f\x18\x7a\x8d\x78\x2c\x86\x42\xa8\xc0\xce\xc2\xd5\xd5\xde\x24\xf5\x1a\xdf\xbe\x97\xba\x38\xc3\x53\x98\x63\xf3\x0e\xd8\xc4\xc3\x0a\x59\xba\x35\xf6\xc8\xe1\x87\x5f\xb0\x44\x17\x0f\x41\x99\x61\xa1\x98\x8d\x8f\xb1\x80\x52\x5b\x91\xbf\xcc\xbf\x4a\xea\x30\x4b\xc5\xda\x3b\x66\xd6\xd7\xaa\xee\x7e\xd6\x4b\x31\x09\x15\x59\xd9\xcb\xd2\x13\xee\xb1\x9f\xa5\x51\x32\x2b\xdb\x02\x76\xc9\xb8\x0a\x2a\x54\xf9\x07\xb8\x48\x3b\x9d\x63\x41\xcf\x18\xc5\x73\x05\xf3\x60\xfc\xc0\xc0\xb8\x81\xf1\x55\x81\x71\x9b\x51\xc9\x99\xeb\x02\x7f\x7c\x28\xcf\xfa\xf2\xe4\xe1\xbc\x4a\xac\x06\xd2\x0d\xa4\x23\x03\xe9\x4f\x08\xd2\x8b\x13\xf3\xc3\x1a\x44\x3f\x34\x88\x6e\x10\xfd\x31\x11\xbd\x15\x4d\x1f\x5f\xff\xbd\xd1\xfb\x16\x96\xbc\x32\x18\xdf\x1c\xc6\xf7\xa6\x53\x83\xf3\xe9\x5b\x18\x9c\x37\x38\x8f\x9e\x36\xce\xef\x15\x60\xbe\x34\x73\xaf\xc1\xf9\x83\x8e\xc1\xf9\x06\x70\x3e\xf7\xd8\x15\x81\xf9\xfc\x53\x9e\x00\xce\xef\x1a\x9c\x6f\x1a\xe7\x77\x0d\xce\x6b\x6f\xf1\xec\x70\xde\xea\x58\x06\xe6\xd7\x1a\xe6\x0f\x96\x84\xf9\xae\x81\xf9\xe7\x3a\x9d\x7f\x72\x38\xdf\x37\x38\xdf\x34\xce\xf7\x0d\xce\x6b\x6f\x61\x70\xbe\xb2\x93\x06\xe7\x9f\x0c\xce\x2f\xce\xa7\xdb\xaf\x01\xfa\x1d\x03\xf4\x06\xe8\xef\x08\xf4\x03\x27\x88\x52\x72\x07\x11\x8c\x88\x81\x08\xbc\x3b\x2e\xcc\x6f\xa3\x65\x9f\x65\xb3\x80\xca\xbb\x3d\xed\xd6\x14\x62\x49\x9a\x70\xa6\x8c\x7a\x31\x2f\xa8\x71\xc9\x35\xd2\x77\x86\x03\x1e\xd0\xb8\x8f\xf7\x26\xf7\x8a\xa7\x3c\x01\x89\xbf\xb1\x25\xb9\x80\x53\xb0\x19\x77\x6a\x04\x7f\x7c\x6b\xc1\x5f\x10\xb8\x7c\x08\xd1\x57\x3e\xe7\x09\x08\xff\x4f\x02\x97\x35\x42\x7f\xbb\x22\x2c\xf8\x34\x14\x30\xc2\x17\xc0\xf1\x18\x10\x07\xe1\x33\x2a\x00\xe5\x18\xa3\x21\xbd\xb7\x23\xbd\x25\x5a\xa7\x51\x4a\xb3\x86\xb5\x6a\xa4\xd7\xc6\xdc\x29\x34\x1c\x16\x9d\x60\xc7\x21\x74\x5c\xb6\x9c\xb0\xf2\x94\x05\xd4\x29\x34\x9e\xf6\xd4\x8e\x77\x15\x15\x1a\x4c\x37\x1b\xfd\xd2\xdf\x3f\xdc\x7d\xdf\xd3\x6d\x54\xdd\xf2\xd9\xc6\xd1\xd8\x14\x3f\x72\x1a\x48\x6a\x27\xe0\xc5\xe3\x48\x02\xf7\x99\x8b\x25\x1c\x2b\x73\xd5\x2e\x85\xa9\xcf\x68\x44\x4d\x3b\x5b\xfd\x8a\xd1\xc1\x7c\x6c\x13\x39\x2b\x8f\xc0\x90\x8f\x67\x1e\x4c\x8a\x64\x9c\xdd\x84\xbf\x3b\x20\x6c\x4e\xfc\x78\x73\x52\x66\xd4\x8d\x66\xd4\x96\x89\xfc\x04\xb0\xf4\xb0\x9f\xa7\xa7\x13\xe2\xc0\x5f\xc0\xd9\x71\xea\x2f\xf2\x94\x6f\x42\xc6\x13\x97\x8c\x27\xf2\x6d\xac\xff\x1c\x73\x8e\xe6\x06\xf3\x37\xdb\xc4\x76\x5b\xcb\xc7\x8b\x24\xbb\x92\x45\x73\xb8\x00\x2e\xe0\x6b\x5d\x37\x1b\x67\xa6\x91\x5e\x97\xc0\xcf\xad\x7c\xc0\xc9\x85\x5a\xd8\x4c\xe4\x5f\x09\x99\xcb\x05\x99\x3a\x73\xf1\x54\x45\x99\x5c\xb8\x7d\x74\x69\x0e\x7e\x2e\x80\xc9\x8a\x18\x51\xfc\xba\x2a\x56\x94\x03\x4e\x14\xc9\x16\x11\x1a\x5f\xba\xc4\x5e\x8e\x2a\x14\x0a\x4b\x7f\x27\x42\xb2\x31\xc7\x5e\xad\x85\x25\x90\x59\x94\xbe\x35\x7d\x53\xf2\x94\x65\x17\x9b\xb5\x33\x8d\x4c\xef\x53\xe0\x0d\xd5\x14\x2c\x27\x88\xb8\xf2\x33\xf9\x59\xc4\x50\x6b\x56\x7e\x8c\x86\x81\x3a\x0d\xa8\x86\xbf\x6a\xf4\xa8\xc4\x8e\x4a\xe4\xa8\x13\x9e\xef\x12\x99\x1a\x56\xa5\x83\x9e\x45\x2f\x75\x1c\x3b\x71\x0b\x07\x92\x59\xc5\xda\x6a\x79\xcc\x4a\xf2\x30\x98\x72\x07\x4c\xa9\xc5\x85\xfd\x9b\xc6\x77\x7a\xfd\x87\x83\x85\xf9\x9b\x37\x9e\x39\x2c\x54\x2e\x55\x64\x21\xf3\xd7\x1b\x94\x49\x32\x22\x76\xb4\x4b\xda\x40\x88\xf5\x49\x97\x47\xe5\xa2\x83\x01\x14\x64\x00\xc5\x00\xca\x03\x4d\x52\x6e\x8f\x39\xa5\xb9\xc8\x43\x82\xce\xfc\xfd\x05\xeb\x0e\x3a\x12\x28\xa6\xd2\xc0\x0d\xb2\xce\x22\x49\x18\xa0\x31\x40\x63\x80\xe6\x89\x02\x4d\x71\x72\xb3\xd3\x7b\x30\x9c\x39\x9c\x9f\xf8\xb6\xee\x38\x13\xe5\x64\x6d\xbb\xe4\x02\x0c\xd6\x58\x1f\xc8\x05\x50\x10\x06\x6c\x0c\xd8\x18\xb0\x79\xaa\x60\x53\x9a\xd5\x3c\x24\xda\xcc\xdf\x4d\x65\xd0\x46\xa1\x0d\x07\xec\xcc\x0c\xdc\x58\xa7\x80\x1d\x62\xf0\xc6\xe0\xcd\xad\xf1\xe6\x01\xbe\x80\xb9\x53\xb3\xa3\xa4\xaf\xa5\x41\x2e\xfa\x04\x26\xfa\x23\xd0\xb3\x70\x56\xfd\x33\x98\x85\x65\xff\x48\x45\x88\x8d\xd0\xff\xb1\xa1\x40\x40\x7f\x04\x10\x80\x83\x30\x75\x90\x72\x65\x48\x32\x34\x04\x04\x53\xb0\x03\x09\x0e\x7a\xf3\xf9\xcd\x09\xda\xa4\xa1\x9f\x46\x23\x4c\x5c\x70\x5a\x88\x32\x8e\xc6\x4c\x22\x98\xfa\x84\x83\xf3\x72\xab\x89\x2c\xe1\x85\xca\xdb\xad\xf9\x7e\xe9\xae\xf9\xba\x87\xc9\x12\x5e\x9e\x29\x78\x78\xba\xf9\x23\x80\x41\x68\xff\x03\x61\x4f\xc0\x09\x5c\x70\x96\xde\x0d\xf4\x23\x80\xad\x57\x1b\xad\x70\xdc\xbf\xde\x88\xc1\x3f\xc2\xfd\x90\x63\x72\x09\x4e\xd8\xf2\x7d\xef\xfb\x59\x82\x05\xe8\xdd\x79\x0a\x5b\x7f\x4e\x95\xf7\xf9\x9f\xd0\x2b\xa9\x2c\xd2\xbb\x65\x3a\xf6\x8a\xe3\x6e\xfd\x32\x1d\x3b\x35\x26\x48\x19\x85\xfb\xcc\x21\x34\xfb\x7b\xf4\xf2\x27\xb3\xbf\x67\x49\xa2\x50\xa0\x07\x82\x79\x31\x91\x27\x14\xc9\x09\xa0\x51\x20\x03\x0e\x2d\x34\x0c\x24\xa2\x2c\xfc\xef\xf2\x41\xe8\xc3\xc2\x38\xed\xee\x41\x35\x7d\x38\x34\x9b\x8c\x0c\x7d\x78\x1c\xfa\x90\xde\x6e\x28\x44\x23\x14\xe2\x73\x22\x4f\x43\x23\x0c\x8d\x30\x34\x62\x05\x69\x84\x9c\x60\x19\xf3\x00\x84\x25\x72\x01\x0b\x89\x18\xb5\xa1\x85\x1c\xe2\x28\xc2\xc0\x03\x8a\x58\x20\xc3\x5b\xb0\x94\xe0\xf9\x52\x84\xa4\x83\x83\xe4\x33\x15\xa6\x90\x13\xe0\x30\x62\x1c\x10\xe6\x80\x52\x1f\x8a\x46\x8c\xa7\x57\xcd\x22\x4e\x22\x18\xa3\xf7\xc3\x2e\x4a\x81\xf9\x5a\x7a\xb1\x6b\xe8\x85\xa1\x17\x8f\x42\x2f\xa2\x71\x66\xb8\x45\x23\xdc\xe2\x7d\xe4\xb4\x0c\xb1\x30\xc4\xc2\x10\x8b\x15\x25\x16\x59\x54\x22\xb0\x6d\x10\x62\x14\xb8\xee\xec\x56\xf8\x7f\xe3\x2c\xb0\xfe\x5e\x0d\xfc\xef\x19\xf8\x37\xf0\xff\x38\xf0\x4f\x28\x11\x13\x43\x00\x9a\x22\x00\xb1\x38\x0d\x05\x30\x14\xc0\x50\x80\x55\xa5\x00\x49\x6c\x81\x3a\x88\xe3\xfa\x40\x42\x4b\x8b\x22\x5c\x32\xba\x21\xd1\x10\x54\x0d\x09\x6f\x1e\x63\x72\xbb\xb0\xc1\xcd\xf3\xf9\x6a\x79\x43\xdf\xf0\x06\xc3\x1b\x1e\x85\x37\xc4\xcb\x72\x86\x36\x34\x42\x1b\x7e\x8b\xa4\x69\x58\x83\x61\x0d\x6b\xca\x1a\x1e\x22\xb1\x73\x6f\xa7\x1a\x46\xbb\xd9\xe2\x7e\x55\x62\x27\x07\x1f\x22\xad\x38\xe0\xbb\x6c\xe6\x01\x95\x6f\x19\x1d\x91\xb1\x55\x1e\xc9\x27\xcc\x11\x68\xf3\xd7\xe2\x95\x2f\x6f\x90\x0c\x6a\x63\x7b\x02\x67\xc4\x03\x16\x94\x3c\x86\xda\xcd\x70\x8c\xed\xf3\x31\x8f\x93\x6c\x73\x98\xa9\xaa\xff\x0c\xc7\x4f\x49\x8c\x76\xc2\xc8\xb2\xa1\x63\xfd\xf2\xbe\xb7\x7b\xd8\x7f\xab\x0f\x54\x75\x8e\x7e\x6f\x67\xbf\x85\xba\xbd\xc3\x16\xda\xed\xb4\x50\x67\xeb\xe0\x30\x77\x96\xfe\x2f\xbd\xc3\x43\x7b\x77\xcf\x2a\x59\xc7\x52\xec\xac\x3c\x42\xeb\x46\xa7\x35\xc6\x81\x02\xe8\xab\x1c\x47\x49\xde\xaf\xdb\xe9\xe4\x69\x4a\x52\xd1\x29\xbb\x98\xa2\x71\xa6\xbe\xfd\x43\x38\xaa\x4a\xa8\xae\x5f\xf1\x11\xf3\x73\xe0\xa2\x2e\xcb\xbb\xd6\x3c\x77\x0a\xe6\x59\xfc\x90\x7d\xc9\x3a\x77\x4b\x6d\x4f\x88\xa3\x0c\x21\xe1\x10\x95\x3b\x30\xf6\xb2\xb5\x23\x1d\x47\xad\x79\x94\xc9\xc3\xbe\x4f\xe8\xf8\x2c\x32\xc5\x6e\x55\xf9\x1c\x2f\x1b\x3b\xf3\xc8\x51\x87\x6c\x59\xc2\xb4\xe0\xac\x2e\x12\x1d\x2d\xf4\x79\x49\x63\x1c\xd3\xf1\x82\xc6\x7a\x73\x1c\x93\x87\xa7\xef\xb0\xc4\x27\x09\x49\xd3\x8c\xa3\x4c\x10\x6d\x46\x29\xd8\x12\xb2\xef\x6a\xaa\x6b\xce\xc2\x27\x17\x06\x5c\x35\x7b\x74\x83\x31\xa1\x7f\x02\x17\xf1\x24\x63\x6f\xab\xb7\xb5\x6b\x69\x54\x51\xc8\x11\x99\xe6\xd5\x10\x17\xbe\x67\x34\xc9\x72\xb7\xfa\x9d\x7f\x68\xf5\x1c\xca\xf7\xa8\xb2\xda\x5b\x94\xcc\x3e\x62\x7f\x8e\xae\x46\x11\x51\xc9\x73\x62\x55\x23\xa3\xb7\xb5\x3e\x6d\xbf\x29\x54\xb0\xf4\x86\x39\x02\x17\x3e\xe6\xe7\x6e\xc4\x47\x35\xcb\x0f\xa7\x41\xe9\x66\x2d\xe5\x4c\x76\xba\x2d\xd4\xed\x1e\xb4\x50\xf7\xe0\x30\x74\x26\xdd\x83\x9c\x33\x19\x05\x6e\xd5\x8c\x21\x6c\x59\x6f\x27\x6a\xa6\xd7\x69\xa1\xee\xe1\x4e\xae\x81\xb9\x5b\x8c\x24\x1e\xba\x61\x3b\x81\x57\xf8\x9e\xdd\x8d\xf6\x0c\x9d\x07\x43\x18\x70\xf0\xdd\xf8\x23\x30\xd9\x0e\x9f\x41\xb4\xc1\x67\xa0\x72\x7c\x93\x4b\x44\x1d\x99\xae\x6c\xe2\xf5\xdf\x1b\x25\xb8\x68\x7f\x8b\xbf\x76\xda\xf8\x87\x4e\x6f\x41\x7d\xad\x6e\xab\x67\x55\xb1\x5f\x6b\xa7\x23\xac\x4a\x96\x5b\xac\x49\xb2\x77\x03\x4a\x09\x1d\x23\x9f\x39\xa2\x0c\x88\x82\xd0\xb1\x0b\xa1\x40\xb3\x3a\x35\xf4\x75\xfb\x3f\xd0\xed\x5f\xd5\xce\xb7\x7f\x16\x12\x67\xeb\x75\xb5\xe9\x77\xaa\xfd\xcc\x42\xdb\x57\x17\x7e\x8a\x1d\x57\x38\xcb\xbe\x17\x1c\x3f\x49\x1c\x42\x05\x90\xdf\x00\xe3\x63\xb0\xbe\x29\xc6\xc7\xd4\xc0\x60\xfc\x5d\x30\x7e\xaf\x29\x8c\xef\x57\x61\x7c\xce\xa2\x0c\xca\x37\x8e\xf2\x06\xc5\xd7\x06\xc5\x7d\xb0\xef\x03\xbd\x51\x1b\x19\xfe\xd0\x1c\x7f\xf8\x42\xf1\x05\x26\x6e\x68\x0d\x86\x43\x98\x58\xc0\xf3\xe1\x09\xa5\x15\x9f\x5b\x13\x85\x3d\x13\x0c\x30\xc1\x00\x43\x23\xee\x89\x46\xa8\xa5\x99\xcd\xe4\xff\x54\x62\x42\x81\x0f\x3c\xf0\x18\x9f\x0d\x02\x81\xc7\x30\x18\xce\x24\xd4\x02\x78\xb4\x9a\x56\x01\xd7\x55\x27\x9d\x44\x2b\x6b\x94\x39\xd0\xfc\x79\x27\x8f\x0a\xe3\x2a\x46\xee\x10\x21\x39\x19\xaa\x4c\x3c\x46\xd1\x84\x09\x69\xf0\xbc\x49\x3c\xbf\xe5\xbc\xdf\xd9\xdd\xc5\x3b\xd8\xe0\xf9\xdd\xf0\xbc\x78\x48\xed\x6d\xf1\x7c\xa7\x12\xcf\xcd\xc4\xdf\x4c\xfc\x0d\x62\x2f\x9d\xf3\x12\x4e\xc1\x1d\x70\x25\x8e\x26\xe2\x3e\x73\x06\x19\x78\xa7\x13\x70\x21\x31\x5f\xee\xc0\xdf\x25\xf1\xfb\x5b\xdf\xfb\xfe\x32\xfe\xdc\x97\xcf\x9c\x7b\x3f\xb3\xac\x26\xc7\xe5\xf8\x11\x10\xfe\x23\x9e\xaa\x09\x3a\x4a\xc4\x8a\x36\x5d\x2c\x24\xea\x23\x8f\xd0\x40\x82\xa8\x58\x0f\x7f\x6e\x50\xff\xd8\x79\x90\xb7\x4f\x48\x5c\x9c\x48\x51\x73\x14\x6b\xb7\xd9\x33\xd7\x43\x84\xfc\xcd\xf3\xe5\xac\x9c\x23\x94\x7c\xb9\xb0\x5c\xf3\x44\x93\x18\x3d\x3c\x3d\x01\x7e\xaa\xfa\xb3\x53\x0f\x6d\x61\x01\xc2\x02\xfd\x0c\xdf\x7d\x01\x8a\x2d\x9b\xe2\xd8\xcf\x97\xdf\x24\xc5\x31\x4d\x8d\xc9\x81\x6a\x54\xfa\x8e\x70\xb0\xe3\x2c\xdc\x5c\xf5\xca\xe5\x45\x3e\x52\x70\xf8\xee\x78\xd0\x9b\x8b\x07\xca\x8e\xdb\xb9\x38\xa9\x76\xdd\x07\x42\xcf\xab\xf2\xc0\xb4\x99\x61\xae\x3c\x94\xb4\xd2\xc8\x42\xca\xf7\x4c\x57\xce\x8b\xe2\x55\xa1\xe8\x0a\xf1\xde\xf6\x74\x56\xb3\x3c\x71\x0f\x4a\x0a\xe8\x42\x35\xbd\x5d\x5e\x4d\xcf\x26\xfc\x53\x12\x93\x00\xa7\x9d\x0f\xc1\xe4\x85\xf4\x6e\x45\x12\xa5\x4f\x98\x13\xe5\x47\xa3\x4d\xe5\xde\x5a\x48\xe9\xb7\x85\x02\x1a\xfe\xfb\x52\xed\xe9\x50\xcc\x53\xbd\x4d\x16\x65\x0a\x71\x28\x6b\xce\xa4\x54\x37\x99\x52\x7d\xef\xc9\xca\xf9\x07\x3f\xb1\x8c\x6a\x9d\x2d\xae\xd5\x46\xac\xe5\x27\x20\x7b\x37\x9d\x80\xec\x97\xc3\x69\xd1\x86\x28\x33\xff\xb8\xff\xf9\x47\x53\xf3\x8e\x3b\x6c\xad\x7a\x3e\xf3\x8e\x47\x8b\x4b\x3d\x58\x58\x4a\x6d\xc8\xf2\x99\xf3\x14\xf6\x61\x9d\x2c\x1b\xb3\x32\xc7\xea\x2f\x4d\x20\x1e\x10\x82\xcd\xde\x2b\xbd\x7c\xa5\xf6\x5e\x1d\xd4\x7d\x97\x3d\x03\x81\xbb\x6e\xbd\x7a\x7b\xf2\x05\x7d\x09\x27\x61\x77\xdc\x7f\xf5\x74\x78\xd3\x8d\x03\xb7\x07\xdd\x6a\x2d\xec\xcf\x3f\xc5\x65\x4d\x36\x92\x3f\xef\x18\xac\xa5\xc1\xc7\x4a\x52\x21\xca\x1c\x18\xa4\xa4\x26\x4f\x87\x8e\x32\x62\x64\xfb\x41\x1c\x6b\x49\x8e\xeb\x51\xc6\x72\x74\x75\x85\xb6\x3e\x07\xde\x29\x96\x80\xae\xaf\x35\xa6\xf4\xf7\x5d\x63\x30\xcd\xb0\xa4\xf9\xc1\xda\x39\x2c\xa9\xb1\x78\xed\x63\x32\xab\xd4\x39\x9b\x28\xcc\x8a\x93\x28\x13\x6c\xd1\xca\x57\x8b\x43\xd5\x7c\x3d\xb6\xdf\x2c\x87\xfa\x23\x60\x12\x3f\x28\x87\xb2\x55\x86\x47\xa1\xdf\x0f\x40\xac\x46\x5a\x66\x41\xb7\xa3\xa7\x16\x34\x48\xb9\x6a\x4e\x93\x6a\x76\xa9\xdc\x30\xae\x74\x04\x8c\xa1\xea\x50\xb5\x95\x65\x62\x36\x67\xca\x2a\x73\x02\x9c\xcf\xcf\x26\xec\xf2\x77\xc0\x8e\xea\x42\xfe\xb6\x08\x24\x35\x3b\xb2\x99\x5b\xf0\x34\x0e\x08\xbb\x56\x9b\x0d\x52\x3f\x21\x67\xee\x3c\x44\x53\x1e\x22\x14\xc6\x59\x1e\x51\xa3\xa1\x0c\x19\x33\xfa\xfa\xf5\xeb\xd7\xf6\xc7\x8f\xed\x77\xef\xd0\xef\xbf\x1f\x79\xde\x91\x28\x30\x2d\x1f\x4b\x09\x9c\x56\xb7\x95\x1e\xe6\x47\x1c\x07\xe8\xe2\xa5\xbe\xb4\x5b\x65\xc2\x92\x08\x94\xf1\xd8\x2e\x4b\x20\x94\xe5\xe1\x7e\xbf\xcb\x0b\x69\x4b\x2e\x05\xd2\x18\x71\xc0\xc2\x10\x8d\x2b\xce\x52\x3a\x65\xbd\xe3\xc4\x75\x91\xc3\x2e\xa9\x55\xba\xec\x0b\x77\xcb\xc9\x62\x9a\x08\x55\xa2\x2d\xfa\xa5\x98\x54\x58\xcd\x14\x73\x22\xa6\xd1\xc1\x7e\xb9\xba\x80\x12\x8d\x10\xdc\x4c\xfa\xa7\xf1\x11\x91\xeb\xa9\x80\xe3\xd5\x51\x00\xfa\xc7\x7a\xaa\xe0\x6d\xb3\x2a\x88\x21\x48\xfd\xbc\x99\x22\x3e\x10\x8f\xac\xeb\x38\x78\xf7\xf8\xe3\x20\x12\xff\xba\x8e\x82\xdf\x56\x61\x14\x9c\x30\x67\xf5\xa4\x9f\x67\xd0\xb7\x12\xfe\xb6\xb3\x7d\x75\x85\xb6\x3e\x25\xf1\x29\x74\x7d\x5d\x2a\x68\xef\x08\x1b\xbb\xd0\x3e\x0f\x86\xc0\x29\x48\x10\x6d\x9b\x79\x7e\x20\xa1\xcd\x21\x9a\xe3\x88\xb6\xcf\x9c\xff\xba\xc0\xbc\x9d\x85\xbd\xb2\xa0\xd7\x3f\xc3\x0a\x9f\x39\xaf\x7f\x1d\x0c\x6c\x28\x6e\x24\xd0\x34\xee\x17\xa5\xfc\xd0\xa3\x6d\x85\x74\xac\x89\x65\x7b\xeb\xd5\xf6\xcd\xe5\x22\x24\x27\x74\xbc\x9c\x5c\xe2\xbf\xb4\x80\xda\xf3\x0f\x9c\xe2\xa1\x0b\xc5\x90\xa9\x90\x58\x4d\x6f\x4b\x23\xeb\x86\xd1\xd4\xda\x65\xe6\x5c\xf9\x1d\x52\x5a\xf3\x6b\xf5\xc9\x30\x4c\x4e\x94\x17\x4a\xd6\x36\xe3\xb9\x54\xc0\x67\x27\xce\xe3\x26\xc4\xb9\xd2\xc6\x8b\xb6\x91\x51\x79\x3e\x59\xf6\x9e\x47\x90\xab\xb8\xd6\x7a\x08\xf3\x9d\x19\x3f\xeb\xa5\xf0\xdf\x1e\x66\x89\x6e\xf1\x4a\x9c\x0a\xf1\x3f\x4a\x3a\x13\xc7\x54\x84\x4a\x28\xab\x20\x25\x4e\x85\x62\xb3\x4a\xb7\x2e\xab\x74\x2f\xd0\x23\x2d\xac\x1d\xd6\x9c\xaf\x90\x6d\xbe\xbf\xf3\xc2\xda\x47\xb5\x49\xc4\xe4\x27\x2d\x50\x44\xcd\xc6\xd2\x2c\x55\x78\x8d\x17\xcb\x56\x3d\x11\x69\x25\x93\x8a\x96\xdc\xa6\x75\x7b\xaa\xd1\x42\xe9\x23\xfe\xe3\xf5\x86\x49\x19\x6a\x2e\x65\x48\x77\x99\x26\x6b\xe8\x6e\x7c\x44\xd9\xbb\xe1\x23\xeb\x98\x35\xd4\xed\xd4\x6c\x96\xda\x6f\x9c\xdd\x98\xcc\xa1\x26\xc9\x50\xb7\xd3\xaf\x56\xdc\x8e\x61\x43\x4f\x37\x49\xe8\xb1\x12\x7e\x2a\x3f\x70\x66\xf2\x7d\xd2\x6e\x55\xb2\x8d\x44\xa4\x8f\xbe\xd4\xf8\x04\x53\x7e\x1c\xb0\x23\xd6\x71\x63\x1d\x98\xc4\x9f\x95\x52\xc3\xba\x26\x3e\xac\x44\xfa\x4f\xac\x0b\x93\x01\xd4\x98\x1e\x6e\x3f\x20\x4c\x1e\x50\x83\x6a\x30\x79\x40\x26\x0f\xc8\xe4\x01\x35\x9a\x07\xb4\x8a\xb1\xce\x67\x9d\xe5\x13\x8b\xb9\x39\x01\xaf\xa8\x44\x1b\x49\xf4\x79\x44\xeb\xbc\x49\x1a\xcf\xba\xe8\xf4\xa1\x32\x79\xd6\x45\x9e\x8d\x24\xf3\xac\xf6\x18\x59\x33\x8d\xae\x44\xb6\x8e\x1e\x59\x37\x09\x3b\x26\x61\x67\xa5\x16\xc8\x5e\xa0\xc7\x5a\xd3\xea\xf6\x4a\xaa\x8e\x76\x55\x37\xb8\x17\xfe\x13\xc8\x4b\xc6\xcf\x4d\xce\xce\x22\x5d\xd4\x1d\xab\x9f\x2d\x30\x9a\x75\xaa\x95\xfb\x84\xe2\x4a\x66\xeb\x10\x8e\x25\x68\x2c\x88\x46\x43\x70\xc0\xc1\x06\x72\x11\x13\xa1\xd2\xa7\x10\xef\xc0\x3c\x1a\xff\x16\xe2\xad\x53\x76\x56\xee\x5b\x88\xa7\x91\xcc\xd1\x31\xa6\x4e\x64\xe9\x77\xa2\x1f\xeb\x9a\x83\xa3\x05\x96\x4a\xe8\x9a\x19\xd8\xb1\x7f\xaf\xd9\x39\x4f\xfc\xc3\x88\xe6\x13\xca\xf7\x00\xdb\xbd\x9a\x2f\x32\x66\x67\x97\x19\xd8\x36\xb0\x7d\x27\xd8\x56\x93\x47\x8f\x48\x83\xdb\x0f\x88\xdb\x67\xb1\xd0\x0d\x70\x37\x15\x1b\x30\xe0\xbc\x1a\xe0\xfc\x22\x6e\x36\x1c\x73\xe1\x70\x8a\x92\x2f\x23\xdd\x58\xc2\x9e\x80\x87\xb3\x63\x98\x23\x18\x8b\xd2\xd2\xd4\x84\x1f\xf3\xf3\xe8\x4a\x89\xc7\x99\xda\xad\x68\x51\x35\x16\x9c\xf5\x73\x46\x6d\x2b\x7d\x8e\x04\xcf\x77\xb1\x24\x74\x9c\xf6\xdf\x72\x89\x90\x9a\xd1\xe8\x30\x1c\x1d\xe2\xa9\x03\x2d\xa1\xb6\x1b\x38\xf0\xa6\xfa\x8c\xc4\x4a\xf5\x58\x5e\xe0\x4a\x52\x71\x79\x72\x92\x66\x05\x4f\xc8\x01\x92\xbe\x0c\x69\xfd\x08\x80\xcf\xd4\x92\x2e\x67\x1e\xc8\x09\x04\xba\x29\x6b\x82\xec\xe6\x4a\xc7\x30\x2d\x04\x46\x2d\x71\x4e\xfc\x2f\xdc\xfd\x1c\x0a\xa8\xdc\xb9\x64\xb8\x6b\x9d\x2b\x8e\xb7\x9c\xfe\xdd\xe4\x3c\xd7\xc2\xcb\x67\x3c\x22\x67\xe6\x89\xca\x0a\x2b\xac\xf1\x39\x7e\xc5\x85\xf2\xfc\x28\x48\x8f\xf5\x2b\x5d\x56\x35\xba\x6e\xad\x45\x2b\x85\x32\xeb\x06\xca\xac\xbc\x49\xd3\xa5\xf6\x22\xba\x40\x14\x11\x70\xa3\x33\x4e\xcb\xf1\xef\x65\x45\xb3\x9c\x70\xb2\x31\xad\x8d\x6a\xdd\xb4\xe6\x3c\x63\x49\xab\xb1\x03\x21\x99\xb7\xa4\xc5\xe4\xac\x72\x31\x87\x56\x53\xb0\x11\xa1\x24\xf9\xf8\x9e\x52\xd8\x20\x02\x86\x6c\xb9\x83\xd0\x11\x9b\x7f\xba\x42\xe8\x1a\xda\x5b\xaf\x36\xae\x5b\xa8\x40\x39\x16\xda\x4c\x01\x94\x52\x93\xa9\x0d\x42\x2e\xe3\x06\xe6\xdc\xbb\xc8\x19\xdc\x83\x08\x16\x78\x92\xed\xcd\xad\x57\x2f\xab\xa8\xdc\xf6\xf2\xd6\x12\xf3\x11\xbd\x7d\x89\xc7\xca\x2c\xc4\x1f\xc9\xab\x59\xf9\xda\x92\x0c\xc2\xb2\xea\x8b\x63\x5b\x8c\x84\xa4\x55\x04\x02\xce\xa2\x86\x72\xf3\x0c\xf5\x6f\x48\x2a\xae\x23\xa8\x20\x4a\x2d\x31\x48\xa4\xc7\xf6\xb2\xcb\x76\x37\xa1\x3f\xc9\xc9\xbc\x51\x8c\x38\xbb\xcd\x27\xf6\xb9\xa2\xfa\xf1\xcd\xb1\x28\x07\x09\x47\xd5\x1d\x81\xd5\xd7\x8e\x5b\xed\xea\x67\xaf\xe6\x0e\x62\xed\x7a\xd9\xdf\x7d\xed\xef\xae\xfe\x63\xa7\xa3\xd7\x68\x24\xad\xa7\xfd\xdd\x75\xa2\x61\xf9\x3d\x79\x87\x90\x51\x97\x1d\x54\xfd\x53\xf4\x86\xf7\xf4\x86\xf5\xa7\xf4\x76\xf5\x1f\xda\xb9\xcd\xfb\x8e\xde\xdf\xa4\x2f\x39\xf1\xfd\x64\x34\x73\x0b\x19\x0d\x8e\x70\x1d\x6d\xa3\xbf\x42\x6b\x52\x75\x17\x19\x39\x78\x71\xfd\xe2\xff\x03\x00\x00\xff\xff\x90\x4e\xdc\x6b\xde\x14\x01\x00") +var _monitoringZyncGrafanaDashboard1JsonTpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5d\x6b\x6f\xdb\xb8\x9a\xfe\xde\x5f\xc1\xd5\xcc\x39\x49\xbb\x76\x62\x3b\x71\x6e\x40\xb1\x68\xda\xe9\xce\x59\xb4\xdd\x4c\x9a\x0e\xd0\x29\x0a\x1f\x5a\x7a\x63\x73\x23\x91\x2a\x49\x25\x76\x83\xcc\x6f\x5f\x88\xba\x51\x37\xdb\x71\x94\xc4\x89\xf9\x61\xa6\x31\x29\x51\xd4\x7b\x7b\x1e\x92\x2f\xc5\xeb\x17\x08\x59\x98\x52\x26\xb1\x24\x8c\x0a\xeb\x08\x85\x45\x08\x59\x2e\x11\xd2\x3a\x42\xdf\xd4\x2f\x14\x97\xaa\x9a\x61\x40\x5c\xf9\x2f\x6a\x1d\xa1\x6e\x2b\x2b\x75\xb0\xc4\x82\x05\xdc\x06\xeb\x08\x59\xed\x36\xfa\x6f\x8e\xcf\x31\xc5\xa8\xdd\xb6\xb4\xcb\x80\xe2\xa1\x1b\x5e\x22\x79\x00\x5a\xf9\x98\x38\x15\xa5\xc4\x66\xf4\x2d\x73\x19\x0f\xdb\xe4\xa3\x21\xde\xec\xb4\x50\xaf\xdb\x6d\xa1\x5e\xbf\xdf\x42\xdd\x97\x7a\xd3\x14\x7b\xea\xd9\x6f\xb2\xd7\x41\xff\x44\x6f\x5c\xe0\x52\xe8\xd7\xc9\xa9\xaf\xae\x73\xb0\x18\x0f\x19\xe6\x8e\x15\xd7\xdd\xa8\x7f\xbf\xbf\x40\xe8\x26\xbc\xdc\x02\x87\xc8\x42\x6f\xad\x11\x05\xf9\x2f\xc7\x3a\x42\x34\x70\xdd\xa8\x84\x63\x7f\x7c\xc6\x98\x2b\x89\x9f\xc8\xc4\x22\x12\xb8\xea\x42\x5a\xe2\x12\x7a\x11\x8a\xf7\xdb\x77\xf5\xd3\xc7\x14\x5c\x91\x0a\x38\x11\xaf\x65\x33\xd7\xc5\xbe\x80\xf0\x11\xe7\xd8\x15\xa9\x34\xac\x11\x27\xce\x09\xcb\x34\x14\x89\xad\xa0\x85\x2b\xeb\x08\xf5\x76\xb5\x82\x89\x75\x84\x3a\xda\xef\x69\xf8\x3b\x79\xdf\xb4\x6d\x12\x3e\xee\xa0\x9f\xfe\xce\x7a\xf7\x3d\x2d\x93\x44\x2a\x51\x58\x7f\x4d\xa9\x6d\x65\xc5\xb1\x34\x39\xbb\x8a\xe4\x18\xb7\x9a\xbe\x11\x76\x09\x16\x4a\x89\xaa\xef\xd9\x43\x87\x58\x95\xe4\xdf\x32\x54\xca\x07\xa0\x23\xa9\xde\xac\x93\x2b\x87\xaa\xcb\x75\xab\xfb\x55\xfb\x99\x5e\x72\x4e\x5c\x57\x97\x52\xbd\x20\x0f\x0a\x82\xec\xf6\xe6\x08\xb2\x5b\x2d\xc8\xfd\xfd\xf4\xb7\x0b\x23\xa0\x4e\xfe\x51\xf8\x72\x54\x7c\x8f\x50\xf1\x01\xe7\x40\x65\x45\x8d\x87\x27\x55\xa5\x84\x56\x94\x8a\x31\xbb\x2a\xbb\x91\x64\x12\xbb\x15\x57\x5f\x62\x37\xc8\x84\x5a\x7a\x19\x97\x50\x55\xab\xb7\xa6\x0a\xaf\x88\x23\x73\x96\x57\xb0\x6e\x55\x14\x3a\xc8\x09\x23\x54\x7e\x64\xca\xb5\x55\x41\xa6\x16\xe6\xa7\x01\x27\x7b\xa2\x0f\xdc\x06\x2a\xf1\x08\x4a\x9a\xf6\xc3\xa6\x38\x76\x48\x10\xde\xd3\xcb\x97\x97\x0d\x83\x03\x75\x80\x83\x0a\x1c\xe7\x2e\x93\xd9\x83\x05\x70\x02\xe2\x7f\x2f\x81\x73\xe2\x40\xa1\xd3\xc2\xc7\x36\x54\xd9\x9f\x90\xd8\xbe\x28\xca\x42\x48\xf0\x7d\x70\x3e\x10\x5a\xee\xaf\xc4\x7c\x04\x52\x68\x21\x54\x0f\xa2\x61\x74\x99\xf8\xaa\x77\x22\xf0\x36\x39\x96\xb0\xc9\x31\x71\xc5\x80\xc3\x8f\x00\x84\x14\x03\xa5\xb4\xeb\x30\xa8\xa9\x4e\xbd\xde\xf8\x35\xfd\x7b\xa3\xe5\x33\xe7\xf5\xdf\x1b\x3f\xa7\xd4\x6e\x7f\xc3\xed\x9f\x9d\xf6\xe1\xf7\xff\xcc\xfe\xda\xb8\xf9\xd6\xf5\xbe\xbf\x7c\x89\x86\x53\xb4\x29\x24\x96\x81\xd0\x23\x65\xe8\x14\x8c\x7b\x38\x34\x36\x4b\x12\x0f\x06\x91\x4c\xf2\x97\x10\x2a\x81\x5f\x2a\xbb\xb1\xba\x5e\x75\xdd\x7b\x6c\x4b\x15\x9c\xbb\xb9\xea\xc8\xea\xdf\xa7\xcf\xb8\xbe\xfe\xf7\xf5\x75\xd4\x8f\x9b\x9b\x7f\xdf\xdc\xe4\x1b\xe3\x70\xae\x22\xaa\xf5\xc6\x4a\x8b\x6f\xe2\xbf\xb4\xf0\x33\xe6\x20\xc6\xcc\x75\x4a\x61\xc9\x83\xf7\x9c\x79\x5a\x48\x4e\xcb\x4f\x61\x14\xdb\x58\xe1\x86\xcf\x63\x72\x2e\xcb\x77\x68\x01\x0e\x25\x7a\x40\x3e\x70\x24\xc0\x66\xd4\x41\x9b\xc3\x29\x2a\x8a\xd3\x92\x69\xe8\xbf\xd6\xfd\x10\x73\x15\xc3\x0b\x9e\x28\x18\x97\x85\x40\xa2\x9c\x70\x90\x84\x51\x42\x1d\x72\x49\x9c\x00\xbb\x56\xc9\x1f\x93\x6b\x14\xe2\x64\x1d\x98\xe0\x09\x29\x44\xb3\x61\x60\x5f\x44\xc6\xa7\xbf\x61\x18\x35\x62\x5f\x0c\x85\x50\x81\x9d\x85\xab\xab\xa3\x49\x1a\x35\xbe\x7d\x2f\x75\x71\x8a\x27\x30\xc3\xe6\x1d\xb0\x89\x87\x15\xb2\x74\x6b\xec\x91\xc3\x0f\xbf\x60\x89\x2e\x1e\x82\x32\xc3\x42\x31\x1b\x1d\x63\x01\xa5\xb6\xa2\x78\x99\x7f\x95\x34\x60\x96\x8a\xb5\x77\xcc\xac\xaf\x55\xdd\xfd\xac\x97\x62\x1c\x2a\xb2\xb2\x97\xa5\x27\xdc\x63\x3f\x4b\x5e\x32\x2d\xdb\x02\x76\xc9\xa8\x0a\x2a\x54\xf9\x07\xb8\x4c\x3b\x9d\x63\x41\xcf\x18\xc5\x73\x05\xb3\x60\xfc\xc0\xc0\xb8\x81\xf1\x55\x81\x71\x9b\x51\xc9\x99\xeb\x02\x7f\x7c\x28\xcf\xfa\xf2\xe4\xe1\xbc\x4a\xac\x06\xd2\x0d\xa4\x23\x03\xe9\x4f\x08\xd2\x8b\x03\xf3\xc3\x1a\x44\x3f\x34\x88\x6e\x10\xfd\x31\x11\xbd\x15\x0d\x1f\x5f\xff\xbd\xd1\xfb\x16\x96\xbc\x32\x18\xdf\x1c\xc6\xf7\x26\x13\x83\xf3\xe9\x5b\x18\x9c\x37\x38\x8f\x9e\x36\xce\xef\x15\x60\xbe\x34\x72\xaf\xc1\xf9\x83\x8e\xc1\xf9\x06\x70\x3e\xf7\xd8\x15\x81\xf9\xfc\x53\x9e\x00\xce\xef\x1a\x9c\x6f\x1a\xe7\x77\x0d\xce\x6b\x6f\xf1\xec\x70\xde\xea\x58\x06\xe6\xd7\x1a\xe6\x0f\x16\x84\xf9\xae\x81\xf9\xe7\x3a\x9c\x7f\x72\x38\xdf\x37\x38\xdf\x34\xce\xf7\x0d\xce\x6b\x6f\x61\x70\xbe\xb2\x93\x06\xe7\x9f\x0c\xce\xcf\xcf\xa7\xdb\xaf\x01\xfa\x1d\x03\xf4\x06\xe8\xef\x08\xf4\x03\x27\x88\x52\x72\x07\x11\x8c\x88\x81\x08\xbc\x3b\x2e\xcc\x6f\xa3\x45\x9f\x65\xb3\x80\xca\xbb\x3d\x6d\x69\x0a\xb1\x20\x4d\x38\x53\x46\x3d\x9f\x17\xd4\x84\xe4\x1a\xe9\x3b\xc3\x01\x0f\x68\xdc\xc7\x7b\x93\x7b\xc5\x53\x9e\x80\xc4\xdf\xd8\x92\x5c\xc2\x29\xd8\x8c\x3b\x35\x82\x3f\x5e\x5a\xf0\x97\x04\xae\x1e\x42\xf4\x95\xcf\x79\x02\xc2\xff\x93\xc0\x55\x8d\xd0\xdf\xae\x08\x0b\x3e\x0d\x05\x8c\xf0\x25\x70\x3c\x02\xc4\x41\xf8\x8c\x0a\x40\x39\xc6\x68\x48\xef\x72\xa4\xb7\x44\xeb\x34\x4a\x69\xd6\xb0\x56\x8d\xf4\xda\x98\x3b\x85\x86\xc3\xa2\x13\xec\x38\x84\x8e\xca\x96\x13\x56\x9e\xb2\x80\x3a\x85\xc6\xd3\x9e\xda\xf1\xae\xa2\x42\x83\xe9\x66\xa3\x5f\xfa\xfb\x87\xbb\xef\x7b\xba\x8d\xaa\x5b\x3e\xdb\x38\xf2\x4d\xf1\x23\xa7\x81\xa4\x76\x0c\x5e\xec\x47\x12\xb8\xcf\x5c\x2c\xe1\x58\x99\xab\x76\x29\x4c\x7c\x46\x23\x6a\xda\xd9\xea\x57\x78\x07\xf3\xb1\x4d\xe4\xb4\xec\x81\x21\x1f\xcf\x22\x98\x14\x89\x9f\xdd\x86\xbf\x3b\x20\x6c\x4e\xfc\x78\x73\x52\x66\xd4\x8d\x66\xd4\x96\x89\xfc\x18\xb0\xf4\xb0\x9f\xa7\xa7\x63\xe2\xc0\x5f\xc0\xd9\x71\x1a\x2f\xf2\x94\x6f\x4c\x46\x63\x97\x8c\xc6\xf2\x6d\xac\xff\x1c\x73\x8e\xc6\x06\xb3\x37\xdb\xc4\x76\x5b\xcb\xc7\x8b\x24\xbb\x92\x45\x73\xb8\x04\x2e\xe0\x6b\x5d\x37\x1b\x67\xa6\x91\x5e\x17\xc0\xcf\xad\xfc\x84\x93\x0b\xb5\xb0\x99\xc8\xbf\x12\x32\x17\x9b\x64\xea\xcc\xc4\x53\x35\xcb\xe4\xc2\xf2\xb3\x4b\x33\xf0\x73\x0e\x4c\x56\xcc\x11\xc5\xaf\xab\xe6\x8a\x72\xc0\x89\x22\xd9\x22\x42\xe3\x4b\x17\xd8\xcb\x51\x85\x42\x61\xe9\xef\x44\x48\x36\xe2\xd8\xab\xb5\xb0\x04\x32\x8b\xd2\xb7\x26\x6f\x4a\x91\xb2\x1c\x62\xb3\x76\x26\x91\xe9\x7d\x0a\xbc\xa1\x1a\x82\xe5\x04\x11\x57\x7e\x26\x3f\x8b\x18\x6a\x4d\xcb\x8f\xd1\x30\x50\xa7\x01\xd5\xf0\x57\x8d\x1e\x95\xd8\x51\x89\x1c\x75\xc2\xf3\x5d\x22\x53\xc3\xaa\x0c\xd0\xd3\xe8\xa5\x8e\xe3\x20\x6e\xe1\x40\x32\xab\x58\x5b\x2d\x8f\x69\x49\x1e\x06\x53\xee\x80\x29\xb5\xb8\xb0\x7f\xdb\xf9\x9d\x5e\xff\xe1\x60\x61\xf6\xe6\x8d\x67\x0e\x0b\x95\x4b\x15\xd9\x94\xf9\xeb\x0d\xca\x24\x39\x27\x76\xb4\x4b\xda\x40\x88\xf5\x49\x97\x47\xe5\xa2\x83\x01\x14\x64\x00\xc5\x00\xca\x03\x0d\x52\x96\xc7\x9c\xd2\x58\xe4\x21\x41\x67\xf6\xfe\x82\x75\x07\x1d\x09\x14\x53\x69\xe0\x06\x59\x67\x91\x24\x0c\xd0\x18\xa0\x31\x40\xf3\x44\x81\xa6\x38\xb8\xd9\xe9\x3d\x18\xce\x1c\xce\x4e\x7c\x5b\x77\x9c\x89\x72\xb2\xb6\x5d\x72\x09\x06\x6b\xac\x0f\xe4\x12\x28\x08\x03\x36\x06\x6c\x0c\xd8\x3c\x55\xb0\x29\x8d\x6a\x1e\x12\x6d\x66\xef\xa6\x32\x68\xa3\xd0\x86\x03\x76\xa6\x06\x6e\xac\x53\xc0\x0e\x31\x78\x63\xf0\x66\x69\xbc\x79\x80\x2f\x60\xee\xd4\xec\x28\xe9\x6b\x69\x90\xf3\x3e\x81\x89\xfe\x08\xf4\x2c\x9c\x55\xff\x0c\x66\x61\xd9\x3f\x52\x11\x62\xe7\xe8\xff\xd8\x50\x20\xa0\x3f\x02\x08\xc0\x41\x98\x3a\x48\x85\x32\x24\x19\x1a\x02\x82\x09\xd8\x81\x04\x07\xbd\xf9\xfc\xe6\x04\x6d\xd2\x30\x4e\xa3\x73\x4c\x5c\x70\x5a\x88\x32\x8e\x46\x4c\x22\x98\xf8\x84\x83\xf3\x72\xab\x89\x2c\xe1\xb9\xca\xdb\xad\xf9\x7e\xe9\xae\xf9\xba\x87\xc9\x12\x5e\x9c\x29\x78\x78\xb2\xf9\x23\x80\x41\x68\xff\x03\x61\x8f\xc1\x09\x5c\x70\x16\xde\x0d\xf4\x23\x80\xad\x57\x1b\xad\xd0\xef\x5f\x6f\xc4\xe0\x1f\xe1\x7e\xc8\x31\xb9\x04\x27\x6c\xf9\xbe\xf7\xfd\x2c\xc0\x02\xf4\xee\x3c\x85\xad\x3f\xa7\x2a\xfa\xfc\x4f\x18\x95\x54\x16\xe9\xdd\x32\x1d\x7b\x45\xbf\x5b\xbf\x4c\xc7\x4e\x8d\x09\x52\x46\xe1\x3e\x73\x08\xcd\xfe\x1e\xbd\xfc\xc9\xec\xef\x59\x90\x28\x14\xe8\x81\x60\x5e\x4c\xe4\x09\x45\x72\x0c\xe8\x3c\x90\x01\x87\x16\x1a\x06\x12\x51\x16\xfe\x77\xf5\x20\xf4\x61\xee\x3c\xed\xee\x41\x35\x7d\x38\x34\x9b\x8c\x0c\x7d\x78\x1c\xfa\x90\xde\x6e\x28\x44\x23\x14\xe2\x73\x22\x4f\x43\x23\x0c\x8d\x30\x34\x62\x05\x69\x84\x1c\x63\x19\xf3\x00\x84\x25\x72\x01\x0b\x89\x18\xb5\xa1\x85\x1c\xe2\x28\xc2\xc0\x03\x8a\x58\x20\xc3\x5b\xb0\x94\xe0\xf9\x52\x84\xa4\x83\x83\xe4\x53\x35\x4d\x21\xc7\xc0\xe1\x9c\x71\x40\x98\x03\x4a\x63\x28\x3a\x67\x3c\xbd\x6a\x1a\x71\x12\xc1\x18\xbd\x1f\x76\x51\x9a\x98\xaf\xa5\x17\xbb\x86\x5e\x18\x7a\xf1\x28\xf4\x22\xf2\x33\xc3\x2d\x1a\xe1\x16\xef\xa3\xa0\x65\x88\x85\x21\x16\x86\x58\xac\x28\xb1\xc8\x66\x25\x02\xdb\x06\x21\xce\x03\xd7\x9d\x2e\x85\xff\xb7\xce\x02\xeb\xef\xd5\xc0\xff\x9e\x81\x7f\x03\xff\x8f\x03\xff\x84\x12\x31\x36\x04\xa0\x29\x02\x10\x8b\xd3\x50\x00\x43\x01\x0c\x05\x58\x55\x0a\x90\xcc\x2d\x50\x07\x71\x5c\x3f\x91\xd0\xd2\x66\x11\xae\x18\xdd\x90\x68\x08\xaa\x86\x84\x37\x8f\x30\x59\x6e\xda\xe0\xf6\xf9\x7c\xb5\xbc\xa1\x6f\x78\x83\xe1\x0d\x8f\xc2\x1b\xe2\x65\x39\x43\x1b\x1a\xa1\x0d\xbf\x45\xd2\x34\xac\xc1\xb0\x86\x35\x65\x0d\x0f\x91\xd8\xb9\xb7\x53\x0d\xa3\xdd\x6c\x71\xbf\x2a\xb1\x93\x83\x0f\x91\x56\x1c\xf0\x5d\x36\xf5\x20\xe7\x9e\x89\x0f\x9f\x30\x47\xa0\xcd\x5f\xb3\x6b\x5e\xde\x22\x01\xd4\xc6\xf6\x18\xce\x88\x07\x2c\x28\x45\x09\xb5\x83\xe1\x18\xdb\x17\x23\x1e\x27\xd6\xe6\x70\x52\x55\xff\x19\xfa\x4c\x49\x74\x76\xc2\xc2\x32\x77\xb1\x7e\x79\xdf\xdb\x3d\xec\xbf\xd5\x9d\x53\x9d\x9d\xdf\xdb\xd9\x6f\xa1\x6e\xef\xb0\x85\x76\x3b\x2d\xd4\xd9\x3a\x38\xcc\x9d\x9f\xff\x4b\xef\xf0\xd0\xde\xdd\xb3\x4a\x16\xb1\x10\x23\x2b\x7b\x65\x9d\x47\x5a\x23\x1c\x28\x50\xbe\xce\xf1\x92\xe4\xfd\xba\x9d\x4e\x9e\x9a\x24\x15\x9d\x72\x58\x29\x1a\x64\x1a\xcf\x3f\x84\x9e\x54\x42\x72\xfd\x8a\x8f\x98\x5f\x00\x17\x75\x99\xdd\xb5\x26\xb9\x53\x30\xc9\xe2\xc7\xeb\x4b\x16\xb9\x5b\x6a\x7b\x4c\x1c\x65\x08\x09\x6f\xa8\xdc\x75\xb1\x97\xad\x17\xe9\xd8\x69\xcd\xa2\x49\x1e\xf6\x7d\x42\x47\x67\x91\x29\x76\xab\xca\x67\x44\xd6\x38\x80\x47\xc1\x39\x64\xc8\x12\x26\x85\x00\x75\x99\xe8\x68\x6e\x9c\x4b\x1a\xe3\x98\x8e\xe6\x34\xd6\x9b\x11\x8c\x3c\x3c\x79\x87\x25\x3e\x49\x88\x99\x66\x1c\x65\x52\x68\x33\x4a\xc1\x96\x90\x7d\x4b\x53\x5d\x73\x16\x3e\xb9\xe0\x70\xd5\x8c\xd1\x0d\x46\x84\xfe\x09\x5c\xc4\x03\x8b\xbd\xad\xde\xd6\xae\xa5\xd1\x43\x21\xcf\xc9\x24\xaf\x86\xb8\xf0\x3d\xa3\x49\x66\xbb\xd5\xef\xfc\x43\xab\xe7\x50\xbe\x47\x95\xd5\xde\xa2\x64\xf6\x11\xfb\x33\x74\x75\x1e\x91\x93\x3c\x0f\x56\x35\x32\x7a\x5b\xeb\xd3\xf6\x9b\x42\x05\x4b\x6f\x98\x21\x70\xe1\x63\x7e\xe1\x46\x1c\x54\xb3\xfc\x70\xe8\x93\x6e\xd0\x52\xc1\x64\xa7\xdb\x42\xdd\xee\x41\x0b\x75\x0f\x0e\xc3\x60\xd2\x3d\xc8\x05\x93\xf3\xc0\xad\x1a\x25\x84\x2d\xeb\xed\x44\xcd\xf4\x3a\x2d\xd4\x3d\xdc\xc9\x35\x30\x73\x5b\x91\xc4\x43\x37\x6c\x27\xf0\x0a\xdf\xb0\xbb\xd5\x3e\xa1\x8b\x60\x08\x03\x0e\xbe\x1b\x7f\xf8\x25\xdb\xd5\x33\x88\x36\xf5\x0c\x54\x5e\x6f\x72\x89\xa8\x23\xd0\x95\x4d\xbc\xfe\x7b\x43\x03\x8a\xf6\xb7\xf8\xdb\xa6\x8d\x7f\xd6\x74\x09\xa2\x6b\x75\x5b\x3d\xab\x8a\xeb\x5a\x3b\x1d\x61\x55\x72\xda\x62\x4d\x92\xab\x1b\x50\x4a\xe8\x08\xf9\xcc\x11\x65\x28\x14\x84\x8e\x5c\x08\x45\x99\xd5\x29\xa7\xd7\x2d\xff\x40\xb7\x7c\x55\x3b\xdb\xf2\x59\x48\x93\xad\xd7\xd5\x46\xdf\xa9\x8e\x30\x73\xad\x5e\x5d\xf8\x29\x0e\x59\xe1\x98\xfa\x5e\x10\xfc\x24\x09\x05\x15\x10\x7e\x0b\x74\x8f\x61\xfa\xb6\xe8\x1e\x93\x02\x83\xee\x77\x41\xf7\xbd\xa6\xd0\xbd\x5f\x85\xee\x39\x8b\x32\xf8\xde\x38\xbe\x1b\xfc\x5e\x1b\xfc\xf6\xc1\x6e\x16\xb7\x51\x1b\x19\xce\xd0\x04\x67\xf8\x42\xf1\x25\x26\x6e\x68\x01\x86\x37\x98\x91\xff\xf3\xe1\x06\xa5\x35\x9d\xa5\xc9\xc1\x9e\x19\xfa\x9b\xa1\xbf\xa1\x0e\xf7\x44\x1d\xd4\xe2\xcb\x66\xf2\x7f\x2a\x31\xa1\xc0\x07\x1e\x78\x8c\x4f\x07\x81\xc0\x23\x18\x0c\xa7\x12\x6a\xa1\x3b\x5a\x2f\xcb\x01\x75\xd5\x29\x26\xd1\xaa\x19\x65\x0e\x34\x7f\x96\xc9\xa3\x02\xb8\x9a\x05\x77\x88\x90\x9c\x0c\x55\x96\x1d\xa3\x68\xcc\x84\x34\x48\xde\x24\x92\x2f\x39\xca\x77\x76\x77\xf1\x0e\x36\x48\x7e\x37\x24\x2f\x1e\x40\xbb\x2c\x92\xef\x54\x22\xb9\x19\xe6\x9b\x61\xbe\xc1\xea\x85\xf3\x59\xc2\x61\xb7\x03\xae\xc4\xd1\xe0\xdb\x67\xce\x20\x83\xed\x74\xd0\x2d\x24\xe6\x8b\x1d\xe6\x3b\x17\xb9\xbf\xf5\xbd\xef\x2f\xe3\x8f\x78\xf9\xcc\xb9\xf7\x93\xc8\x6a\x32\x57\x8e\x1f\x01\xdb\x3f\xe2\x89\x1a\x94\xa3\x44\xa0\x68\xd3\xc5\x42\xa2\x3e\xf2\x08\x0d\x24\x88\x8a\x15\xef\xe7\x06\xf2\x8f\x9d\xdd\xb8\x7c\x9a\xe1\xfc\xf4\x88\x9a\x03\x56\xbb\xcd\x9e\xa4\x1e\x62\xe3\x6f\x9e\x2f\xa7\xe5\xcc\x9f\xe4\x7b\x84\xe5\x9a\x27\x9a\x9a\xe8\xe1\xc9\x09\xf0\x53\xd5\x9f\x9d\x7a\x50\x0b\x0b\x10\x16\xe8\x67\xf8\xee\x73\xf0\x6b\xd1\xc4\xc5\x7e\xbe\xfc\x36\x89\x8b\x69\xc2\x4b\x0e\x4e\xa3\xd2\x77\x84\x83\x1d\xe7\xd6\xe6\xaa\x57\x2e\xdb\xf1\xc1\x27\x81\xef\x8e\x04\xbd\x99\x48\xa0\x2c\xb8\x9d\x9b\x15\xd5\xae\xfb\x40\xe8\x45\x55\x5e\x97\x36\x1a\xcc\x95\x87\x32\x56\xba\x98\x4b\xf3\x9e\xdd\xaa\x78\x51\xb0\x6a\xca\xb9\x42\xb0\xcb\x9e\xb3\x6a\x96\x1e\x1a\x52\x4c\x40\xe7\xaa\xe6\xed\xe2\xaa\x79\x06\x53\x3b\x25\x01\x09\x70\xda\xf9\xe9\x95\xbc\x78\xde\xad\x48\x82\xf3\x09\x73\xa2\xbc\x66\xb4\xa9\xc2\x58\x0b\x29\xcd\xb6\x50\x40\xc3\x7f\x5f\xaa\xbd\x18\x8a\x5b\xaa\xb7\xc9\x66\x90\x42\xa4\xc9\x9a\x33\xa9\xd0\x4d\xa6\x42\xdf\x7b\x92\x71\xfe\xc1\x4f\x2c\x13\x5a\xe7\x83\x6b\xb5\x81\x6a\xf1\x21\xc6\xde\x6d\x87\x18\xfb\xe5\xa9\xb2\x68\x23\x93\x19\x61\xdc\xff\x08\xa3\xa9\x91\xc5\x1d\xb6\x44\x3d\x9f\x91\xc5\x23\xcc\x39\x3d\xd8\x94\x93\xda\x42\xe5\x33\xe7\x29\xec\x9c\x3a\x59\x74\x3e\xca\x1c\x84\xbf\x30\x75\x78\x40\xf0\x35\xbb\xa5\xf4\xf2\x95\xda\x2d\x75\x50\xf7\x25\xf5\x2c\xfc\x2f\xbf\x59\xea\xed\xc9\x17\xf4\x25\x1c\x72\x2d\xbd\x63\xea\xe9\xb0\xa4\x5b\x4f\xc4\x1e\x74\xab\x25\xbf\x3f\xfb\xac\x95\x35\xd9\xee\xfd\xbc\xe7\x54\x2d\x0d\x32\x56\x92\xf8\x50\xe6\xc0\x20\xa5\x30\x79\xf2\x73\x94\xd1\x20\xdb\x0f\xe2\x39\x95\xe4\x50\x1d\x65\x2c\x47\xd7\xd7\x68\xeb\x73\xe0\x9d\x62\x09\xe8\xe6\x46\xe3\x45\x7f\x2f\x3f\xd7\xd2\x0c\x27\x9a\x3d\xf9\x3a\x83\x13\x35\x36\xff\xfa\x98\x3c\x2a\x0d\xc8\x66\xb6\x65\xc5\x29\x93\x99\x54\xd1\xca\x57\x8b\x31\xd5\x7c\xdd\xb5\xdf\x14\x63\xfa\x23\x60\x12\x3f\x10\x63\xb2\x55\x66\x46\xa1\xaf\x0f\x40\xa3\xce\xb5\xbc\x80\x6e\x47\x4f\x0c\x68\x90\x60\xd5\x9c\xf0\xd4\xec\x42\xb7\xe1\x57\xa9\xd5\x8f\xa0\xea\xa0\xb3\x95\xe5\x5d\x36\x67\xca\x2a\x73\x02\x9c\xcd\xc6\xc6\xec\xea\x77\xc0\x8e\xea\x42\xfe\xb6\x08\x18\x35\x3b\xb2\x99\x5b\x88\x2e\x0e\x08\xbb\x56\x9b\x0d\x12\x3d\x21\xa7\xee\x2c\x14\x53\x11\x22\x14\xc6\x59\x1e\x45\x23\x57\x86\x8c\x0d\x7d\xfd\xfa\xf5\x6b\xfb\xe3\xc7\xf6\xbb\x77\xe8\xf7\xdf\x8f\x3c\xef\x48\x14\xd8\x95\x8f\xa5\x04\x4e\xab\xdb\x4a\x0f\xd8\x23\x8e\x03\x74\xfe\x02\x5e\xda\xad\x32\x49\x49\x04\xca\x78\x6c\x97\x25\xe0\xc9\xf2\x67\xbf\xdf\xe5\x85\xb4\xe5\x94\x02\x51\x8c\x78\x5f\xc1\x45\xe3\x8a\xb3\x94\x42\x59\xef\x38\x71\x5d\xe4\xb0\x2b\x6a\x95\x2e\xfb\xc2\xdd\x72\xaa\x97\x26\x42\x95\x20\x8b\x7e\x29\x26\x03\x56\xb3\xc3\x9c\x88\x69\x74\xd8\x5e\xae\x2e\xa0\x44\x23\x01\xb7\x93\xfe\x69\x7c\x6c\xe3\x7a\x2a\xe0\x78\x75\x14\x80\xfe\xb1\x9e\x2a\x78\xdb\xac\x0a\x62\x08\x52\x3f\x6f\xa7\x88\x0f\xc4\x23\xeb\xea\x07\xef\x1e\xdf\x0f\x22\xf1\xaf\xab\x17\xfc\xb6\x0a\x5e\x70\xc2\x9c\xd5\x93\x7e\x9e\x41\x2f\x25\xfc\x6d\x67\xfb\xfa\x1a\x6d\x7d\x4a\x66\xa3\xd0\xcd\x4d\xa9\xa0\xbd\x23\x6c\xec\x42\xfb\x22\x18\x02\xa7\x20\x41\xb4\x6d\xe6\xf9\x81\x84\x36\x87\x68\x8c\x23\xda\x3e\x73\xfe\xeb\x12\xf3\x76\x36\xc9\x95\x4d\x71\xfd\x33\xac\xf0\x99\xf3\xfa\xd7\xc1\xc0\x86\xe2\x06\x00\x4d\xe3\x7e\x51\xca\x0f\xed\x6d\x2b\xa4\x63\x4d\x2c\xdb\x5b\xaf\xb6\x6f\x2f\x17\x21\x39\xa1\xa3\xc5\xe4\x12\xff\xa5\x4d\xa2\x3d\xe7\x69\x52\x3c\x74\xa1\x38\x41\x2a\x24\x56\x03\xdb\x92\x4f\xdd\x72\xee\xb4\x76\x09\x39\x57\x7e\x87\x84\xd4\xfc\x0a\x7c\xe2\x80\xc9\xf9\xee\x42\x49\xd9\x66\x3c\x97\xda\xf7\x8c\x04\x79\xdc\x84\x20\x57\xd4\x60\xd1\x36\x32\x6a\x4e\x52\x5d\xef\xd9\x5f\x5c\xc5\xa9\x9e\xbb\x18\xdf\xad\xb7\xb7\xac\x87\x92\x7f\x7b\x98\xc5\xb6\xf9\x6b\x6a\x6a\xca\xfe\x51\xd2\x90\x38\xa6\x22\x54\x42\x59\x05\x29\x1d\x2a\x14\x9b\xf5\xb6\x75\x59\x6f\x7b\x81\x1e\x69\x89\xec\xb0\xe6\x24\x83\x6c\x2b\xfc\x1d\x96\xc8\x3e\xaa\xad\x1c\x26\xaf\xa8\x56\xf8\x35\x1b\x3c\xb3\x84\xde\x35\x5e\xf6\x5a\xf5\x04\xa2\x95\x4c\x06\x5a\x70\x1b\xd5\x32\xc4\xa2\x85\xd2\xc6\xff\xe3\xf5\x86\x49\xf5\x69\x2e\xd5\x47\x0f\x93\x26\xdb\xe7\x6e\xec\x43\x59\xba\x61\x1f\xeb\x98\xed\xd3\xed\xd4\x6c\x66\xda\x6f\x90\xcb\x98\x8c\x9f\x66\xa8\x4f\xb7\xd3\xaf\x56\xd6\x8e\xe1\x3e\x4f\x37\xb9\xe7\xb1\x12\x75\x2a\x3f\x28\x66\xf2\x74\xd2\x6e\x55\x32\x8c\x44\xa4\x8f\xbe\x44\xf8\x04\x53\x75\x1c\xb0\x23\xa6\x71\x6b\x1d\x98\x84\x9d\x95\x52\xc3\xba\x26\x2c\xac\x44\xda\x4e\xac\x0b\x93\xb9\xd3\x98\x1e\x96\x77\x08\x93\xbf\xd3\xa0\x1a\x4c\xfe\x8e\xc9\xdf\x31\xf9\x3b\x8d\xe6\xef\xac\xd6\xcc\xe6\xb3\xce\xce\x89\x05\xdc\x84\x68\x57\x54\x96\x8d\x24\xe8\x3c\x8a\x45\xde\x26\xfd\xe6\xf9\xeb\xf1\xa1\x32\x70\x9e\xbf\x24\x1b\x49\xc2\x59\x55\x8f\x58\x1b\x2d\xae\x44\x96\x8d\x3e\x53\x6e\x12\x6d\x4c\xa2\xcd\x4a\x2d\x75\xbd\x40\x8f\xb5\x3a\xd5\xed\x95\x54\x1d\xed\x71\x6e\x64\x37\xfa\x27\x90\x57\x8c\x5f\x98\x5c\x9b\x7a\xf9\xd7\x1d\x36\x9f\x2d\x0f\x9a\x15\xa7\x95\xfb\x40\xe1\x4a\x66\xd9\x10\x8e\x25\x68\x6c\x87\x46\xce\x37\xe0\x60\x03\xb9\x8c\x09\x4f\xe9\x43\x83\x4b\xf1\x8c\xc6\xbf\x34\xb8\x74\xaa\xcd\xca\x7d\x69\xf0\x34\x92\x36\x3a\xc6\xd4\x89\x6c\xfc\x4e\x64\x63\x5d\x73\x67\xb4\xc9\xa1\x12\x96\x66\x06\x76\xec\xdf\x6b\x56\xcd\x13\xff\xec\xa0\xf9\x34\xf1\x3d\x00\x76\xaf\xe6\x7b\x87\xd9\x79\x5f\x06\xb0\x0d\x60\xdf\x09\xb0\xd5\x50\xd1\x23\xd2\x20\xf6\x83\x20\xf6\x59\x2c\x6e\x03\xd9\x4d\xcd\x01\x18\x58\x5e\x0d\x58\x7e\x11\x37\x1b\xfa\x5c\xe8\x4e\x51\xea\x64\xa4\x1b\x4b\xd8\x63\xf0\x70\x76\x5c\x71\x04\x60\x51\x52\x99\x1a\xd8\x63\x7e\x11\x5d\x29\xf1\x28\x53\xbb\x15\x2d\x89\xc6\x82\xb3\x7e\x4e\xa9\x6d\xa5\xcf\x91\xe0\xf9\x2e\x96\x84\x8e\xd2\xfe\x5b\x2e\x11\x52\x33\x1a\x1d\x80\xa3\x23\x2f\x75\x88\x25\xd4\x76\x03\x07\xde\x54\x9f\x28\x58\xa9\x1e\xcb\x0b\x5c\x49\x2a\x2e\x4f\xce\x9d\xac\x60\x08\x39\x28\xd2\x17\x11\xad\x1f\x01\xf0\xa9\x5a\x90\xe5\xcc\x03\x39\x86\x40\x37\x65\x4d\x90\xdd\x5c\xe9\x08\x26\x85\x09\x50\x4b\x5c\x10\xff\x0b\x77\x3f\x87\x02\x2a\x77\x2e\x71\x77\xad\x73\x45\x7f\xcb\xe9\xdf\x4d\x4e\x3f\x2d\xbc\x7c\xc6\x20\x72\x66\x9e\xa8\xac\xb0\x3e\x1a\x9f\x7d\x57\x5c\xe6\xce\x7b\x41\x7a\x14\x5e\xe9\xb2\x2a\xef\x5a\x5a\x8b\x56\x0a\x62\xd6\x2d\x94\x59\x79\x93\xa6\x4b\xed\x45\x74\x81\x28\x0a\xe0\x46\x27\x82\x96\xe7\xb9\x17\x15\xcd\x62\xc2\xc9\x7c\x5a\xf3\x6a\xdd\xb4\x66\x3c\x63\x41\xab\xb1\x03\x21\x99\xb7\xa0\xc5\xe4\xac\x72\x3e\x7b\x56\x83\xaf\x73\x42\x49\xf2\xc9\x3b\xa5\xb0\x41\x04\x0c\xd9\x82\x06\xa1\xe7\x6c\xf6\x79\x05\x61\x68\x68\x6f\xbd\xda\xb8\x69\xa1\x02\xe5\x98\x6b\x33\x05\x50\x4a\x4d\xa6\x62\xb2\x71\x91\x00\x50\x79\xd7\xbc\x00\x70\x0f\xaf\x3d\x27\x7a\x6c\x6f\x6e\xbd\x7a\x59\x45\xdf\xb6\x17\xb7\x90\x98\x83\xe8\xed\x4b\x3c\x52\xa6\x20\xfe\x48\x5e\xcd\xca\xd7\x96\x64\x10\x96\x55\x5f\x1c\xdb\x5f\x24\x24\xad\x22\x10\x70\x16\x35\x94\x1b\x55\xa8\x7f\x43\x22\x71\x13\xc1\x03\x51\x0a\x89\x81\x21\x3d\xd8\x96\x5d\xb5\xbb\x09\xe5\x49\xce\xae\x8d\xe6\x82\xb3\xdb\x7c\x62\x5f\x28\x62\x1f\xdf\x1c\x8b\x72\x90\xf0\x52\xdd\xf9\xad\xbe\x76\x2c\x69\x57\x3f\xa3\x34\x77\x60\x69\xd7\xcb\xfe\xee\x6b\x7f\x77\xf5\x1f\x3b\x1d\xbd\x46\x23\x66\x3d\xed\xef\xae\x13\xb9\xe2\xf7\xe4\x1d\x42\x16\x5d\x0e\x4a\xf5\x4f\xd1\x1b\xde\xd3\x1b\xd6\x9f\xd2\xdb\xd5\x7f\x68\x27\x1b\xef\x3b\x7a\x7f\x93\xbe\xe4\xc4\xf7\x93\xd1\x2c\x14\x64\xd4\x37\xc2\x72\xb4\x8d\xfe\x0a\xad\x49\xd5\x5d\x66\x84\xe0\xc5\xcd\x8b\xff\x0f\x00\x00\xff\xff\x23\x65\x66\x79\xdc\x13\x01\x00") func monitoringZyncGrafanaDashboard1JsonTplBytes() ([]byte, error) { return bindataRead( From 05bf6a03e30763435079557ec8a1259ea11bf70e Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Fri, 26 Jan 2024 13:51:22 -0500 Subject: [PATCH 15/23] THREESCALE-10698 Updated DeploymentConfig references in CSV/CRD --- apis/apps/v1alpha1/apimanager_types.go | 4 ++-- bundle/manifests/3scale-operator.clusterserviceversion.yaml | 6 +++--- bundle/manifests/apps.3scale.net_apimanagers.yaml | 2 +- config/crd/bases/apps.3scale.net_apimanagers.yaml | 2 +- .../bases/3scale-operator.clusterserviceversion.yaml | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apis/apps/v1alpha1/apimanager_types.go b/apis/apps/v1alpha1/apimanager_types.go index 072b7e779..ad646d720 100644 --- a/apis/apps/v1alpha1/apimanager_types.go +++ b/apis/apps/v1alpha1/apimanager_types.go @@ -98,7 +98,7 @@ type APIManagerStatus struct { // +patchStrategy=merge Conditions common.Conditions `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,2,rep,name=conditions"` - // APIManager Deployment Configs + // APIManager Deployments // +operator-sdk:csv:customresourcedefinitions:type=status,displayName="Deployments",xDescriptors="urn:alm:descriptor:com.tectonic.ui:podStatuses" Deployments olm.DeploymentStatus `json:"deployments"` } @@ -130,7 +130,7 @@ func (s *APIManagerStatus) Equals(other *APIManagerStatus, logger logr.Logger) b // APIManager is the Schema for the apimanagers API // +kubebuilder:resource:path=apimanagers,scope=Namespaced // +operator-sdk:csv:customresourcedefinitions:displayName="APIManager" -// +operator-sdk:csv:customresourcedefinitions:resources={{"Deployment","apps.openshift.io/v1"}} +// +operator-sdk:csv:customresourcedefinitions:resources={{"Deployment","apps/v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"PersistentVolumeClaim","v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"Service","v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"Route","route.openshift.io/v1"}} diff --git a/bundle/manifests/3scale-operator.clusterserviceversion.yaml b/bundle/manifests/3scale-operator.clusterserviceversion.yaml index a9a3fcca2..cae638510 100644 --- a/bundle/manifests/3scale-operator.clusterserviceversion.yaml +++ b/bundle/manifests/3scale-operator.clusterserviceversion.yaml @@ -242,9 +242,9 @@ spec: kind: APIManager name: apimanagers.apps.3scale.net resources: - - kind: DeploymentConfig + - kind: Deployment name: "" - version: apps.openshift.io/v1 + version: apps/v1 - kind: ImageStream name: "" version: image.openshift.io/v1 @@ -264,7 +264,7 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:label statusDescriptors: - - description: APIManager Deployment Configs + - description: APIManager Deployments displayName: Deployments path: deployments x-descriptors: diff --git a/bundle/manifests/apps.3scale.net_apimanagers.yaml b/bundle/manifests/apps.3scale.net_apimanagers.yaml index a766ab353..77a3295af 100644 --- a/bundle/manifests/apps.3scale.net_apimanagers.yaml +++ b/bundle/manifests/apps.3scale.net_apimanagers.yaml @@ -10540,7 +10540,7 @@ spec: type: object type: array deployments: - description: APIManager Deployment Configs + description: APIManager Deployments properties: ready: description: Deployments are ready to serve requests diff --git a/config/crd/bases/apps.3scale.net_apimanagers.yaml b/config/crd/bases/apps.3scale.net_apimanagers.yaml index 7a9d90ab3..1e8317584 100644 --- a/config/crd/bases/apps.3scale.net_apimanagers.yaml +++ b/config/crd/bases/apps.3scale.net_apimanagers.yaml @@ -19931,7 +19931,7 @@ spec: type: object type: array deployments: - description: APIManager Deployment Configs + description: APIManager Deployments properties: ready: description: Deployments are ready to serve requests diff --git a/config/manifests/bases/3scale-operator.clusterserviceversion.yaml b/config/manifests/bases/3scale-operator.clusterserviceversion.yaml index 4533a416d..f6bb04200 100644 --- a/config/manifests/bases/3scale-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/3scale-operator.clusterserviceversion.yaml @@ -56,9 +56,9 @@ spec: kind: APIManager name: apimanagers.apps.3scale.net resources: - - kind: DeploymentConfig + - kind: Deployment name: "" - version: apps.openshift.io/v1 + version: apps/v1 - kind: ImageStream name: "" version: image.openshift.io/v1 @@ -78,7 +78,7 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:label statusDescriptors: - - description: APIManager Deployment Configs + - description: APIManager Deployments displayName: Deployments path: deployments x-descriptors: From 3bc9b0d10ca1e047432b95f4c27c035cecadf194 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Tue, 30 Jan 2024 17:44:02 -0500 Subject: [PATCH 16/23] THREESCALE-10698 Added mutator for Deployment container images --- pkg/3scale/amp/operator/apicast_reconciler.go | 3 + .../amp/operator/memcached_reconciler.go | 1 + pkg/3scale/amp/operator/redis_reconciler.go | 1 + .../amp/operator/system_mysql_reconciler.go | 1 + .../operator/system_postgresql_reconciler.go | 1 + pkg/3scale/amp/operator/system_reconciler.go | 3 + .../amp/operator/system_searchd_reconciler.go | 1 + pkg/3scale/amp/operator/zync_reconciler.go | 4 + pkg/reconcilers/deployment.go | 32 ++++ pkg/reconcilers/deployment_test.go | 166 ++++++++++++++++++ 10 files changed, 213 insertions(+) diff --git a/pkg/3scale/amp/operator/apicast_reconciler.go b/pkg/3scale/amp/operator/apicast_reconciler.go index a43a4173b..6a496092b 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler.go +++ b/pkg/3scale/amp/operator/apicast_reconciler.go @@ -92,6 +92,7 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, } if r.apiManager.Spec.Apicast.StagingSpec.Replicas != nil { @@ -139,6 +140,8 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, + reconcilers.DeploymentPodInitContainerImageMutator, } if r.apiManager.Spec.Apicast.ProductionSpec.Replicas != nil { diff --git a/pkg/3scale/amp/operator/memcached_reconciler.go b/pkg/3scale/amp/operator/memcached_reconciler.go index 45f2c6405..59d3fc84e 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler.go +++ b/pkg/3scale/amp/operator/memcached_reconciler.go @@ -38,6 +38,7 @@ func (r *MemcachedReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) err = r.ReconcileDeployment(memcached.Deployment(ampImages.Options.SystemMemcachedImage), mutator) if err != nil { diff --git a/pkg/3scale/amp/operator/redis_reconciler.go b/pkg/3scale/amp/operator/redis_reconciler.go index 10c17ae65..ac69d2cfd 100644 --- a/pkg/3scale/amp/operator/redis_reconciler.go +++ b/pkg/3scale/amp/operator/redis_reconciler.go @@ -68,6 +68,7 @@ func (r *RedisReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) redisDeployment := r.Deployment(redis) err = r.ReconcileDeployment(redisDeployment, deploymentMutator) diff --git a/pkg/3scale/amp/operator/system_mysql_reconciler.go b/pkg/3scale/amp/operator/system_mysql_reconciler.go index 36835fe91..8c30188d4 100644 --- a/pkg/3scale/amp/operator/system_mysql_reconciler.go +++ b/pkg/3scale/amp/operator/system_mysql_reconciler.go @@ -39,6 +39,7 @@ func (r *SystemMySQLReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) err = r.ReconcileDeployment(systemMySQL.Deployment(systemMySQLImage.Options.Image), deploymentMutator) if err != nil { diff --git a/pkg/3scale/amp/operator/system_postgresql_reconciler.go b/pkg/3scale/amp/operator/system_postgresql_reconciler.go index bf395270b..8d91316d1 100644 --- a/pkg/3scale/amp/operator/system_postgresql_reconciler.go +++ b/pkg/3scale/amp/operator/system_postgresql_reconciler.go @@ -40,6 +40,7 @@ func (r *SystemPostgreSQLReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) err = r.ReconcileDeployment(systemPostgreSQL.Deployment(systemPostgreSQLImage.Options.Image), deploymentMutator) if err != nil { diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index a73225a8f..7b7884368 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -168,6 +168,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPodTemplateAnnotationsMutator, r.systemAppDeploymentResourceMutator, reconcilers.DeploymentRemoveDuplicateEnvVarMutator, + reconcilers.DeploymentPodContainerImageMutator, } if r.apiManager.Spec.System.AppSpec.Replicas != nil { systemAppDeploymentMutators = append(systemAppDeploymentMutators, reconcilers.DeploymentReplicasMutator) @@ -221,6 +222,8 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, + reconcilers.DeploymentPodInitContainerImageMutator, } if r.apiManager.Spec.System.SidekiqSpec.Replicas != nil { sidekiqDeploymentMutators = append(sidekiqDeploymentMutators, reconcilers.DeploymentReplicasMutator) diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler.go b/pkg/3scale/amp/operator/system_searchd_reconciler.go index 954acb9ed..d19a42ffa 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler.go @@ -59,6 +59,7 @@ func (r *SystemSearchdReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPodTemplateAnnotationsMutator, reconcilers.DeploymentProbesMutator, reconcilers.DeploymentArgsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) err = r.ReconcileDeployment(searchd.Deployment(ampImages.Options.SystemSearchdImage), searchdDeploymentMutator) if err != nil { diff --git a/pkg/3scale/amp/operator/zync_reconciler.go b/pkg/3scale/amp/operator/zync_reconciler.go index 2973ea0db..eaabbed81 100644 --- a/pkg/3scale/amp/operator/zync_reconciler.go +++ b/pkg/3scale/amp/operator/zync_reconciler.go @@ -59,6 +59,8 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, + reconcilers.DeploymentPodInitContainerImageMutator, } if r.apiManager.Spec.Zync.AppSpec.Replicas != nil { zyncMutators = append(zyncMutators, reconcilers.DeploymentReplicasMutator) @@ -86,6 +88,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, } if r.apiManager.Spec.Zync.QueSpec.Replicas != nil { zyncQueMutators = append(zyncQueMutators, reconcilers.DeploymentReplicasMutator) @@ -125,6 +128,7 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { reconcilers.DeploymentPriorityClassMutator, reconcilers.DeploymentTopologySpreadConstraintsMutator, reconcilers.DeploymentPodTemplateAnnotationsMutator, + reconcilers.DeploymentPodContainerImageMutator, ) err = r.ReconcileDeployment(zync.DatabaseDeployment(ampImages.Options.ZyncDatabasePostgreSQLImage), zyncDBDMutator) if err != nil { diff --git a/pkg/reconcilers/deployment.go b/pkg/reconcilers/deployment.go index a4e25f36a..e3b4e1ab7 100644 --- a/pkg/reconcilers/deployment.go +++ b/pkg/reconcilers/deployment.go @@ -81,6 +81,8 @@ func GenericBackendDeploymentMutators() []DMutateFn { DeploymentPodTemplateAnnotationsMutator, DeploymentArgsMutator, DeploymentProbesMutator, + DeploymentPodContainerImageMutator, + DeploymentPodInitContainerImageMutator, } } @@ -303,3 +305,33 @@ func DeploymentProbesMutator(desired, existing *k8sappsv1.Deployment) (bool, err return updated, nil } + +// DeploymentPodContainerImageMutator ensures that the deployment's pod's containers are reconciled +func DeploymentPodContainerImageMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + for i, desiredContainer := range desired.Spec.Template.Spec.Containers { + existingContainer := &existing.Spec.Template.Spec.Containers[i] + + if !reflect.DeepEqual(existingContainer.Image, desiredContainer.Image) { + existingContainer.Image = desiredContainer.Image + updated = true + } + } + return updated, nil +} + +// DeploymentPodInitContainerImageMutator ensures that the deployment's pod's containers are reconciled +func DeploymentPodInitContainerImageMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { + updated := false + + for i, desiredContainer := range desired.Spec.Template.Spec.InitContainers { + existingContainer := &existing.Spec.Template.Spec.InitContainers[i] + + if !reflect.DeepEqual(existingContainer.Image, desiredContainer.Image) { + existingContainer.Image = desiredContainer.Image + updated = true + } + } + return updated, nil +} diff --git a/pkg/reconcilers/deployment_test.go b/pkg/reconcilers/deployment_test.go index 443ab7a70..ae9ddbac9 100644 --- a/pkg/reconcilers/deployment_test.go +++ b/pkg/reconcilers/deployment_test.go @@ -1153,3 +1153,169 @@ func TestDeploymentAnnotationsMutator(t *testing.T) { }) } } + +func TestDeploymentPodContainerImageMutator(t *testing.T) { + type args struct { + desired *k8sappsv1.Deployment + existing *k8sappsv1.Deployment + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "No Image Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image"}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image"}, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, + { + name: "Image Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image-desired"}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image-existing"}, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeploymentPodContainerImageMutator(tt.args.desired, tt.args.existing) + if (err != nil) != tt.wantErr { + t.Errorf("DeploymentPodContainerImageMutator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DeploymentPodContainerImageMutator() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDeploymentPodInitContainerImageMutator(t *testing.T) { + type args struct { + desired *k8sappsv1.Deployment + existing *k8sappsv1.Deployment + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "No Image Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image"}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Image: "test-image"}, + }, + }, + }, + }, + }, + }, + want: false, + wantErr: false, + }, + { + name: "Image Update Required", + args: args{ + desired: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Image: "test-image-desired"}, + }, + }, + }, + }, + }, + existing: &k8sappsv1.Deployment{ + Spec: k8sappsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Image: "test-image-existing"}, + }, + }, + }, + }, + }, + }, + want: true, + wantErr: false, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeploymentPodInitContainerImageMutator(tt.args.desired, tt.args.existing) + if (err != nil) != tt.wantErr { + t.Errorf("DeploymentPodInitContainerImageMutator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DeploymentPodInitContainerImageMutator() = %v, want %v", got, tt.want) + } + }) + } +} From 03036e12160d3711ec6c92a4ca5e65914044302c Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Tue, 6 Feb 2024 17:18:34 -0500 Subject: [PATCH 17/23] THREESCALE-10722 Refactored system-app Jobs to be recreated for each image change --- pkg/3scale/amp/component/system.go | 2 +- pkg/3scale/amp/operator/system_reconciler.go | 26 +++++++- .../amp/operator/system_reconciler_test.go | 22 +++++++ pkg/helper/deployment.go | 18 +++++- pkg/helper/job.go | 64 +++++++++++++++++++ 5 files changed, 127 insertions(+), 5 deletions(-) diff --git a/pkg/3scale/amp/component/system.go b/pkg/3scale/amp/component/system.go index 7ad4bda2e..d0458ceb4 100644 --- a/pkg/3scale/amp/component/system.go +++ b/pkg/3scale/amp/component/system.go @@ -931,7 +931,7 @@ func (system *System) SidekiqDeployment(containerImage string) *k8sappsv1.Deploy Volumes: system.SidekiqPodVolumes(), InitContainers: []v1.Container{ { - Name: "check-svc", + Name: SystemSideKiqInitContainerName, Image: containerImage, Command: []string{ "bash", diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index 7b7884368..d69993013 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -146,6 +146,28 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { // Used to synchronize rollout of system Deployments systemComponentNotReady := false + // If the image has changed, delete the PreHook/PostHook Jobs so they can be recreated with the new image + imageChanged, err := helper.HasJobImageChanged(component.SystemAppPreHookJobName, r.apiManager.GetNamespace(), ampImages.Options.SystemImage, r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if imageChanged { + err = helper.DeleteJob(component.SystemAppPreHookJobName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + } + imageChanged, err = helper.HasJobImageChanged(component.SystemAppPostHookJobName, r.apiManager.GetNamespace(), ampImages.Options.SystemImage, r.Client()) + if err != nil { + return reconcile.Result{}, err + } + if imageChanged { + err = helper.DeleteJob(component.SystemAppPostHookJobName, r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + } + // SystemApp PreHook Job preHookJob := system.AppPreHookJob(ampImages.Options.SystemImage) err = r.ReconcileJob(preHookJob, reconcilers.CreateOnlyMutator) @@ -180,7 +202,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { } } - // Block reconciling PostHook Job unless system-app Deployment is ready + // Block reconciling PostHook Job unless BOTH the PreHook Job has completed and the system-app Deployment is ready and not in the process of updating deployment := &k8sappsv1.Deployment{} err = r.Client().Get(context.TODO(), client.ObjectKey{ Namespace: r.apiManager.GetNamespace(), @@ -189,7 +211,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { if err != nil && !k8serr.IsNotFound(err) { return reconcile.Result{}, err } - if k8serr.IsNotFound(err) || !helper.IsDeploymentAvailable(deployment) { + if k8serr.IsNotFound(err) || !helper.IsDeploymentAvailable(deployment) || helper.IsDeploymentProgressing(deployment) || !helper.HasJobCompleted(preHookJob.Name, preHookJob.Namespace, r.Client()) { systemComponentNotReady = true } diff --git a/pkg/3scale/amp/operator/system_reconciler_test.go b/pkg/3scale/amp/operator/system_reconciler_test.go index 8bd3127e2..7b1ce7a33 100644 --- a/pkg/3scale/amp/operator/system_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_reconciler_test.go @@ -39,6 +39,17 @@ func TestSystemReconcilerCreate(t *testing.T) { apimanager := basicApimanagerSpecTestSystemOptions() appPreHookJob := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: component.SystemAppPreHookJobName, Namespace: apimanager.Namespace}, + Spec: batchv1.JobSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: SystemImageURL(), + }, + }, + }, + }, + }, Status: batchv1.JobStatus{ Conditions: []batchv1.JobCondition{ { @@ -149,6 +160,17 @@ func TestReplicaSystemReconciler(t *testing.T) { appPreHookJob := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: component.SystemAppPreHookJobName, Namespace: namespace}, + Spec: batchv1.JobSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: SystemImageURL(), + }, + }, + }, + }, + }, Status: batchv1.JobStatus{ Conditions: []batchv1.JobCondition{ { diff --git a/pkg/helper/deployment.go b/pkg/helper/deployment.go index c759918af..0c48acf53 100644 --- a/pkg/helper/deployment.go +++ b/pkg/helper/deployment.go @@ -5,8 +5,7 @@ import ( corev1 "k8s.io/api/core/v1" ) -// IsDeploymentAvailable returns true when the provided Deployment -// has the "Available" condition set to true +// IsDeploymentAvailable returns true when the provided Deployment has the "Available" condition set to true func IsDeploymentAvailable(d *k8sappsv1.Deployment) bool { dConditions := d.Status.Conditions for _, dCondition := range dConditions { @@ -16,3 +15,18 @@ func IsDeploymentAvailable(d *k8sappsv1.Deployment) bool { } return false } + +// IsDeploymentProgressing returns true when the provided Deployment is progressing with new ReplicaSet +func IsDeploymentProgressing(d *k8sappsv1.Deployment) bool { + if d.Status.UnavailableReplicas > 0 { + return true + } + + for _, dCondition := range d.Status.Conditions { + if dCondition.Type == k8sappsv1.DeploymentProgressing && dCondition.Reason == "ReplicaSetUpdated" { + return true + } + } + + return false +} diff --git a/pkg/helper/job.go b/pkg/helper/job.go index 4c85f531b..c67f5bc1f 100644 --- a/pkg/helper/job.go +++ b/pkg/helper/job.go @@ -7,6 +7,8 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,3 +56,65 @@ func HasJobCompleted(jName string, jNamespace string, client k8sclient.Client) b return false } + +// HasJobImageChanged returns true if the Job's existing image is different from the provided image +func HasJobImageChanged(jName string, jNamespace string, desiredImage string, client k8sclient.Client) (bool, error) { + job := &batchv1.Job{} + err := client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: jNamespace, + Name: jName, + }, job) + + // Return error if can't get Job + if err != nil && !k8serr.IsNotFound(err) { + return false, fmt.Errorf("error getting job %s: %w", job.Name, err) + } + + // Return false if the Job doesn't exist yet + if k8serr.IsNotFound(err) { + return false, nil + } + + // Return true if the Job is using the old image + if job.Spec.Template.Spec.Containers[0].Image != desiredImage { + return true, nil + } + + return false, nil +} + +func DeleteJob(jName string, jNamespace string, client k8sclient.Client) error { + job := &batchv1.Job{} + err := client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: jNamespace, + Name: jName, + }, job) + + // Breakout if the Job has already been deleted + if k8serr.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("error getting job %s: %v", job.Name, err) + } + + // Return error if Job is currently running + if !HasJobCompleted(job.Name, job.Namespace, client) { + return fmt.Errorf("won't delete job %s because it's still running", job.Name) + } + + deleteOptions := []k8sclient.DeleteOption{ + k8sclient.GracePeriodSeconds(int64(0)), + k8sclient.PropagationPolicy(metav1.DeletePropagationForeground), + } + + // Delete the Job + err = client.Delete(context.TODO(), job, deleteOptions...) + if err != nil { + if !k8serr.IsNotFound(err) { + return fmt.Errorf("error deleting job %s: %v", job.Name, err) + } + } + + return nil +} From 939d7b208d42c34930dfb5815d827159fcfecfde Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Wed, 14 Feb 2024 15:01:21 -0500 Subject: [PATCH 18/23] THREESCALE-10722 Refactored system-app Jobs to be recreated on Deployment generation change --- pkg/3scale/amp/component/system.go | 10 +++- pkg/3scale/amp/operator/system_reconciler.go | 46 +++++++++++++++---- .../amp/operator/system_reconciler_test.go | 1 + pkg/helper/job.go | 45 +++++++++++++++--- 4 files changed, 83 insertions(+), 19 deletions(-) diff --git a/pkg/3scale/amp/component/system.go b/pkg/3scale/amp/component/system.go index d0458ceb4..6ad6a9d78 100644 --- a/pkg/3scale/amp/component/system.go +++ b/pkg/3scale/amp/component/system.go @@ -746,7 +746,7 @@ func (system *System) AppDeployment(containerImage string) *k8sappsv1.Deployment } } -func (system *System) AppPreHookJob(containerImage string) *batchv1.Job { +func (system *System) AppPreHookJob(containerImage string, currentSystemAppGeneration int64) *batchv1.Job { var completions int32 = 1 return &batchv1.Job{ @@ -757,6 +757,9 @@ func (system *System) AppPreHookJob(containerImage string) *batchv1.Job { ObjectMeta: metav1.ObjectMeta{ Name: SystemAppPreHookJobName, Labels: system.Options.CommonAppLabels, + Annotations: map[string]string{ + helper.SystemAppGenerationAnnotation: strconv.FormatInt(currentSystemAppGeneration, 10), + }, }, Spec: batchv1.JobSpec{ Completions: &completions, @@ -783,7 +786,7 @@ func (system *System) AppPreHookJob(containerImage string) *batchv1.Job { } } -func (system *System) AppPostHookJob(containerImage string) *batchv1.Job { +func (system *System) AppPostHookJob(containerImage string, currentSystemAppGeneration int64) *batchv1.Job { var completions int32 = 1 return &batchv1.Job{ @@ -794,6 +797,9 @@ func (system *System) AppPostHookJob(containerImage string) *batchv1.Job { ObjectMeta: metav1.ObjectMeta{ Name: SystemAppPostHookJobName, Labels: system.Options.CommonAppLabels, + Annotations: map[string]string{ + helper.SystemAppGenerationAnnotation: strconv.FormatInt(currentSystemAppGeneration, 10), + }, }, Spec: batchv1.JobSpec{ Completions: &completions, diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index d69993013..d1f599c8d 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -8,7 +8,7 @@ import ( k8sappsv1 "k8s.io/api/apps/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" - "sigs.k8s.io/controller-runtime/pkg/client" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -146,30 +146,36 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { // Used to synchronize rollout of system Deployments systemComponentNotReady := false - // If the image has changed, delete the PreHook/PostHook Jobs so they can be recreated with the new image - imageChanged, err := helper.HasJobImageChanged(component.SystemAppPreHookJobName, r.apiManager.GetNamespace(), ampImages.Options.SystemImage, r.Client()) + // If the system-app Deployment generation has changed, delete the PreHook/PostHook Jobs so they can be recreated + generationChanged, err := helper.HasAppGenerationChanged(component.SystemAppPreHookJobName, component.SystemAppDeploymentName, r.apiManager.GetNamespace(), r.Client()) if err != nil { return reconcile.Result{}, err } - if imageChanged { + if generationChanged { err = helper.DeleteJob(component.SystemAppPreHookJobName, r.apiManager.GetNamespace(), r.Client()) if err != nil { return reconcile.Result{}, err } } - imageChanged, err = helper.HasJobImageChanged(component.SystemAppPostHookJobName, r.apiManager.GetNamespace(), ampImages.Options.SystemImage, r.Client()) + generationChanged, err = helper.HasAppGenerationChanged(component.SystemAppPostHookJobName, component.SystemAppDeploymentName, r.apiManager.GetNamespace(), r.Client()) if err != nil { return reconcile.Result{}, err } - if imageChanged { + if generationChanged { err = helper.DeleteJob(component.SystemAppPostHookJobName, r.apiManager.GetNamespace(), r.Client()) if err != nil { return reconcile.Result{}, err } } + // Used to synchronize the system-app Deployment with the PreHook/PostHook Jobs + currentAppDeploymentGeneration, err := getSystemAppDeploymentGeneration(r.apiManager.GetNamespace(), r.Client()) + if err != nil { + return reconcile.Result{}, err + } + // SystemApp PreHook Job - preHookJob := system.AppPreHookJob(ampImages.Options.SystemImage) + preHookJob := system.AppPreHookJob(ampImages.Options.SystemImage, currentAppDeploymentGeneration) err = r.ReconcileJob(preHookJob, reconcilers.CreateOnlyMutator) if err != nil { return reconcile.Result{}, err @@ -204,7 +210,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { // Block reconciling PostHook Job unless BOTH the PreHook Job has completed and the system-app Deployment is ready and not in the process of updating deployment := &k8sappsv1.Deployment{} - err = r.Client().Get(context.TODO(), client.ObjectKey{ + err = r.Client().Get(context.TODO(), k8sclient.ObjectKey{ Namespace: r.apiManager.GetNamespace(), Name: component.SystemAppDeploymentName, }, deployment) @@ -217,7 +223,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { // SystemApp PostHook Job if !systemComponentNotReady { - err = r.ReconcileJob(system.AppPostHookJob(ampImages.Options.SystemImage), reconcilers.CreateOnlyMutator) + err = r.ReconcileJob(system.AppPostHookJob(ampImages.Options.SystemImage, currentAppDeploymentGeneration), reconcilers.CreateOnlyMutator) if err != nil { return reconcile.Result{}, err } @@ -315,6 +321,26 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, nil } +func getSystemAppDeploymentGeneration(namespace string, client k8sclient.Client) (int64, error) { + deployment := &k8sappsv1.Deployment{} + err := client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: namespace, + Name: component.SystemAppDeploymentName, + }, deployment) + + // Return error if can't get Deployment + if err != nil && !k8serr.IsNotFound(err) { + return 0, fmt.Errorf("error getting deployment %s: %w", deployment.Name, err) + } + + // Return 1 if the Deployment doesn't exist yet + if k8serr.IsNotFound(err) { + return 1, nil + } + + return deployment.Generation, nil +} + func (r *SystemReconciler) systemAppDeploymentResourceMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { desiredName := common.ObjectInfo(desired) update := false @@ -343,7 +369,7 @@ func (r *SystemReconciler) systemAppDeploymentResourceMutator(desired, existing return update, nil } -func System(cr *appsv1alpha1.APIManager, client client.Client) (*component.System, error) { +func System(cr *appsv1alpha1.APIManager, client k8sclient.Client) (*component.System, error) { optsProvider := NewSystemOptionsProvider(cr, cr.Namespace, client) opts, err := optsProvider.GetSystemOptions() if err != nil { diff --git a/pkg/3scale/amp/operator/system_reconciler_test.go b/pkg/3scale/amp/operator/system_reconciler_test.go index 7b1ce7a33..68c3c2841 100644 --- a/pkg/3scale/amp/operator/system_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_reconciler_test.go @@ -245,6 +245,7 @@ func TestReplicaSystemReconciler(t *testing.T) { // bump the amount of replicas in the deployment deployment.Spec.Replicas = &twoValue + deployment.Generation = oneValue64 err = cl.Update(context.TODO(), deployment) if err != nil { subT.Errorf("error updating deployment of %s: %v", tc.objName, err) diff --git a/pkg/helper/job.go b/pkg/helper/job.go index c67f5bc1f..07df6853a 100644 --- a/pkg/helper/job.go +++ b/pkg/helper/job.go @@ -3,8 +3,10 @@ package helper import ( "context" "fmt" + "strconv" "strings" + k8sappsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" @@ -14,6 +16,10 @@ import ( k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + SystemAppGenerationAnnotation = "system-app-deployment-generation" +) + // UIDBasedJobName returns a Job name that is compromised of the provided prefix, // a hyphen and the provided uid: "-". The returned name is a // DNS1123 Label compliant name. Due to UIDs are 36 characters of length this @@ -57,26 +63,51 @@ func HasJobCompleted(jName string, jNamespace string, client k8sclient.Client) b return false } -// HasJobImageChanged returns true if the Job's existing image is different from the provided image -func HasJobImageChanged(jName string, jNamespace string, desiredImage string, client k8sclient.Client) (bool, error) { +// HasAppGenerationChanged returns true if the system-app Deployment's generation doesn't match the Job's annotation tracking it +func HasAppGenerationChanged(jName string, dName string, namespace string, client k8sclient.Client) (bool, error) { job := &batchv1.Job{} err := client.Get(context.TODO(), k8sclient.ObjectKey{ - Namespace: jNamespace, + Namespace: namespace, Name: jName, }, job) - // Return error if can't get Job if err != nil && !k8serr.IsNotFound(err) { return false, fmt.Errorf("error getting job %s: %w", job.Name, err) } - // Return false if the Job doesn't exist yet if k8serr.IsNotFound(err) { return false, nil } - // Return true if the Job is using the old image - if job.Spec.Template.Spec.Containers[0].Image != desiredImage { + deployment := &k8sappsv1.Deployment{} + err = client.Get(context.TODO(), k8sclient.ObjectKey{ + Namespace: namespace, + Name: dName, + }, deployment) + // Return error if can't get Deployment + if err != nil && !k8serr.IsNotFound(err) { + return false, fmt.Errorf("error getting deployment %s: %w", deployment.Name, err) + } + // Return false if the Deployment doesn't exist yet + if k8serr.IsNotFound(err) { + return false, nil + } + + // Parse the Job's observed Deployment generation from its annotations + var trackedGeneration int64 = 1 + if job.Annotations != nil { + for key, val := range job.Annotations { + if key == SystemAppGenerationAnnotation { + trackedGeneration, err = strconv.ParseInt(val, 10, 64) + if err != nil { + return false, fmt.Errorf("failed to parse system-app Deployment's generation from job %s annotations: %w", job.Name, err) + } + } + } + } + + // Return true if the Deployment's version doesn't match the version tracked in the Job's annotation + if trackedGeneration != deployment.ObjectMeta.Generation { return true, nil } From c1d07abe9f39485aefdcadc4ede0c26db12f5e0f Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Wed, 14 Feb 2024 15:42:43 -0500 Subject: [PATCH 19/23] THREESCALE-10722 Fix unit test logging --- pkg/3scale/amp/operator/apicast_reconciler_test.go | 2 +- pkg/3scale/amp/operator/backend_reconciler_test.go | 2 +- pkg/3scale/amp/operator/system_reconciler_test.go | 2 +- pkg/3scale/amp/operator/zync_reconciler_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/3scale/amp/operator/apicast_reconciler_test.go b/pkg/3scale/amp/operator/apicast_reconciler_test.go index e0779c0d0..986134fac 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler_test.go +++ b/pkg/3scale/amp/operator/apicast_reconciler_test.go @@ -877,7 +877,7 @@ func TestReplicaApicastReconciler(t *testing.T) { } if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, deployment.Spec.Replicas) + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, *deployment.Spec.Replicas) } }) } diff --git a/pkg/3scale/amp/operator/backend_reconciler_test.go b/pkg/3scale/amp/operator/backend_reconciler_test.go index 246fdb0d8..fe8722994 100644 --- a/pkg/3scale/amp/operator/backend_reconciler_test.go +++ b/pkg/3scale/amp/operator/backend_reconciler_test.go @@ -240,7 +240,7 @@ func TestReplicaBackendReconciler(t *testing.T) { } if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, deployment.Spec.Replicas) + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, *deployment.Spec.Replicas) } }) } diff --git a/pkg/3scale/amp/operator/system_reconciler_test.go b/pkg/3scale/amp/operator/system_reconciler_test.go index 68c3c2841..ca9597e2d 100644 --- a/pkg/3scale/amp/operator/system_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_reconciler_test.go @@ -263,7 +263,7 @@ func TestReplicaSystemReconciler(t *testing.T) { } if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, deployment.Spec.Replicas) + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, *deployment.Spec.Replicas) } }) } diff --git a/pkg/3scale/amp/operator/zync_reconciler_test.go b/pkg/3scale/amp/operator/zync_reconciler_test.go index 3169110de..57122f7cb 100644 --- a/pkg/3scale/amp/operator/zync_reconciler_test.go +++ b/pkg/3scale/amp/operator/zync_reconciler_test.go @@ -367,7 +367,7 @@ func TestReplicaZyncReconciler(t *testing.T) { } if tc.expectedAmountOfReplicas != *deployment.Spec.Replicas { - subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, deployment.Spec.Replicas) + subT.Errorf("expected replicas do not match. expected: %d actual: %d", tc.expectedAmountOfReplicas, *deployment.Spec.Replicas) } }) } From 7dd0318d1edbb8dc0016a9baedc149b9dfb4c05d Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Wed, 21 Feb 2024 17:07:19 -0500 Subject: [PATCH 20/23] THREESCALE-10392 Fix RBAC issue with zync-que Role --- pkg/3scale/amp/component/zync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/3scale/amp/component/zync.go b/pkg/3scale/amp/component/zync.go index 856be0880..3d1fcc589 100644 --- a/pkg/3scale/amp/component/zync.go +++ b/pkg/3scale/amp/component/zync.go @@ -110,7 +110,7 @@ func (zync *Zync) QueRole() *rbacv1.Role { { APIGroups: []string{"apps"}, Resources: []string{ - "deployment", + "deployments", }, Verbs: []string{ "get", From 96d5af3e7a6dca1ef0307423743bedbeb6fb4591 Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Mon, 26 Feb 2024 12:41:59 -0500 Subject: [PATCH 21/23] THREESCALE-10391 Refactored system_reconciler for readability/performance --- pkg/3scale/amp/operator/system_reconciler.go | 50 ++++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index d1f599c8d..17ec49297 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -144,7 +144,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { } // Used to synchronize rollout of system Deployments - systemComponentNotReady := false + systemComponentsReady := true // If the system-app Deployment generation has changed, delete the PreHook/PostHook Jobs so they can be recreated generationChanged, err := helper.HasAppGenerationChanged(component.SystemAppPreHookJobName, component.SystemAppDeploymentName, r.apiManager.GetNamespace(), r.Client()) @@ -183,25 +183,25 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { // Block reconciling system-app Deployment until PreHook Job has completed if !helper.HasJobCompleted(preHookJob.Name, preHookJob.Namespace, r.Client()) { - systemComponentNotReady = true - } - - // SystemApp Deployment - systemAppDeploymentMutators := []reconcilers.DMutateFn{ - reconcilers.DeploymentAffinityMutator, - reconcilers.DeploymentTolerationsMutator, - reconcilers.DeploymentPodTemplateLabelsMutator, - reconcilers.DeploymentPriorityClassMutator, - reconcilers.DeploymentTopologySpreadConstraintsMutator, - reconcilers.DeploymentPodTemplateAnnotationsMutator, - r.systemAppDeploymentResourceMutator, - reconcilers.DeploymentRemoveDuplicateEnvVarMutator, - reconcilers.DeploymentPodContainerImageMutator, - } - if r.apiManager.Spec.System.AppSpec.Replicas != nil { - systemAppDeploymentMutators = append(systemAppDeploymentMutators, reconcilers.DeploymentReplicasMutator) - } - if !systemComponentNotReady { + systemComponentsReady = false + } + + if systemComponentsReady { + // SystemApp Deployment + systemAppDeploymentMutators := []reconcilers.DMutateFn{ + reconcilers.DeploymentAffinityMutator, + reconcilers.DeploymentTolerationsMutator, + reconcilers.DeploymentPodTemplateLabelsMutator, + reconcilers.DeploymentPriorityClassMutator, + reconcilers.DeploymentTopologySpreadConstraintsMutator, + reconcilers.DeploymentPodTemplateAnnotationsMutator, + r.systemAppDeploymentResourceMutator, + reconcilers.DeploymentRemoveDuplicateEnvVarMutator, + reconcilers.DeploymentPodContainerImageMutator, + } + if r.apiManager.Spec.System.AppSpec.Replicas != nil { + systemAppDeploymentMutators = append(systemAppDeploymentMutators, reconcilers.DeploymentReplicasMutator) + } err = r.ReconcileDeployment(system.AppDeployment(ampImages.Options.SystemImage), reconcilers.DeploymentMutator(systemAppDeploymentMutators...)) if err != nil { return reconcile.Result{}, err @@ -218,11 +218,11 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } if k8serr.IsNotFound(err) || !helper.IsDeploymentAvailable(deployment) || helper.IsDeploymentProgressing(deployment) || !helper.HasJobCompleted(preHookJob.Name, preHookJob.Namespace, r.Client()) { - systemComponentNotReady = true + systemComponentsReady = false } // SystemApp PostHook Job - if !systemComponentNotReady { + if systemComponentsReady { err = r.ReconcileJob(system.AppPostHookJob(ampImages.Options.SystemImage, currentAppDeploymentGeneration), reconcilers.CreateOnlyMutator) if err != nil { return reconcile.Result{}, err @@ -235,7 +235,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } if !isMigrated { - systemComponentNotReady = true + systemComponentsReady = false } // Sidekiq Deployment @@ -268,7 +268,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } if !isMigrated { - systemComponentNotReady = true + systemComponentsReady = false } // SystemApp PDB @@ -314,7 +314,7 @@ func (r *SystemReconciler) Reconcile() (reconcile.Result, error) { } // Requeue if any of the system-app Deployment's components aren't ready - if systemComponentNotReady { + if !systemComponentsReady { return reconcile.Result{RequeueAfter: 5 * time.Second}, nil } From d43cc61f968d43da776fe27c7892988e55a5ad1f Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Tue, 27 Feb 2024 11:03:49 -0500 Subject: [PATCH 22/23] THREESCALE-10163 Tidied up imports --- controllers/apps/apimanager_controller.go | 4 +--- controllers/apps/apimanager_status_reconciler.go | 2 +- pkg/3scale/amp/component/system.go | 6 +++--- pkg/3scale/amp/component/system_searchd.go | 2 +- pkg/3scale/amp/component/zync.go | 7 +++---- pkg/3scale/amp/operator/apicast_options_provider_test.go | 2 +- pkg/3scale/amp/operator/apicast_reconciler_test.go | 5 ++--- pkg/3scale/amp/operator/backend_reconciler_test.go | 2 +- pkg/3scale/amp/operator/memcached_options_provider.go | 2 +- pkg/3scale/amp/operator/memcached_options_provider_test.go | 3 ++- pkg/3scale/amp/operator/memcached_reconciler.go | 1 + pkg/3scale/amp/operator/memcached_reconciler_test.go | 2 +- pkg/3scale/amp/operator/redis_reconciler.go | 2 +- pkg/3scale/amp/operator/redis_reconciler_test.go | 3 +-- pkg/3scale/amp/operator/system_mysql_reconciler.go | 1 + pkg/3scale/amp/operator/system_options_provider.go | 2 +- pkg/3scale/amp/operator/system_options_provider_test.go | 2 +- .../amp/operator/system_postgresql_options_provider.go | 3 ++- .../operator/system_postgresql_options_provider_test.go | 3 ++- pkg/3scale/amp/operator/system_reconciler.go | 4 ++-- pkg/3scale/amp/operator/system_reconciler_test.go | 2 +- pkg/3scale/amp/operator/system_searchd_options_provider.go | 2 +- .../amp/operator/system_searchd_options_provider_test.go | 2 +- pkg/3scale/amp/operator/system_searchd_reconciler.go | 2 +- pkg/3scale/amp/operator/system_searchd_reconciler_test.go | 2 +- pkg/3scale/amp/operator/zync_options_provider.go | 2 +- pkg/3scale/amp/operator/zync_options_provider_test.go | 2 +- pkg/3scale/amp/operator/zync_reconciler_test.go | 5 ++--- pkg/handlers/apimanager_managed_routes_event_mapper.go | 3 ++- .../apimanager_managed_routes_event_mapper_test.go | 7 +++---- pkg/reconcilers/role.go | 3 ++- pkg/reconcilers/role_test.go | 5 +++-- pkg/reconcilers/service.go | 1 + 33 files changed, 49 insertions(+), 47 deletions(-) diff --git a/controllers/apps/apimanager_controller.go b/controllers/apps/apimanager_controller.go index caeb00a14..f3e09bdc5 100644 --- a/controllers/apps/apimanager_controller.go +++ b/controllers/apps/apimanager_controller.go @@ -19,10 +19,9 @@ package controllers import ( "context" "fmt" - k8sappsv1 "k8s.io/api/apps/v1" routev1 "github.com/openshift/api/route/v1" - + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apimachinerymetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,7 +39,6 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/operator" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/handlers" - "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/version" diff --git a/controllers/apps/apimanager_status_reconciler.go b/controllers/apps/apimanager_status_reconciler.go index f01589d50..2e62af2c7 100644 --- a/controllers/apps/apimanager_status_reconciler.go +++ b/controllers/apps/apimanager_status_reconciler.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - k8sappsv1 "k8s.io/api/apps/v1" "sort" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -14,6 +13,7 @@ import ( "github.com/RHsyseng/operator-utils/pkg/olm" "github.com/go-logr/logr" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/3scale/amp/component/system.go b/pkg/3scale/amp/component/system.go index 6ad6a9d78..899497c91 100644 --- a/pkg/3scale/amp/component/system.go +++ b/pkg/3scale/amp/component/system.go @@ -1,12 +1,11 @@ package component import ( - "github.com/3scale/3scale-operator/pkg/reconcilers" - k8sappsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" "sort" "strconv" + k8sappsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,6 +13,7 @@ import ( "github.com/3scale/3scale-operator/apis/apps" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) const ( diff --git a/pkg/3scale/amp/component/system_searchd.go b/pkg/3scale/amp/component/system_searchd.go index 5d32e6d13..f5b48fa93 100644 --- a/pkg/3scale/amp/component/system_searchd.go +++ b/pkg/3scale/amp/component/system_searchd.go @@ -2,8 +2,8 @@ package component import ( "github.com/3scale/3scale-operator/pkg/reconcilers" - k8sappsv1 "k8s.io/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" diff --git a/pkg/3scale/amp/component/zync.go b/pkg/3scale/amp/component/zync.go index 3d1fcc589..30a5256be 100644 --- a/pkg/3scale/amp/component/zync.go +++ b/pkg/3scale/amp/component/zync.go @@ -1,13 +1,12 @@ package component import ( + "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" - k8sappsv1 "k8s.io/api/apps/v1" - - policyv1 "k8s.io/api/policy/v1" - "github.com/3scale/3scale-operator/pkg/helper" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" diff --git a/pkg/3scale/amp/operator/apicast_options_provider_test.go b/pkg/3scale/amp/operator/apicast_options_provider_test.go index 04829d160..caab1d1c4 100644 --- a/pkg/3scale/amp/operator/apicast_options_provider_test.go +++ b/pkg/3scale/amp/operator/apicast_options_provider_test.go @@ -1,7 +1,6 @@ package operator import ( - "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "strconv" "testing" @@ -19,6 +18,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) const ( diff --git a/pkg/3scale/amp/operator/apicast_reconciler_test.go b/pkg/3scale/amp/operator/apicast_reconciler_test.go index 986134fac..2fedc24e4 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler_test.go +++ b/pkg/3scale/amp/operator/apicast_reconciler_test.go @@ -5,9 +5,6 @@ import ( "fmt" "testing" - "github.com/3scale/3scale-operator/pkg/common" - "k8s.io/apimachinery/pkg/util/intstr" - grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" appsv1 "github.com/openshift/api/apps/v1" configv1 "github.com/openshift/api/config/v1" @@ -20,6 +17,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" fakeclientset "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" @@ -30,6 +28,7 @@ import ( "github.com/3scale/3scale-operator/apis/apps" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" + "github.com/3scale/3scale-operator/pkg/common" "github.com/3scale/3scale-operator/pkg/reconcilers" ) diff --git a/pkg/3scale/amp/operator/backend_reconciler_test.go b/pkg/3scale/amp/operator/backend_reconciler_test.go index fe8722994..eb4c416f4 100644 --- a/pkg/3scale/amp/operator/backend_reconciler_test.go +++ b/pkg/3scale/amp/operator/backend_reconciler_test.go @@ -2,7 +2,6 @@ package operator import ( "context" - k8sappsv1 "k8s.io/api/apps/v1" "testing" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -13,6 +12,7 @@ import ( configv1 "github.com/openshift/api/config/v1" imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/3scale/amp/operator/memcached_options_provider.go b/pkg/3scale/amp/operator/memcached_options_provider.go index ef8584832..948fd84ec 100644 --- a/pkg/3scale/amp/operator/memcached_options_provider.go +++ b/pkg/3scale/amp/operator/memcached_options_provider.go @@ -2,12 +2,12 @@ package operator import ( "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" ) diff --git a/pkg/3scale/amp/operator/memcached_options_provider_test.go b/pkg/3scale/amp/operator/memcached_options_provider_test.go index 9f44ed6ad..050606ab0 100644 --- a/pkg/3scale/amp/operator/memcached_options_provider_test.go +++ b/pkg/3scale/amp/operator/memcached_options_provider_test.go @@ -1,7 +1,6 @@ package operator import ( - "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -9,6 +8,8 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" diff --git a/pkg/3scale/amp/operator/memcached_reconciler.go b/pkg/3scale/amp/operator/memcached_reconciler.go index 59d3fc84e..c9d10422a 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler.go +++ b/pkg/3scale/amp/operator/memcached_reconciler.go @@ -5,6 +5,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/pkg/upgrade" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) diff --git a/pkg/3scale/amp/operator/memcached_reconciler_test.go b/pkg/3scale/amp/operator/memcached_reconciler_test.go index d0e2f3903..c1565e526 100644 --- a/pkg/3scale/amp/operator/memcached_reconciler_test.go +++ b/pkg/3scale/amp/operator/memcached_reconciler_test.go @@ -2,12 +2,12 @@ package operator import ( "context" - appsv1 "github.com/openshift/api/apps/v1" "testing" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/reconcilers" + appsv1 "github.com/openshift/api/apps/v1" k8sappsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/3scale/amp/operator/redis_reconciler.go b/pkg/3scale/amp/operator/redis_reconciler.go index ac69d2cfd..301b6affa 100644 --- a/pkg/3scale/amp/operator/redis_reconciler.go +++ b/pkg/3scale/amp/operator/redis_reconciler.go @@ -1,7 +1,6 @@ package operator import ( - "github.com/3scale/3scale-operator/pkg/upgrade" imagev1 "github.com/openshift/api/image/v1" k8sappsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -11,6 +10,7 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" ) // RedisReconciler is a generic DependencyReconciler that reconciles diff --git a/pkg/3scale/amp/operator/redis_reconciler_test.go b/pkg/3scale/amp/operator/redis_reconciler_test.go index 72b49faf2..e20b646aa 100644 --- a/pkg/3scale/amp/operator/redis_reconciler_test.go +++ b/pkg/3scale/amp/operator/redis_reconciler_test.go @@ -7,10 +7,9 @@ import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/reconcilers" - k8sappsv1 "k8s.io/api/apps/v1" - appsv1 "github.com/openshift/api/apps/v1" imagev1 "github.com/openshift/api/image/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/3scale/amp/operator/system_mysql_reconciler.go b/pkg/3scale/amp/operator/system_mysql_reconciler.go index 8c30188d4..87e8c0317 100644 --- a/pkg/3scale/amp/operator/system_mysql_reconciler.go +++ b/pkg/3scale/amp/operator/system_mysql_reconciler.go @@ -5,6 +5,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/pkg/upgrade" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) diff --git a/pkg/3scale/amp/operator/system_options_provider.go b/pkg/3scale/amp/operator/system_options_provider.go index db8674d90..d02bfd70e 100644 --- a/pkg/3scale/amp/operator/system_options_provider.go +++ b/pkg/3scale/amp/operator/system_options_provider.go @@ -2,7 +2,6 @@ package operator import ( "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" "path/filepath" v1 "k8s.io/api/core/v1" @@ -13,6 +12,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) type SystemOptionsProvider struct { diff --git a/pkg/3scale/amp/operator/system_options_provider_test.go b/pkg/3scale/amp/operator/system_options_provider_test.go index bc102f32e..c79a094c7 100644 --- a/pkg/3scale/amp/operator/system_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_options_provider_test.go @@ -1,7 +1,6 @@ package operator import ( - "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -10,6 +9,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" diff --git a/pkg/3scale/amp/operator/system_postgresql_options_provider.go b/pkg/3scale/amp/operator/system_postgresql_options_provider.go index e37e1ab05..764069657 100644 --- a/pkg/3scale/amp/operator/system_postgresql_options_provider.go +++ b/pkg/3scale/amp/operator/system_postgresql_options_provider.go @@ -2,7 +2,6 @@ package operator import ( "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" "net/url" "strings" @@ -10,6 +9,8 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" + v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go b/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go index bbede0a6d..72b21d508 100644 --- a/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_postgresql_options_provider_test.go @@ -2,7 +2,6 @@ package operator import ( "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "strings" "testing" @@ -11,6 +10,8 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" diff --git a/pkg/3scale/amp/operator/system_reconciler.go b/pkg/3scale/amp/operator/system_reconciler.go index 17ec49297..30a5793d4 100644 --- a/pkg/3scale/amp/operator/system_reconciler.go +++ b/pkg/3scale/amp/operator/system_reconciler.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" k8sappsv1 "k8s.io/api/apps/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -17,8 +19,6 @@ import ( "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/pkg/upgrade" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" ) type SystemReconciler struct { diff --git a/pkg/3scale/amp/operator/system_reconciler_test.go b/pkg/3scale/amp/operator/system_reconciler_test.go index ca9597e2d..cc30d857e 100644 --- a/pkg/3scale/amp/operator/system_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_reconciler_test.go @@ -2,7 +2,6 @@ package operator import ( "context" - k8sappsv1 "k8s.io/api/apps/v1" "testing" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -15,6 +14,7 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + k8sappsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" diff --git a/pkg/3scale/amp/operator/system_searchd_options_provider.go b/pkg/3scale/amp/operator/system_searchd_options_provider.go index e13db4fea..e5a11ef89 100644 --- a/pkg/3scale/amp/operator/system_searchd_options_provider.go +++ b/pkg/3scale/amp/operator/system_searchd_options_provider.go @@ -2,7 +2,6 @@ package operator import ( "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -11,6 +10,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) type SystemSearchdOptionsProvider struct { diff --git a/pkg/3scale/amp/operator/system_searchd_options_provider_test.go b/pkg/3scale/amp/operator/system_searchd_options_provider_test.go index 56ba91011..8a5af71ed 100644 --- a/pkg/3scale/amp/operator/system_searchd_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_searchd_options_provider_test.go @@ -2,7 +2,6 @@ package operator import ( "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -10,6 +9,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler.go b/pkg/3scale/amp/operator/system_searchd_reconciler.go index d19a42ffa..04b8837a3 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler.go @@ -1,12 +1,12 @@ package operator import ( - "github.com/3scale/3scale-operator/pkg/upgrade" "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/3scale/3scale-operator/pkg/upgrade" ) type SystemSearchdReconciler struct { diff --git a/pkg/3scale/amp/operator/system_searchd_reconciler_test.go b/pkg/3scale/amp/operator/system_searchd_reconciler_test.go index a9e97bf40..3e85db946 100644 --- a/pkg/3scale/amp/operator/system_searchd_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_searchd_reconciler_test.go @@ -2,7 +2,6 @@ package operator import ( "context" - k8sappsv1 "k8s.io/api/apps/v1" "testing" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" @@ -10,6 +9,7 @@ import ( "github.com/3scale/3scale-operator/pkg/reconcilers" appsv1 "github.com/openshift/api/apps/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/3scale/amp/operator/zync_options_provider.go b/pkg/3scale/amp/operator/zync_options_provider.go index edc0d33b3..3e9395a49 100644 --- a/pkg/3scale/amp/operator/zync_options_provider.go +++ b/pkg/3scale/amp/operator/zync_options_provider.go @@ -2,13 +2,13 @@ package operator import ( "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" "net/url" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/3scale/amp/operator/zync_options_provider_test.go b/pkg/3scale/amp/operator/zync_options_provider_test.go index 57607f7b0..d1a375476 100644 --- a/pkg/3scale/amp/operator/zync_options_provider_test.go +++ b/pkg/3scale/amp/operator/zync_options_provider_test.go @@ -2,7 +2,6 @@ package operator import ( "fmt" - "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" @@ -17,6 +16,7 @@ import ( "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/3scale/amp/product" "github.com/3scale/3scale-operator/pkg/helper" + "github.com/3scale/3scale-operator/pkg/reconcilers" ) const ( diff --git a/pkg/3scale/amp/operator/zync_reconciler_test.go b/pkg/3scale/amp/operator/zync_reconciler_test.go index 57122f7cb..1e374bc04 100644 --- a/pkg/3scale/amp/operator/zync_reconciler_test.go +++ b/pkg/3scale/amp/operator/zync_reconciler_test.go @@ -2,11 +2,8 @@ package operator import ( "context" - k8sappsv1 "k8s.io/api/apps/v1" "testing" - policyv1 "k8s.io/api/policy/v1" - appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "github.com/3scale/3scale-operator/pkg/reconcilers" @@ -17,7 +14,9 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/handlers/apimanager_managed_routes_event_mapper.go b/pkg/handlers/apimanager_managed_routes_event_mapper.go index c903118d9..36c006aa9 100644 --- a/pkg/handlers/apimanager_managed_routes_event_mapper.go +++ b/pkg/handlers/apimanager_managed_routes_event_mapper.go @@ -2,11 +2,12 @@ package handlers import ( "context" - "github.com/3scale/3scale-operator/pkg/reconcilers" appscommon "github.com/3scale/3scale-operator/apis/apps" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" + "github.com/3scale/3scale-operator/pkg/reconcilers" + "github.com/go-logr/logr" k8sappsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/handlers/apimanager_managed_routes_event_mapper_test.go b/pkg/handlers/apimanager_managed_routes_event_mapper_test.go index 97b834f0f..1fcb35b44 100644 --- a/pkg/handlers/apimanager_managed_routes_event_mapper_test.go +++ b/pkg/handlers/apimanager_managed_routes_event_mapper_test.go @@ -1,17 +1,15 @@ package handlers import ( - "github.com/3scale/3scale-operator/pkg/reconcilers" "reflect" "testing" "github.com/go-logr/logr" + routev1 "github.com/openshift/api/route/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" - - routev1 "github.com/openshift/api/route/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -19,6 +17,7 @@ import ( appscommon "github.com/3scale/3scale-operator/apis/apps" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" + "github.com/3scale/3scale-operator/pkg/reconcilers" k8sappsv1 "k8s.io/api/apps/v1" ) diff --git a/pkg/reconcilers/role.go b/pkg/reconcilers/role.go index 6982d4cdb..d4ed5bf0c 100644 --- a/pkg/reconcilers/role.go +++ b/pkg/reconcilers/role.go @@ -2,9 +2,10 @@ package reconcilers import ( "fmt" - "github.com/3scale/3scale-operator/pkg/common" "reflect" + "github.com/3scale/3scale-operator/pkg/common" + rbacv1 "k8s.io/api/rbac/v1" ) diff --git a/pkg/reconcilers/role_test.go b/pkg/reconcilers/role_test.go index 483c02456..ea3b22fa3 100644 --- a/pkg/reconcilers/role_test.go +++ b/pkg/reconcilers/role_test.go @@ -1,11 +1,12 @@ package reconcilers import ( + "reflect" + "testing" + "github.com/google/go-cmp/cmp" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "reflect" - "testing" ) func TestRoleRuleMutator(t *testing.T) { diff --git a/pkg/reconcilers/service.go b/pkg/reconcilers/service.go index f3bbcc320..68a33e361 100644 --- a/pkg/reconcilers/service.go +++ b/pkg/reconcilers/service.go @@ -5,6 +5,7 @@ import ( "reflect" "github.com/3scale/3scale-operator/pkg/common" + v1 "k8s.io/api/core/v1" ) From 86f4b3ac87f032226f8f21a38722e372f97cd9ba Mon Sep 17 00:00:00 2001 From: Carl Kyrillos Date: Tue, 27 Feb 2024 17:18:20 -0500 Subject: [PATCH 23/23] THREESCALE-10163 Removed ImageStreams --- apis/apps/v1alpha1/apimanager_types.go | 10 - apis/apps/v1alpha1/apimanager_types_test.go | 10 +- apis/apps/v1alpha1/zz_generated.deepcopy.go | 5 - ...3scale-operator.clusterserviceversion.yaml | 3 - .../apps.3scale.net_apimanagers.yaml | 2 - .../bases/apps.3scale.net_apimanagers.yaml | 2 - ...3scale-operator.clusterserviceversion.yaml | 3 - controllers/apps/apimanager_controller.go | 6 +- ...oxyconfigpromote_status_reconciler_test.go | 9 +- doc/apimanager-reference.md | 1 - pkg/3scale/amp/component/ampimages.go | 236 ------------------ pkg/3scale/amp/component/ampimages_options.go | 19 +- pkg/3scale/amp/component/redis.go | 61 ----- pkg/3scale/amp/component/redis_options.go | 1 - .../amp/component/system_mysql_image.go | 39 --- .../component/system_mysql_image_options.go | 7 +- .../amp/component/system_postgresql_image.go | 39 --- .../system_postgresql_image_options.go | 7 +- .../operator/ampimages_options_provider.go | 1 - .../ampimages_options_provider_test.go | 1 - .../amp/operator/ampimages_reconciler.go | 44 ---- .../amp/operator/ampimages_reconciler_test.go | 114 +-------- .../amp/operator/apicast_reconciler_test.go | 72 +++--- .../amp/operator/backend_reconciler_test.go | 18 +- .../base_apimanager_logic_reconciler.go | 5 - pkg/3scale/amp/operator/helper_test.go | 23 +- .../amp/operator/redis_options_provider.go | 1 - .../operator/redis_options_provider_test.go | 2 - pkg/3scale/amp/operator/redis_reconciler.go | 10 - .../amp/operator/redis_reconciler_test.go | 11 +- .../system_mysql_image_options_provider.go | 1 - ...ystem_mysql_image_options_provider_test.go | 8 +- .../operator/system_mysql_image_reconciler.go | 22 -- .../system_mysql_image_reconciler_test.go | 88 ------- .../operator/system_mysql_reconciler_test.go | 9 +- ...ystem_postgresql_image_options_provider.go | 1 - ..._postgresql_image_options_provider_test.go | 8 +- .../system_postgresql_image_reconciler.go | 23 -- ...system_postgresql_image_reconciler_test.go | 88 ------- .../system_postgresql_reconciler_test.go | 9 +- .../amp/operator/system_reconciler_test.go | 9 +- .../amp/operator/zync_reconciler_test.go | 27 +- pkg/reconcilers/imagestreams.go | 66 ----- pkg/reconcilers/imagestreams_test.go | 103 -------- pkg/upgrade/delete_image_streams_2.15.go | 61 +++++ 45 files changed, 174 insertions(+), 1111 deletions(-) delete mode 100644 pkg/3scale/amp/operator/system_mysql_image_reconciler_test.go delete mode 100644 pkg/3scale/amp/operator/system_postgresql_image_reconciler_test.go delete mode 100644 pkg/reconcilers/imagestreams.go delete mode 100644 pkg/reconcilers/imagestreams_test.go create mode 100644 pkg/upgrade/delete_image_streams_2.15.go diff --git a/apis/apps/v1alpha1/apimanager_types.go b/apis/apps/v1alpha1/apimanager_types.go index ad646d720..6a88b1c1d 100644 --- a/apis/apps/v1alpha1/apimanager_types.go +++ b/apis/apps/v1alpha1/apimanager_types.go @@ -45,7 +45,6 @@ const ( const ( defaultTenantName = "3scale" - defaultImageStreamImportInsecure = false defaultResourceRequirementsEnabled = true ) @@ -134,7 +133,6 @@ func (s *APIManagerStatus) Equals(other *APIManagerStatus, logger logr.Logger) b // +operator-sdk:csv:customresourcedefinitions:resources={{"PersistentVolumeClaim","v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"Service","v1"}} // +operator-sdk:csv:customresourcedefinitions:resources={{"Route","route.openshift.io/v1"}} -// +operator-sdk:csv:customresourcedefinitions:resources={{"ImageStream","image.openshift.io/v1"}} type APIManager struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -161,8 +159,6 @@ type APIManagerCommonSpec struct { // +optional TenantName *string `json:"tenantName,omitempty"` // +optional - ImageStreamTagImportInsecure *bool `json:"imageStreamTagImportInsecure,omitempty"` - // +optional ResourceRequirementsEnabled *bool `json:"resourceRequirementsEnabled,omitempty"` // +optional ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"` @@ -977,7 +973,6 @@ func (apimanager *APIManager) setAPIManagerCommonSpecDefaults() bool { tmpDefaultAppLabel := Default3scaleAppLabel tmpDefaultTenantName := defaultTenantName - tmpDefaultImageStreamTagImportInsecure := defaultImageStreamImportInsecure tmpDefaultResourceRequirementsEnabled := defaultResourceRequirementsEnabled if spec.AppLabel == nil { @@ -990,11 +985,6 @@ func (apimanager *APIManager) setAPIManagerCommonSpecDefaults() bool { changed = true } - if spec.ImageStreamTagImportInsecure == nil { - spec.ImageStreamTagImportInsecure = &tmpDefaultImageStreamTagImportInsecure - changed = true - } - if spec.ResourceRequirementsEnabled == nil { spec.ResourceRequirementsEnabled = &tmpDefaultResourceRequirementsEnabled changed = true diff --git a/apis/apps/v1alpha1/apimanager_types_test.go b/apis/apps/v1alpha1/apimanager_types_test.go index 0ded8b340..792ac1bdb 100644 --- a/apis/apps/v1alpha1/apimanager_types_test.go +++ b/apis/apps/v1alpha1/apimanager_types_test.go @@ -14,7 +14,6 @@ import ( func TestSetDefaults(t *testing.T) { tmpDefaultAppLabel := Default3scaleAppLabel tmpDefaultTenantName := defaultTenantName - tmpDefaultImageStreamTagImportInsecure := defaultImageStreamImportInsecure tmpDefaultResourceRequirementsEnabled := defaultResourceRequirementsEnabled tmpDefaultApicastManagementAPI := defaultApicastManagementAPI tmpDefaultApicastOpenSSLVerify := defaultApicastOpenSSLVerify @@ -32,11 +31,10 @@ func TestSetDefaults(t *testing.T) { }, Spec: APIManagerSpec{ APIManagerCommonSpec: APIManagerCommonSpec{ - WildcardDomain: "test.3scale.com", - AppLabel: &tmpDefaultAppLabel, - TenantName: &tmpDefaultTenantName, - ImageStreamTagImportInsecure: &tmpDefaultImageStreamTagImportInsecure, - ResourceRequirementsEnabled: &tmpDefaultResourceRequirementsEnabled, + WildcardDomain: "test.3scale.com", + AppLabel: &tmpDefaultAppLabel, + TenantName: &tmpDefaultTenantName, + ResourceRequirementsEnabled: &tmpDefaultResourceRequirementsEnabled, }, Apicast: &ApicastSpec{ IncludeResponseCodes: &tmpDefaultApicastResponseCodes, diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 1287f3f84..d34eacd46 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -205,11 +205,6 @@ func (in *APIManagerCommonSpec) DeepCopyInto(out *APIManagerCommonSpec) { *out = new(string) **out = **in } - if in.ImageStreamTagImportInsecure != nil { - in, out := &in.ImageStreamTagImportInsecure, &out.ImageStreamTagImportInsecure - *out = new(bool) - **out = **in - } if in.ResourceRequirementsEnabled != nil { in, out := &in.ResourceRequirementsEnabled, &out.ResourceRequirementsEnabled *out = new(bool) diff --git a/bundle/manifests/3scale-operator.clusterserviceversion.yaml b/bundle/manifests/3scale-operator.clusterserviceversion.yaml index cae638510..e7c858243 100644 --- a/bundle/manifests/3scale-operator.clusterserviceversion.yaml +++ b/bundle/manifests/3scale-operator.clusterserviceversion.yaml @@ -245,9 +245,6 @@ spec: - kind: Deployment name: "" version: apps/v1 - - kind: ImageStream - name: "" - version: image.openshift.io/v1 - kind: PersistentVolumeClaim name: "" version: v1 diff --git a/bundle/manifests/apps.3scale.net_apimanagers.yaml b/bundle/manifests/apps.3scale.net_apimanagers.yaml index 77a3295af..83b4b7267 100644 --- a/bundle/manifests/apps.3scale.net_apimanagers.yaml +++ b/bundle/manifests/apps.3scale.net_apimanagers.yaml @@ -3860,8 +3860,6 @@ spec: type: object x-kubernetes-map-type: atomic type: array - imageStreamTagImportInsecure: - type: boolean monitoring: properties: enablePrometheusRules: diff --git a/config/crd/bases/apps.3scale.net_apimanagers.yaml b/config/crd/bases/apps.3scale.net_apimanagers.yaml index 1e8317584..db8a72852 100644 --- a/config/crd/bases/apps.3scale.net_apimanagers.yaml +++ b/config/crd/bases/apps.3scale.net_apimanagers.yaml @@ -7190,8 +7190,6 @@ spec: type: object x-kubernetes-map-type: atomic type: array - imageStreamTagImportInsecure: - type: boolean monitoring: properties: enablePrometheusRules: diff --git a/config/manifests/bases/3scale-operator.clusterserviceversion.yaml b/config/manifests/bases/3scale-operator.clusterserviceversion.yaml index f6bb04200..7d787f470 100644 --- a/config/manifests/bases/3scale-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/3scale-operator.clusterserviceversion.yaml @@ -59,9 +59,6 @@ spec: - kind: Deployment name: "" version: apps/v1 - - kind: ImageStream - name: "" - version: image.openshift.io/v1 - kind: PersistentVolumeClaim name: "" version: v1 diff --git a/controllers/apps/apimanager_controller.go b/controllers/apps/apimanager_controller.go index f3e09bdc5..175ac98d7 100644 --- a/controllers/apps/apimanager_controller.go +++ b/controllers/apps/apimanager_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "github.com/3scale/3scale-operator/pkg/upgrade" routev1 "github.com/openshift/api/route/v1" k8sappsv1 "k8s.io/api/apps/v1" @@ -260,6 +261,9 @@ func (r *APIManagerReconciler) reconcileAPIManagerLogic(cr *appsv1alpha1.APIMana return result, err } + // 3scale 2.14 -> 2.15 + err = upgrade.DeleteImageStreams(r.WatchedNamespace, r.Client()) + return ctrl.Result{}, nil } @@ -335,12 +339,10 @@ func (r *APIManagerReconciler) dependencyReconcilerForComponents(cr *appsv1alpha if cr.Spec.System.DatabaseSpec != nil && cr.Spec.System.DatabaseSpec.PostgreSQL != nil { systemDatabaseReconcilerConstructor = operator.CompositeDependencyReconcilerConstructor( operator.NewSystemPostgreSQLReconciler, - operator.NewSystemPostgreSQLImageReconciler, ) } else { systemDatabaseReconcilerConstructor = operator.CompositeDependencyReconcilerConstructor( operator.NewSystemMySQLReconciler, - operator.NewSystemMySQLImageReconciler, ) } diff --git a/controllers/capabilities/proxyconfigpromote_status_reconciler_test.go b/controllers/capabilities/proxyconfigpromote_status_reconciler_test.go index 115404dfc..2bfe99a2a 100644 --- a/controllers/capabilities/proxyconfigpromote_status_reconciler_test.go +++ b/controllers/capabilities/proxyconfigpromote_status_reconciler_test.go @@ -39,11 +39,10 @@ func getApiManger() (apimanager *appsv1alpha1.APIManager) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Backend: &appsv1alpha1.BackendSpec{ ListenerSpec: &appsv1alpha1.BackendListenerSpec{Replicas: &oneValue}, diff --git a/doc/apimanager-reference.md b/doc/apimanager-reference.md index 3bea95d9b..ebe021381 100644 --- a/doc/apimanager-reference.md +++ b/doc/apimanager-reference.md @@ -90,7 +90,6 @@ One APIManager custom resource per project is allowed. | WildcardDomain | `wildcardDomain` | string | Yes | N/A | Root domain for the wildcard routes. Eg. example.com will generate 3scale-admin.example.com. | | AppLabel | `appLabel` | string | No | `3scale-api-management` | The value of the `app` label that will be applied to the API management solution | TenantName | `tenantName` | string | No | `3scale` | Tenant name under the root that Admin UI will be available with -admin suffix. -| ImageStreamTagImportInsecure | `imageStreamTagImportInsecure` | bool | No | `false` | Set to true if the server may bypass certificate verification or connect directly over HTTP during image import | | ImagePullSecrets | `imagePullSecrets` | \[\][corev1.LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#localobjectreference-v1-core) | No | `[ { name: "threescale-registry-auth" } ]` | List of image pull secrets to be used on the managed Deployments ServiceAccounts. See [imagePullSecrets field in K8s ServiceAccount documentation](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#serviceaccount-v1-core) for details on Image pull secrets. If not specified, `threescale-registry-auth` is used. Secret names that contain `dockercfg-` or `token-` anywhere in part of its name cannot be specified. If an update to this attribute is performed the corresponding Deployment pods have to be redeployed by the user to make the changes effective | | ResourceRequirementsEnabled | `resourceRequirementsEnabled` | bool | No | `true` | When true, 3Scale API management solution is deployed with the optimal resource requirements and limits. Setting this to false removes those resource requirements. ***Warning*** Only set it to false for development and evaluation environments. When set to `true`, default compute resources are set for the APIManager components. See [Default APIManager components compute resources](#Default-APIManager-components-compute-resources) to see the default assigned values | | ApicastSpec | `apicast` | \*ApicastSpec | No | See [ApicastSpec](#ApicastSpec) | Spec of the Apicast part | diff --git a/pkg/3scale/amp/component/ampimages.go b/pkg/3scale/amp/component/ampimages.go index 24fe2d3a5..8f84bd49a 100644 --- a/pkg/3scale/amp/component/ampimages.go +++ b/pkg/3scale/amp/component/ampimages.go @@ -1,15 +1,10 @@ package component import ( - imagev1 "github.com/openshift/api/image/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const ( - InsecureImportPolicy = false -) - type AmpImages struct { Options *AmpImagesOptions } @@ -18,237 +13,6 @@ func NewAmpImages(options *AmpImagesOptions) *AmpImages { return &AmpImages{Options: options} } -func (ampImages *AmpImages) BackendImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "amp-backend", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "backend", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP backend", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "amp-backend " + ampImages.Options.AmpRelease, - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.BackendImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) ZyncImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "amp-zync", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "zync", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP Zync", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP Zync " + ampImages.Options.AmpRelease, - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.ZyncImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) APICastImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "amp-apicast", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "apicast", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP APIcast", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP APIcast " + ampImages.Options.AmpRelease, - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.ApicastImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) SystemImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "amp-system", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP System", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "AMP system " + ampImages.Options.AmpRelease, - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.SystemImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) ZyncDatabasePostgreSQLImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "zync-database-postgresql", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "Zync database PostgreSQL", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "Zync " + ampImages.Options.AmpRelease + " PostgreSQL", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.ZyncDatabasePostgreSQLImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) SystemMemcachedImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-memcached", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "System Memcached", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + ampImages.Options.AmpRelease + " Memcached", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.SystemMemcachedImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - -func (ampImages *AmpImages) SystemSearchdImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-searchd", - Labels: map[string]string{ - "app": ampImages.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "System Searchd", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: ampImages.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + ampImages.Options.AmpRelease + " Searchd", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: ampImages.Options.SystemSearchdImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: ampImages.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - func (ampImages *AmpImages) DeploymentsServiceAccount() *v1.ServiceAccount { return &v1.ServiceAccount{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/3scale/amp/component/ampimages_options.go b/pkg/3scale/amp/component/ampimages_options.go index 8aba7a2a8..0d98921fe 100644 --- a/pkg/3scale/amp/component/ampimages_options.go +++ b/pkg/3scale/amp/component/ampimages_options.go @@ -7,16 +7,15 @@ import ( // AmpImagesOptions container object with all required to create components type AmpImagesOptions struct { - AppLabel string `validate:"required"` - AmpRelease string `validate:"required"` - ApicastImage string `validate:"required"` - BackendImage string `validate:"required"` - SystemImage string `validate:"required"` - ZyncImage string `validate:"required"` - ZyncDatabasePostgreSQLImage string `validate:"required"` - SystemMemcachedImage string `validate:"required"` - SystemSearchdImage string `validate:"required"` - InsecureImportPolicy bool + AppLabel string `validate:"required"` + AmpRelease string `validate:"required"` + ApicastImage string `validate:"required"` + BackendImage string `validate:"required"` + SystemImage string `validate:"required"` + ZyncImage string `validate:"required"` + ZyncDatabasePostgreSQLImage string `validate:"required"` + SystemMemcachedImage string `validate:"required"` + SystemSearchdImage string `validate:"required"` ImagePullSecrets []v1.LocalObjectReference `validate:"required"` } diff --git a/pkg/3scale/amp/component/redis.go b/pkg/3scale/amp/component/redis.go index 55afc57b0..a170057e8 100644 --- a/pkg/3scale/amp/component/redis.go +++ b/pkg/3scale/amp/component/redis.go @@ -6,7 +6,6 @@ import ( "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" - imagev1 "github.com/openshift/api/image/v1" k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -359,36 +358,6 @@ func (redis *Redis) buildPVCSpec() v1.PersistentVolumeClaimSpec { } } -func (redis *Redis) BackendImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "backend-redis", - Labels: redis.Options.BackendCommonLabels, - Annotations: map[string]string{ - "openshift.io/display-name": "Backend Redis", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: redis.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "Backend " + redis.Options.AmpRelease + " Redis", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: redis.Options.BackendImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: *redis.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - func (redis *Redis) BackendRedisSecret() *v1.Secret { return &v1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -571,36 +540,6 @@ func (redis *Redis) SystemPVC() *v1.PersistentVolumeClaim { } } -func (redis *Redis) SystemImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-redis", - Labels: redis.Options.SystemCommonLabels, - Annotations: map[string]string{ - "openshift.io/display-name": "System Redis", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: redis.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + redis.Options.AmpRelease + " Redis", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: redis.Options.SystemImage, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: *redis.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} - func (redis *Redis) SystemRedisSecret() *v1.Secret { return &v1.Secret{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/3scale/amp/component/redis_options.go b/pkg/3scale/amp/component/redis_options.go index 0c510c2d3..962a56fb0 100644 --- a/pkg/3scale/amp/component/redis_options.go +++ b/pkg/3scale/amp/component/redis_options.go @@ -14,7 +14,6 @@ type RedisOptions struct { SystemImageTag string `validate:"required"` BackendRedisContainerResourceRequirements *v1.ResourceRequirements `validate:"required"` SystemRedisContainerResourceRequirements *v1.ResourceRequirements `validate:"required"` - InsecureImportPolicy *bool `validate:"required"` BackendRedisPVCStorageClass *string SystemRedisPVCStorageClass *string diff --git a/pkg/3scale/amp/component/system_mysql_image.go b/pkg/3scale/amp/component/system_mysql_image.go index a0875620e..2f85ef6f0 100644 --- a/pkg/3scale/amp/component/system_mysql_image.go +++ b/pkg/3scale/amp/component/system_mysql_image.go @@ -1,11 +1,5 @@ package component -import ( - imagev1 "github.com/openshift/api/image/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - type SystemMySQLImage struct { Options *SystemMySQLImageOptions } @@ -13,36 +7,3 @@ type SystemMySQLImage struct { func NewSystemMySQLImage(options *SystemMySQLImageOptions) *SystemMySQLImage { return &SystemMySQLImage{Options: options} } - -func (s *SystemMySQLImage) ImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-mysql", - Labels: map[string]string{ - "app": s.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "System MySQL", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: s.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + s.Options.AmpRelease + " MySQL", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: s.Options.Image, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: *s.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} diff --git a/pkg/3scale/amp/component/system_mysql_image_options.go b/pkg/3scale/amp/component/system_mysql_image_options.go index dbee8ed63..094d85262 100644 --- a/pkg/3scale/amp/component/system_mysql_image_options.go +++ b/pkg/3scale/amp/component/system_mysql_image_options.go @@ -3,10 +3,9 @@ package component import "github.com/go-playground/validator/v10" type SystemMySQLImageOptions struct { - AppLabel string `validate:"required"` - AmpRelease string `validate:"required"` - Image string `validate:"required"` - InsecureImportPolicy *bool `validate:"required"` + AppLabel string `validate:"required"` + AmpRelease string `validate:"required"` + Image string `validate:"required"` } func NewSystemMySQLImageOptions() *SystemMySQLImageOptions { diff --git a/pkg/3scale/amp/component/system_postgresql_image.go b/pkg/3scale/amp/component/system_postgresql_image.go index 7e16ece4d..652657eb3 100644 --- a/pkg/3scale/amp/component/system_postgresql_image.go +++ b/pkg/3scale/amp/component/system_postgresql_image.go @@ -1,11 +1,5 @@ package component -import ( - imagev1 "github.com/openshift/api/image/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - type SystemPostgreSQLImage struct { Options *SystemPostgreSQLImageOptions } @@ -13,36 +7,3 @@ type SystemPostgreSQLImage struct { func NewSystemPostgreSQLImage(options *SystemPostgreSQLImageOptions) *SystemPostgreSQLImage { return &SystemPostgreSQLImage{Options: options} } - -func (s *SystemPostgreSQLImage) ImageStream() *imagev1.ImageStream { - return &imagev1.ImageStream{ - ObjectMeta: metav1.ObjectMeta{ - Name: "system-postgresql", - Labels: map[string]string{ - "app": s.Options.AppLabel, - "threescale_component": "system", - }, - Annotations: map[string]string{ - "openshift.io/display-name": "System database", - }, - }, - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: s.Options.AmpRelease, - Annotations: map[string]string{ - "openshift.io/display-name": "System " + s.Options.AmpRelease + " PostgreSQL", - }, - From: &v1.ObjectReference{ - Kind: "DockerImage", - Name: s.Options.Image, - }, - ImportPolicy: imagev1.TagImportPolicy{ - Insecure: *s.Options.InsecureImportPolicy, - }, - }, - }, - }, - } -} diff --git a/pkg/3scale/amp/component/system_postgresql_image_options.go b/pkg/3scale/amp/component/system_postgresql_image_options.go index 7384f97a4..53ca65196 100644 --- a/pkg/3scale/amp/component/system_postgresql_image_options.go +++ b/pkg/3scale/amp/component/system_postgresql_image_options.go @@ -3,10 +3,9 @@ package component import "github.com/go-playground/validator/v10" type SystemPostgreSQLImageOptions struct { - AppLabel string `validate:"required"` - AmpRelease string `validate:"required"` - Image string `validate:"required"` - InsecureImportPolicy *bool `validate:"required"` + AppLabel string `validate:"required"` + AmpRelease string `validate:"required"` + Image string `validate:"required"` } func NewSystemPostgreSQLImageOptions() *SystemPostgreSQLImageOptions { diff --git a/pkg/3scale/amp/operator/ampimages_options_provider.go b/pkg/3scale/amp/operator/ampimages_options_provider.go index 5cb31ed83..bd753dc9b 100644 --- a/pkg/3scale/amp/operator/ampimages_options_provider.go +++ b/pkg/3scale/amp/operator/ampimages_options_provider.go @@ -21,7 +21,6 @@ func NewAmpImagesOptionsProvider(apimanager *appsv1alpha1.APIManager) *AmpImages func (a *AmpImagesOptionsProvider) GetAmpImagesOptions() (*component.AmpImagesOptions, error) { a.ampImagesOptions.AppLabel = *a.apimanager.Spec.AppLabel a.ampImagesOptions.AmpRelease = product.ThreescaleRelease - a.ampImagesOptions.InsecureImportPolicy = *a.apimanager.Spec.ImageStreamTagImportInsecure a.ampImagesOptions.ApicastImage = ApicastImageURL() if a.apimanager.Spec.Apicast != nil && a.apimanager.Spec.Apicast.Image != nil { diff --git a/pkg/3scale/amp/operator/ampimages_options_provider_test.go b/pkg/3scale/amp/operator/ampimages_options_provider_test.go index c088777d7..b6166af5c 100644 --- a/pkg/3scale/amp/operator/ampimages_options_provider_test.go +++ b/pkg/3scale/amp/operator/ampimages_options_provider_test.go @@ -34,7 +34,6 @@ func defaultAmpImageOptions() *component.AmpImagesOptions { ZyncDatabasePostgreSQLImage: component.ZyncPostgreSQLImageURL(), SystemMemcachedImage: SystemMemcachedImageURL(), SystemSearchdImage: SystemSearchdImageURL(), - InsecureImportPolicy: insecureImportPolicy, ImagePullSecrets: component.AmpImagesDefaultImagePullSecrets(), } } diff --git a/pkg/3scale/amp/operator/ampimages_reconciler.go b/pkg/3scale/amp/operator/ampimages_reconciler.go index 7fb33be03..215f78246 100644 --- a/pkg/3scale/amp/operator/ampimages_reconciler.go +++ b/pkg/3scale/amp/operator/ampimages_reconciler.go @@ -24,50 +24,6 @@ func (r *AMPImagesReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // backend IS - err = r.ReconcileImagestream(ampImages.BackendImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // zync IS - err = r.ReconcileImagestream(ampImages.ZyncImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // apicast IS - err = r.ReconcileImagestream(ampImages.APICastImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // system IS - err = r.ReconcileImagestream(ampImages.SystemImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - if !r.apiManager.IsExternal(appsv1alpha1.ZyncDatabase) { - // zync db postresql IS - err = r.ReconcileImagestream(ampImages.ZyncDatabasePostgreSQLImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - } - - // system memcached IS - err = r.ReconcileImagestream(ampImages.SystemMemcachedImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - // system searchd IS - err = r.ReconcileImagestream(ampImages.SystemSearchdImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - err = r.ReconcileServiceAccount(ampImages.DeploymentsServiceAccount(), reconcilers.ServiceAccountImagePullPolicyMutator) if err != nil { return reconcile.Result{}, err diff --git a/pkg/3scale/amp/operator/ampimages_reconciler_test.go b/pkg/3scale/amp/operator/ampimages_reconciler_test.go index 560cb7d56..9bc91b60e 100644 --- a/pkg/3scale/amp/operator/ampimages_reconciler_test.go +++ b/pkg/3scale/amp/operator/ampimages_reconciler_test.go @@ -11,7 +11,6 @@ import ( imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -30,7 +29,6 @@ func TestAMPImagesReconciler(t *testing.T) { wildcardDomain = "test.3scale.net" log = logf.Log.WithName("operator_test") appLabel = "someLabel" - trueValue = true ) ctx := context.TODO() @@ -42,9 +40,8 @@ func TestAMPImagesReconciler(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, }, }, } @@ -85,13 +82,6 @@ func TestAMPImagesReconciler(t *testing.T) { objName string obj client.Object }{ - {"backendCreated", "amp-backend", &imagev1.ImageStream{}}, - {"zyncCreated", "amp-zync", &imagev1.ImageStream{}}, - {"apicastCreated", "amp-apicast", &imagev1.ImageStream{}}, - {"systemCreated", "amp-system", &imagev1.ImageStream{}}, - {"zyncPostgresqlCreated", "zync-database-postgresql", &imagev1.ImageStream{}}, - {"systemMemcachedCreated", "system-memcached", &imagev1.ImageStream{}}, - // TODO: service account created by AMPImagesReconciler. Should not be there. {"serviceAccountCreated", "amp", &v1.ServiceAccount{}}, } @@ -110,103 +100,3 @@ func TestAMPImagesReconciler(t *testing.T) { }) } } - -func TestAMPImagesReconcilerWithAllExternalDatabases(t *testing.T) { - var ( - name = "example-apimanager" - namespace = "operator-unittest" - wildcardDomain = "test.3scale.net" - log = logf.Log.WithName("operator_test") - appLabel = "someLabel" - trueValue = true - ) - - ctx := context.TODO() - - apimanager := &appsv1alpha1.APIManager{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: appsv1alpha1.APIManagerSpec{ - APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - }, - HighAvailability: &appsv1alpha1.HighAvailabilitySpec{ - Enabled: true, - ExternalZyncDatabaseEnabled: &trueValue, - }, - ExternalComponents: appsv1alpha1.AllComponentsExternal(), - }, - } - // Objects to track in the fake client. - objs := []runtime.Object{apimanager} - s := scheme.Scheme - s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := appsv1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - err = imagev1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - err = routev1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - - // Create a fake client to mock API calls. - cl := fake.NewFakeClient(objs...) - clientAPIReader := fake.NewFakeClient(objs...) - recorder := record.NewFakeRecorder(10000) - clientset := fakeclientset.NewSimpleClientset() - - baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) - baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) - - imagesReconciler := NewAMPImagesReconciler(baseAPIManagerLogicReconciler) - _, err = imagesReconciler.Reconcile() - if err != nil { - t.Fatal(err) - } - - cases := []struct { - testName string - objName string - obj client.Object - hasToExist bool - }{ - {"backendCreated", "amp-backend", &imagev1.ImageStream{}, true}, - {"zyncCreated", "amp-zync", &imagev1.ImageStream{}, true}, - {"apicastCreated", "amp-apicast", &imagev1.ImageStream{}, true}, - {"systemCreated", "amp-system", &imagev1.ImageStream{}, true}, - {"zyncPostgresqlCreated", "zync-database-postgresql", &imagev1.ImageStream{}, false}, - {"systemMemcachedCreated", "system-memcached", &imagev1.ImageStream{}, true}, - // TODO: service account created by AMPImagesReconciler. Should not be there. - {"serviceAccountCreated", "amp", &v1.ServiceAccount{}, true}, - } - - for _, tc := range cases { - t.Run(tc.testName, func(subT *testing.T) { - obj := tc.obj - namespacedName := types.NamespacedName{ - Name: tc.objName, - Namespace: namespace, - } - err = cl.Get(context.TODO(), namespacedName, obj) - if tc.hasToExist { - if err != nil { - subT.Errorf("error fetching object %s: %v", tc.objName, err) - } - } else { - if err == nil || !errors.IsNotFound(err) { - subT.Errorf("object %s that shouldn't exist exists or different error than NotFound returned: %v", tc.objName, err) - } - } - - }) - } -} diff --git a/pkg/3scale/amp/operator/apicast_reconciler_test.go b/pkg/3scale/amp/operator/apicast_reconciler_test.go index 2fedc24e4..a491641f6 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler_test.go +++ b/pkg/3scale/amp/operator/apicast_reconciler_test.go @@ -54,11 +54,10 @@ func TestApicastReconciler(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -191,11 +190,10 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -403,11 +401,10 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -646,11 +643,10 @@ func TestApicastServicePortMutator(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -678,11 +674,10 @@ func TestApicastServicePortMutator(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -709,11 +704,10 @@ func TestApicastServicePortMutator(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -900,11 +894,10 @@ func testApicastAPIManagerCreator(stagingReplicas, productionReplicas *int64) *a }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: &appsv1alpha1.ApicastSpec{ ApicastManagementAPI: &apicastManagementAPI, @@ -1605,11 +1598,10 @@ func testApicastAPIManagerTelemetryCreator(apicastSpec *appsv1alpha1.ApicastSpec }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Apicast: apicastSpec, }, diff --git a/pkg/3scale/amp/operator/backend_reconciler_test.go b/pkg/3scale/amp/operator/backend_reconciler_test.go index eb4c416f4..eb74aa88b 100644 --- a/pkg/3scale/amp/operator/backend_reconciler_test.go +++ b/pkg/3scale/amp/operator/backend_reconciler_test.go @@ -47,11 +47,10 @@ func TestNewBackendReconciler(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Backend: &appsv1alpha1.BackendSpec{ ListenerSpec: &appsv1alpha1.BackendListenerSpec{Replicas: &oneValue}, @@ -263,11 +262,10 @@ func backendApiManagerCreator(listenerReplicas, cronReplicas, workerReplicas *in }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Backend: &appsv1alpha1.BackendSpec{ ListenerSpec: &appsv1alpha1.BackendListenerSpec{Replicas: listenerReplicas}, diff --git a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go index c99a04b83..e6e3abfd4 100644 --- a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go +++ b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go @@ -10,7 +10,6 @@ import ( "github.com/go-logr/logr" grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" - imagev1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" k8sappsv1 "k8s.io/api/apps/v1" @@ -56,10 +55,6 @@ func (r *BaseAPIManagerLogicReconciler) ReconcilePodDisruptionBudget(desired *po return r.ReconcileResource(&policyv1.PodDisruptionBudget{}, desired, mutatefn) } -func (r *BaseAPIManagerLogicReconciler) ReconcileImagestream(desired *imagev1.ImageStream, mutatefn reconcilers.MutateFn) error { - return r.ReconcileResource(&imagev1.ImageStream{}, desired, mutatefn) -} - func (r *BaseAPIManagerLogicReconciler) ReconcileDeployment(desired *k8sappsv1.Deployment, mutatefn reconcilers.MutateFn) error { return r.ReconcileResource(&k8sappsv1.Deployment{}, desired, mutatefn) } diff --git a/pkg/3scale/amp/operator/helper_test.go b/pkg/3scale/amp/operator/helper_test.go index f5b3a9208..30c79bbf0 100644 --- a/pkg/3scale/amp/operator/helper_test.go +++ b/pkg/3scale/amp/operator/helper_test.go @@ -13,13 +13,12 @@ import ( ) const ( - wildcardDomain = "test.3scale.net" - appLabel = "someLabel" - apimanagerName = "example-apimanager" - namespace = "someNS" - tenantName = "someTenant" - insecureImportPolicy = false - trueValue = true + wildcardDomain = "test.3scale.net" + appLabel = "someLabel" + apimanagerName = "example-apimanager" + namespace = "someNS" + tenantName = "someTenant" + trueValue = true ) func addExpectedMeteringLabels(src map[string]string, componentName string, componentType helper.ComponentType) { @@ -43,7 +42,6 @@ func addExpectedMeteringLabels(src map[string]string, componentName string, comp func basicApimanager() *appsv1alpha1.APIManager { tmpAppLabel := appLabel tmpTenantName := tenantName - tmpInsecureImportPolicy := insecureImportPolicy tmpTrueValue := trueValue apimanager := &appsv1alpha1.APIManager{ @@ -53,11 +51,10 @@ func basicApimanager() *appsv1alpha1.APIManager { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - WildcardDomain: wildcardDomain, - AppLabel: &tmpAppLabel, - ImageStreamTagImportInsecure: &tmpInsecureImportPolicy, - TenantName: &tmpTenantName, - ResourceRequirementsEnabled: &tmpTrueValue, + WildcardDomain: wildcardDomain, + AppLabel: &tmpAppLabel, + TenantName: &tmpTenantName, + ResourceRequirementsEnabled: &tmpTrueValue, }, System: &appsv1alpha1.SystemSpec{}, }, diff --git a/pkg/3scale/amp/operator/redis_options_provider.go b/pkg/3scale/amp/operator/redis_options_provider.go index 043b0b428..d6c4c6299 100644 --- a/pkg/3scale/amp/operator/redis_options_provider.go +++ b/pkg/3scale/amp/operator/redis_options_provider.go @@ -35,7 +35,6 @@ func (r *RedisOptionsProvider) GetRedisOptions() (*component.RedisOptions, error r.options.AmpRelease = product.ThreescaleRelease r.options.BackendImageTag = product.ThreescaleRelease r.options.SystemImageTag = product.ThreescaleRelease - r.options.InsecureImportPolicy = r.apimanager.Spec.ImageStreamTagImportInsecure r.options.BackendImage = BackendRedisImageURL() if r.apimanager.Spec.Backend != nil && r.apimanager.Spec.Backend.RedisImage != nil { diff --git a/pkg/3scale/amp/operator/redis_options_provider_test.go b/pkg/3scale/amp/operator/redis_options_provider_test.go index 5ceb06156..2798fbce3 100644 --- a/pkg/3scale/amp/operator/redis_options_provider_test.go +++ b/pkg/3scale/amp/operator/redis_options_provider_test.go @@ -137,7 +137,6 @@ func testSystemRedisSecret() *v1.Secret { } func defaultRedisOptions() *component.RedisOptions { - tmpInsecure := insecureImportPolicy return &component.RedisOptions{ AmpRelease: product.ThreescaleRelease, BackendImageTag: product.ThreescaleRelease, @@ -146,7 +145,6 @@ func defaultRedisOptions() *component.RedisOptions { SystemImage: component.SystemRedisImageURL(), BackendRedisContainerResourceRequirements: component.DefaultBackendRedisContainerResourceRequirements(), SystemRedisContainerResourceRequirements: component.DefaultSystemRedisContainerResourceRequirements(), - InsecureImportPolicy: &tmpInsecure, SystemCommonLabels: testRedisSystemCommonLabels(), SystemRedisLabels: testRedisSystemRedisLabels(), SystemRedisPodTemplateLabels: testRedisSystemRedisPodTemplateLabels(), diff --git a/pkg/3scale/amp/operator/redis_reconciler.go b/pkg/3scale/amp/operator/redis_reconciler.go index 301b6affa..6896e7baa 100644 --- a/pkg/3scale/amp/operator/redis_reconciler.go +++ b/pkg/3scale/amp/operator/redis_reconciler.go @@ -1,7 +1,6 @@ package operator import ( - imagev1 "github.com/openshift/api/image/v1" k8sappsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -22,7 +21,6 @@ type RedisReconciler struct { Service func(redis *component.Redis) *corev1.Service ConfigMap func(redis *component.Redis) *corev1.ConfigMap PersistentVolumeClaim func(redis *component.Redis) *corev1.PersistentVolumeClaim - ImageStream func(redis *component.Redis) *imagev1.ImageStream Secret func(redis *component.Redis) *corev1.Secret } @@ -36,7 +34,6 @@ func NewSystemRedisDependencyReconciler(baseAPIManagerLogicReconciler *BaseAPIMa Service: (*component.Redis).SystemService, ConfigMap: (*component.Redis).ConfigMap, PersistentVolumeClaim: (*component.Redis).SystemPVC, - ImageStream: (*component.Redis).SystemImageStream, Secret: (*component.Redis).SystemRedisSecret, } } @@ -49,7 +46,6 @@ func NewBackendRedisDependencyReconciler(baseAPIManagerLogicReconciler *BaseAPIM Service: (*component.Redis).BackendService, ConfigMap: (*component.Redis).ConfigMap, PersistentVolumeClaim: (*component.Redis).BackendPVC, - ImageStream: (*component.Redis).BackendImageStream, Secret: (*component.Redis).BackendRedisSecret, } } @@ -111,12 +107,6 @@ func (r *RedisReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } - // IS - err = r.ReconcileImagestream(r.ImageStream(redis), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - // Secret err = r.ReconcileSecret(r.Secret(redis), reconcilers.DefaultsOnlySecretMutator) if err != nil { diff --git a/pkg/3scale/amp/operator/redis_reconciler_test.go b/pkg/3scale/amp/operator/redis_reconciler_test.go index e20b646aa..013079c7c 100644 --- a/pkg/3scale/amp/operator/redis_reconciler_test.go +++ b/pkg/3scale/amp/operator/redis_reconciler_test.go @@ -42,11 +42,10 @@ func TestRedisBackendDeploymentReconcilerCreate(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - ResourceRequirementsEnabled: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, + AppLabel: &appLabel, + ResourceRequirementsEnabled: &trueValue, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, }, }, } @@ -100,7 +99,6 @@ func TestRedisBackendDeploymentReconcilerCreate(t *testing.T) { {"backend-redis", &v1.Service{}}, {"redis-config", &v1.ConfigMap{}}, {"backend-redis-storage", &v1.PersistentVolumeClaim{}}, - {"backend-redis", &imagev1.ImageStream{}}, }}, {"systemRedis", NewSystemRedisDependencyReconciler, []struct { objName string @@ -108,7 +106,6 @@ func TestRedisBackendDeploymentReconcilerCreate(t *testing.T) { }{ {"system-redis", &k8sappsv1.Deployment{}}, {"system-redis-storage", &v1.PersistentVolumeClaim{}}, - {"system-redis", &imagev1.ImageStream{}}, {"system-redis", &v1.Service{}}, }}, } diff --git a/pkg/3scale/amp/operator/system_mysql_image_options_provider.go b/pkg/3scale/amp/operator/system_mysql_image_options_provider.go index e0c46f348..5e5bd2b07 100644 --- a/pkg/3scale/amp/operator/system_mysql_image_options_provider.go +++ b/pkg/3scale/amp/operator/system_mysql_image_options_provider.go @@ -21,7 +21,6 @@ func NewSystemMysqlImageOptionsProvider(apimanager *appsv1alpha1.APIManager) *Sy func (s *SystemMysqlImageOptionsProvider) GetSystemMySQLImageOptions() (*component.SystemMySQLImageOptions, error) { s.mysqlImageOptions.AppLabel = *s.apimanager.Spec.AppLabel s.mysqlImageOptions.AmpRelease = product.ThreescaleRelease - s.mysqlImageOptions.InsecureImportPolicy = s.apimanager.Spec.ImageStreamTagImportInsecure s.mysqlImageOptions.Image = SystemMySQLImageURL() if s.apimanager.Spec.System.DatabaseSpec != nil && diff --git a/pkg/3scale/amp/operator/system_mysql_image_options_provider_test.go b/pkg/3scale/amp/operator/system_mysql_image_options_provider_test.go index f6d554da2..82fd48a3c 100644 --- a/pkg/3scale/amp/operator/system_mysql_image_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_mysql_image_options_provider_test.go @@ -15,12 +15,10 @@ const ( ) func defaultSystemMySQLImageOptions() *component.SystemMySQLImageOptions { - tmpInsecure := insecureImportPolicy return &component.SystemMySQLImageOptions{ - AppLabel: appLabel, - AmpRelease: product.ThreescaleRelease, - InsecureImportPolicy: &tmpInsecure, - Image: component.SystemMySQLImageURL(), + AppLabel: appLabel, + AmpRelease: product.ThreescaleRelease, + Image: component.SystemMySQLImageURL(), } } diff --git a/pkg/3scale/amp/operator/system_mysql_image_reconciler.go b/pkg/3scale/amp/operator/system_mysql_image_reconciler.go index 1d9827f1a..a723c81dd 100644 --- a/pkg/3scale/amp/operator/system_mysql_image_reconciler.go +++ b/pkg/3scale/amp/operator/system_mysql_image_reconciler.go @@ -3,34 +3,12 @@ package operator import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" - "github.com/3scale/3scale-operator/pkg/reconcilers" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) type SystemMySQLImageReconciler struct { *BaseAPIManagerLogicReconciler } -func NewSystemMySQLImageReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReconciler) DependencyReconciler { - return &SystemMySQLImageReconciler{ - BaseAPIManagerLogicReconciler: baseAPIManagerLogicReconciler, - } -} - -func (r *SystemMySQLImageReconciler) Reconcile() (reconcile.Result, error) { - systemMySQLImage, err := SystemMySQLImage(r.apiManager) - if err != nil { - return reconcile.Result{}, err - } - - err = r.ReconcileImagestream(systemMySQLImage.ImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - return reconcile.Result{}, nil -} - func SystemMySQLImage(apimanager *appsv1alpha1.APIManager) (*component.SystemMySQLImage, error) { optsProvider := NewSystemMysqlImageOptionsProvider(apimanager) opts, err := optsProvider.GetSystemMySQLImageOptions() diff --git a/pkg/3scale/amp/operator/system_mysql_image_reconciler_test.go b/pkg/3scale/amp/operator/system_mysql_image_reconciler_test.go deleted file mode 100644 index da0a76e9f..000000000 --- a/pkg/3scale/amp/operator/system_mysql_image_reconciler_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package operator - -import ( - "context" - "testing" - - appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" - "github.com/3scale/3scale-operator/pkg/reconcilers" - - imagev1 "github.com/openshift/api/image/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - fakeclientset "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -func TestSystemMySQLImageReconciler(t *testing.T) { - var ( - appLabel = "someLabel" - name = "example-apimanager" - namespace = "operator-unittest" - trueValue = true - imageURL = "mysql:test" - log = logf.Log.WithName("operator_test") - ) - - ctx := context.TODO() - - apimanager := &appsv1alpha1.APIManager{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: appsv1alpha1.APIManagerSpec{ - APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - }, - System: &appsv1alpha1.SystemSpec{ - DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ - MySQL: &appsv1alpha1.SystemMySQLSpec{ - Image: &imageURL, - }, - }, - }, - }, - } - s := scheme.Scheme - s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := imagev1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - - // Objects to track in the fake client. - objs := []runtime.Object{} - - // Create a fake client to mock API calls. - cl := fake.NewFakeClient(objs...) - clientAPIReader := fake.NewFakeClient(objs...) - clientset := fakeclientset.NewSimpleClientset() - recorder := record.NewFakeRecorder(10000) - - baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) - baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) - - reconciler := NewSystemMySQLImageReconciler(baseAPIManagerLogicReconciler) - _, err = reconciler.Reconcile() - if err != nil { - t.Fatal(err) - } - - obj := &imagev1.ImageStream{} - - namespacedName := types.NamespacedName{ - Name: "system-mysql", - Namespace: namespace, - } - err = cl.Get(context.TODO(), namespacedName, obj) - // object must exist, that is all required to be tested - if err != nil { - t.Fatal(err) - } -} diff --git a/pkg/3scale/amp/operator/system_mysql_reconciler_test.go b/pkg/3scale/amp/operator/system_mysql_reconciler_test.go index bc5e02f19..8c89b96ad 100644 --- a/pkg/3scale/amp/operator/system_mysql_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_mysql_reconciler_test.go @@ -44,11 +44,10 @@ func TestSystemMySQLReconcilerCreate(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - ResourceRequirementsEnabled: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, + AppLabel: &appLabel, + ResourceRequirementsEnabled: &trueValue, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, }, System: &appsv1alpha1.SystemSpec{ DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ diff --git a/pkg/3scale/amp/operator/system_postgresql_image_options_provider.go b/pkg/3scale/amp/operator/system_postgresql_image_options_provider.go index 322d2c382..9f877af15 100644 --- a/pkg/3scale/amp/operator/system_postgresql_image_options_provider.go +++ b/pkg/3scale/amp/operator/system_postgresql_image_options_provider.go @@ -21,7 +21,6 @@ func NewSystemPostgreSQLImageOptionsProvider(apimanager *appsv1alpha1.APIManager func (s *SystemPostgreSQLImageOptionsProvider) GetSystemPostgreSQLImageOptions() (*component.SystemPostgreSQLImageOptions, error) { s.options.AppLabel = *s.apimanager.Spec.AppLabel s.options.AmpRelease = product.ThreescaleRelease - s.options.InsecureImportPolicy = s.apimanager.Spec.ImageStreamTagImportInsecure s.options.Image = SystemPostgreSQLImageURL() if s.apimanager.Spec.System.DatabaseSpec != nil && diff --git a/pkg/3scale/amp/operator/system_postgresql_image_options_provider_test.go b/pkg/3scale/amp/operator/system_postgresql_image_options_provider_test.go index 1bed2a1ca..e839340dd 100644 --- a/pkg/3scale/amp/operator/system_postgresql_image_options_provider_test.go +++ b/pkg/3scale/amp/operator/system_postgresql_image_options_provider_test.go @@ -15,12 +15,10 @@ const ( ) func defaultSystemPostgreSQLImageOptions() *component.SystemPostgreSQLImageOptions { - tmpInsecure := insecureImportPolicy return &component.SystemPostgreSQLImageOptions{ - AppLabel: appLabel, - AmpRelease: product.ThreescaleRelease, - InsecureImportPolicy: &tmpInsecure, - Image: component.SystemPostgreSQLImageURL(), + AppLabel: appLabel, + AmpRelease: product.ThreescaleRelease, + Image: component.SystemPostgreSQLImageURL(), } } diff --git a/pkg/3scale/amp/operator/system_postgresql_image_reconciler.go b/pkg/3scale/amp/operator/system_postgresql_image_reconciler.go index 127ad2ef5..c0e439dc3 100644 --- a/pkg/3scale/amp/operator/system_postgresql_image_reconciler.go +++ b/pkg/3scale/amp/operator/system_postgresql_image_reconciler.go @@ -3,35 +3,12 @@ package operator import ( appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" - "github.com/3scale/3scale-operator/pkg/reconcilers" - - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) type SystemPostgreSQLImageReconciler struct { *BaseAPIManagerLogicReconciler } -func NewSystemPostgreSQLImageReconciler(baseAPIManagerLogicReconciler *BaseAPIManagerLogicReconciler) DependencyReconciler { - return &SystemPostgreSQLImageReconciler{ - BaseAPIManagerLogicReconciler: baseAPIManagerLogicReconciler, - } -} - -func (r *SystemPostgreSQLImageReconciler) Reconcile() (reconcile.Result, error) { - systemPostgreSQLImage, err := SystemPostgreSQLImage(r.apiManager) - if err != nil { - return reconcile.Result{}, err - } - - err = r.ReconcileImagestream(systemPostgreSQLImage.ImageStream(), reconcilers.GenericImageStreamMutator) - if err != nil { - return reconcile.Result{}, err - } - - return reconcile.Result{}, nil -} - func SystemPostgreSQLImage(apimanager *appsv1alpha1.APIManager) (*component.SystemPostgreSQLImage, error) { optsProvider := NewSystemPostgreSQLImageOptionsProvider(apimanager) opts, err := optsProvider.GetSystemPostgreSQLImageOptions() diff --git a/pkg/3scale/amp/operator/system_postgresql_image_reconciler_test.go b/pkg/3scale/amp/operator/system_postgresql_image_reconciler_test.go deleted file mode 100644 index eb9155c43..000000000 --- a/pkg/3scale/amp/operator/system_postgresql_image_reconciler_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package operator - -import ( - "context" - "testing" - - appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" - "github.com/3scale/3scale-operator/pkg/reconcilers" - - imagev1 "github.com/openshift/api/image/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - fakeclientset "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -func TestSystemPostgreSQLImageReconcilerCreate(t *testing.T) { - var ( - appLabel = "someLabel" - name = "example-apimanager" - namespace = "operator-unittest" - trueValue = true - imageURL = "postgresql:test" - log = logf.Log.WithName("operator_test") - ) - - ctx := context.TODO() - - apimanager := &appsv1alpha1.APIManager{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: appsv1alpha1.APIManagerSpec{ - APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - }, - System: &appsv1alpha1.SystemSpec{ - DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ - PostgreSQL: &appsv1alpha1.SystemPostgreSQLSpec{ - Image: &imageURL, - }, - }, - }, - }, - } - s := scheme.Scheme - s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) - err := imagev1.AddToScheme(s) - if err != nil { - t.Fatal(err) - } - - // Objects to track in the fake client. - objs := []runtime.Object{} - - // Create a fake client to mock API calls. - cl := fake.NewFakeClient(objs...) - clientAPIReader := fake.NewFakeClient(objs...) - clientset := fakeclientset.NewSimpleClientset() - recorder := record.NewFakeRecorder(10000) - - baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) - baseAPIManagerLogicReconciler := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) - - reconciler := NewSystemPostgreSQLImageReconciler(baseAPIManagerLogicReconciler) - _, err = reconciler.Reconcile() - if err != nil { - t.Fatal(err) - } - - obj := &imagev1.ImageStream{} - - namespacedName := types.NamespacedName{ - Name: "system-postgresql", - Namespace: namespace, - } - err = cl.Get(context.TODO(), namespacedName, obj) - // object must exist, that is all required to be tested - if err != nil { - t.Fatal(err) - } -} diff --git a/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go b/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go index 39736b7fe..6e9678633 100644 --- a/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_postgresql_reconciler_test.go @@ -44,11 +44,10 @@ func TestSystemPostgreSQLReconcilerCreate(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - ResourceRequirementsEnabled: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, + AppLabel: &appLabel, + ResourceRequirementsEnabled: &trueValue, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, }, System: &appsv1alpha1.SystemSpec{ DatabaseSpec: &appsv1alpha1.SystemDatabaseSpec{ diff --git a/pkg/3scale/amp/operator/system_reconciler_test.go b/pkg/3scale/amp/operator/system_reconciler_test.go index cc30d857e..a24eaa425 100644 --- a/pkg/3scale/amp/operator/system_reconciler_test.go +++ b/pkg/3scale/amp/operator/system_reconciler_test.go @@ -288,11 +288,10 @@ func testSystemAPIManagerCreator(appReplicas, sidekiqReplicas *int64) *appsv1alp Spec: appsv1alpha1.APIManagerSpec{ Apicast: &appsv1alpha1.ApicastSpec{RegistryURL: &tmpApicastRegistryURL}, APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, System: &appsv1alpha1.SystemSpec{ AppSpec: &appsv1alpha1.SystemAppSpec{Replicas: appReplicas}, diff --git a/pkg/3scale/amp/operator/zync_reconciler_test.go b/pkg/3scale/amp/operator/zync_reconciler_test.go index 1e374bc04..936ea6821 100644 --- a/pkg/3scale/amp/operator/zync_reconciler_test.go +++ b/pkg/3scale/amp/operator/zync_reconciler_test.go @@ -51,11 +51,10 @@ func TestNewZyncReconciler(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Zync: &appsv1alpha1.ZyncSpec{ AppSpec: &appsv1alpha1.ZyncAppSpec{Replicas: &oneValue}, @@ -167,11 +166,10 @@ func TestNewZyncReconcilerWithAllExternalDatabases(t *testing.T) { }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Zync: &appsv1alpha1.ZyncSpec{ AppSpec: &appsv1alpha1.ZyncAppSpec{Replicas: &oneValue}, @@ -389,11 +387,10 @@ func testZyncAPIManagerCreator(zyncReplicas, zyncQueReplicas *int64) *appsv1alph }, Spec: appsv1alpha1.APIManagerSpec{ APIManagerCommonSpec: appsv1alpha1.APIManagerCommonSpec{ - AppLabel: &appLabel, - ImageStreamTagImportInsecure: &trueValue, - WildcardDomain: wildcardDomain, - TenantName: &tenantName, - ResourceRequirementsEnabled: &trueValue, + AppLabel: &appLabel, + WildcardDomain: wildcardDomain, + TenantName: &tenantName, + ResourceRequirementsEnabled: &trueValue, }, Zync: &appsv1alpha1.ZyncSpec{ AppSpec: &appsv1alpha1.ZyncAppSpec{Replicas: zyncReplicas}, diff --git a/pkg/reconcilers/imagestreams.go b/pkg/reconcilers/imagestreams.go deleted file mode 100644 index 0fb721ec5..000000000 --- a/pkg/reconcilers/imagestreams.go +++ /dev/null @@ -1,66 +0,0 @@ -package reconcilers - -import ( - "fmt" - "reflect" - - "github.com/3scale/3scale-operator/pkg/common" - imagev1 "github.com/openshift/api/image/v1" -) - -func GenericImageStreamMutator(existingObj, desiredObj common.KubernetesObject) (bool, error) { - existing, ok := existingObj.(*imagev1.ImageStream) - if !ok { - return false, fmt.Errorf("%T is not a *imagev1.ImageStream", existingObj) - } - desired, ok := desiredObj.(*imagev1.ImageStream) - if !ok { - return false, fmt.Errorf("%T is not a *imagev1.ImageStream", desiredObj) - } - - // merging approach will be implemented - // spec.tags tagrefences in desired must exist in existing. - // If element does not exist, append - // If exists (by name), ensure From field and ImportPolicy fields are equal. - updated := false - - findTagReference := func(tagRefName string, tagRefS []imagev1.TagReference) int { - for i := range tagRefS { - if tagRefS[i].Name == tagRefName { - return i - } - } - return -1 - } - - for idx := range desired.Spec.Tags { - if existingIdx := findTagReference(desired.Spec.Tags[idx].Name, existing.Spec.Tags); existingIdx < 0 { - // does not exist, append - existing.Spec.Tags = append(existing.Spec.Tags, desired.Spec.Tags[idx]) - updated = true - } else { - // exists, reconcile - tmpUpdated := imageStreamReconcile(existing.Spec.Tags, existingIdx, desired.Spec.Tags, idx) - updated = updated || tmpUpdated - } - } - - return updated, nil -} - -func imageStreamReconcile(existingTags []imagev1.TagReference, existingIdx int, desiredTags []imagev1.TagReference, desiredIdx int) bool { - // From and ImportPolicy fields are equal. - updated := false - - if !reflect.DeepEqual(existingTags[existingIdx].From, desiredTags[desiredIdx].From) { - existingTags[existingIdx].From = desiredTags[desiredIdx].From - updated = true - } - - if !reflect.DeepEqual(existingTags[existingIdx].ImportPolicy, desiredTags[desiredIdx].ImportPolicy) { - existingTags[existingIdx].ImportPolicy = desiredTags[desiredIdx].ImportPolicy - updated = true - } - - return updated -} diff --git a/pkg/reconcilers/imagestreams_test.go b/pkg/reconcilers/imagestreams_test.go deleted file mode 100644 index 3926518eb..000000000 --- a/pkg/reconcilers/imagestreams_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package reconcilers - -import ( - "testing" - - imagev1 "github.com/openshift/api/image/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestGenericImageStreamMutator(t *testing.T) { - existing := &imagev1.ImageStream{ - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - ObjectMeta: metav1.ObjectMeta{Name: "myIS", Namespace: "MyNS"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - Name: "tag0", - ImportPolicy: imagev1.TagImportPolicy{Insecure: false, Scheduled: true}, - From: &v1.ObjectReference{Kind: "ImageStreamTag", Name: "3scale-0"}, - }, - imagev1.TagReference{ - Name: "tag1", - ImportPolicy: imagev1.TagImportPolicy{Insecure: false, Scheduled: true}, - From: &v1.ObjectReference{Kind: "ImageStreamTag", Name: "3scale-1"}, - }, - }, - }, - } - - desired := &imagev1.ImageStream{ - TypeMeta: metav1.TypeMeta{APIVersion: "image.openshift.io/v1", Kind: "ImageStream"}, - ObjectMeta: metav1.ObjectMeta{Name: "myIS", Namespace: "MyNS"}, - Spec: imagev1.ImageStreamSpec{ - Tags: []imagev1.TagReference{ - imagev1.TagReference{ - // tag that should be updated - Name: "tag1", - ImportPolicy: imagev1.TagImportPolicy{Insecure: true, Scheduled: false}, - From: &v1.ObjectReference{Kind: "ImageStreamTag", Name: "3scale-1other"}, - }, - imagev1.TagReference{ - // tag that should be added - Name: "tag2", - ImportPolicy: imagev1.TagImportPolicy{Insecure: false, Scheduled: true}, - From: &v1.ObjectReference{Kind: "ImageStreamTag", Name: "3scale-2"}, - }, - }, - }, - } - - update, err := GenericImageStreamMutator(existing, desired) - if err != nil { - t.Fatal(err) - } - if !update { - t.Fatal("when defaults can be applied, reconciler reported no update needed") - } - - if len(existing.Spec.Tags) != 3 { - t.Fatalf("reconciled obj does not have expected number of tags. Expected: 3, got: %d", len(existing.Spec.Tags)) - } - - findTagReference := func(tagRefName string, tagRefS []imagev1.TagReference) int { - for i := range tagRefS { - if tagRefS[i].Name == tagRefName { - return i - } - } - return -1 - } - - // tag0 existed previously in obj, should be left untouched - tag0Index := findTagReference("tag0", existing.Spec.Tags) - if tag0Index < 0 { - t.Fatal("reconciled obj does not have tag0") - } - - // tag1 existed previously in obj, should be updated - tag1Index := findTagReference("tag1", existing.Spec.Tags) - if tag1Index < 0 { - t.Fatal("reconciled obj does not have tag1") - } - tag1 := existing.Spec.Tags[1] - // From and ImportPolicy fields should have been reconciled - if tag1.From.Name != "3scale-1other" { - t.Fatal("reconciled obj tag1 'from' was not reconciled") - } - - if !tag1.ImportPolicy.Insecure { - t.Fatal("reconciled obj tag1 'impoortpolicy.insecure' was not reconciled") - } - - if tag1.ImportPolicy.Scheduled { - t.Fatal("reconciled obj tag1 'impoortpolicy.scheduled' was not reconciled") - } - - // tag2 did not exist previously in obj, should be appended - tag2Index := findTagReference("tag2", existing.Spec.Tags) - if tag2Index < 0 { - t.Fatal("reconciled obj does not have tag2") - } -} diff --git a/pkg/upgrade/delete_image_streams_2.15.go b/pkg/upgrade/delete_image_streams_2.15.go new file mode 100644 index 000000000..939ecc5c2 --- /dev/null +++ b/pkg/upgrade/delete_image_streams_2.15.go @@ -0,0 +1,61 @@ +package upgrade + +import ( + "context" + "fmt" + + imagev1 "github.com/openshift/api/image/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// DeleteImageStreams deletes the ImageStream objects managed by APIManager as they are no longer needed with Deployments +// 3scale 2.14 -> 2.15 +func DeleteImageStreams(namespace string, client k8sclient.Client) error { + var imageStreams = []string{ + "amp-apicast", + "amp-backend", + "amp-system", + "amp-zync", + "backend-redis", + "system-memcached", + "system-mysql", + "system-redis", + "system-searchd", + "zync-database-postgresql", + } + + // First check if any ImageStreams exist, if they've already been deleted then break out + imageStreamList := &imagev1.ImageStreamList{} + listOps := []k8sclient.ListOption{ + k8sclient.InNamespace(namespace), + } + err := client.List(context.TODO(), imageStreamList, listOps...) + if err != nil { + return fmt.Errorf("failed to list ImageStreams: %v", err) + } + if len(imageStreamList.Items) == 0 { + return nil + } + + // Delete the specified ImageStreams + for _, imageStreamName := range imageStreams { + imageStream := &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{ + Name: imageStreamName, + Namespace: namespace, + }, + } + + err := client.Delete(context.TODO(), imageStream) + if err != nil { + if !k8serr.IsNotFound(err) { + return fmt.Errorf("error deleting ImageStream %s: %v", imageStream.Name, err) + } + } + + } + + return nil +}