From 93a94bbc2e226e8955f57adde172b74b15ccbcb2 Mon Sep 17 00:00:00 2001 From: Vladica Obojevic Date: Fri, 1 Sep 2023 16:31:14 +0200 Subject: [PATCH 1/3] add topology spread constraints so ditto services spread equally among cluster nodes Signed-off-by: Vladica Obojevic --- .../templates/connectivity-deployment.yaml | 7 +++ .../ditto/templates/dittoui-deployment.yaml | 7 +++ .../ditto/templates/gateway-deployment.yaml | 7 +++ .../ditto/templates/nginx-deployment.yaml | 7 +++ .../helm/ditto/templates/nginx-ingress.yaml | 7 +++ .../ditto/templates/policies-deployment.yaml | 7 +++ .../ditto/templates/swaggerui-deployment.yaml | 7 +++ .../ditto/templates/things-deployment.yaml | 7 +++ .../templates/thingssearch-deployment.yaml | 7 +++ deployment/helm/ditto/values.yaml | 54 +++++++++++++++++++ 10 files changed, 117 insertions(+) diff --git a/deployment/helm/ditto/templates/connectivity-deployment.yaml b/deployment/helm/ditto/templates/connectivity-deployment.yaml index b644969b47..8b967339c0 100644 --- a/deployment/helm/ditto/templates/connectivity-deployment.yaml +++ b/deployment/helm/ditto/templates/connectivity-deployment.yaml @@ -68,6 +68,13 @@ spec: - name: ditto-log-files-directory mountPath: /var/log/ditto {{- end }} + topologySpreadConstraints: + - maxSkew: {{ .Values.connectivity.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.connectivity.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.connectivity.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity containers: - name: {{ .Chart.Name }}-connectivity image: {{ printf "%s:%s" .Values.connectivity.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.connectivity.image.tag ) ) }} diff --git a/deployment/helm/ditto/templates/dittoui-deployment.yaml b/deployment/helm/ditto/templates/dittoui-deployment.yaml index 3ba80f3927..6d3781617a 100644 --- a/deployment/helm/ditto/templates/dittoui-deployment.yaml +++ b/deployment/helm/ditto/templates/dittoui-deployment.yaml @@ -44,6 +44,13 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + topologySpreadConstraints: + - maxSkew: {{ .Values.dittoui.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.dittoui.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.dittoui.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui containers: - name: {{ .Chart.Name }}-dittoui image: {{ printf "%s:%s" .Values.dittoui.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.dittoui.image.tag ) ) }} diff --git a/deployment/helm/ditto/templates/gateway-deployment.yaml b/deployment/helm/ditto/templates/gateway-deployment.yaml index 7e28022557..30c0efe025 100644 --- a/deployment/helm/ditto/templates/gateway-deployment.yaml +++ b/deployment/helm/ditto/templates/gateway-deployment.yaml @@ -69,6 +69,13 @@ spec: - name: ditto-log-files-directory mountPath: /var/log/ditto {{- end }} + topologySpreadConstraints: + - maxSkew: {{ .Values.gateway.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.gateway.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.gateway.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway containers: - name: {{ .Chart.Name }}-gateway image: {{ printf "%s:%s" .Values.gateway.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.gateway.image.tag ) ) }} diff --git a/deployment/helm/ditto/templates/nginx-deployment.yaml b/deployment/helm/ditto/templates/nginx-deployment.yaml index 3fa88faec5..47f6306a8d 100644 --- a/deployment/helm/ditto/templates/nginx-deployment.yaml +++ b/deployment/helm/ditto/templates/nginx-deployment.yaml @@ -61,6 +61,13 @@ spec: sleep 1; done {{- end }} + topologySpreadConstraints: + - maxSkew: {{ .Values.nginx.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.nginx.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.nginx.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx containers: - name: {{ .Chart.Name }}-nginx image: "{{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag }}" diff --git a/deployment/helm/ditto/templates/nginx-ingress.yaml b/deployment/helm/ditto/templates/nginx-ingress.yaml index 8c05ee6721..9f209c5105 100644 --- a/deployment/helm/ditto/templates/nginx-ingress.yaml +++ b/deployment/helm/ditto/templates/nginx-ingress.yaml @@ -575,6 +575,13 @@ spec: supplementalGroups: [101] seccompProfile: type: RuntimeDefault + topologySpreadConstraints: + - maxSkew: {{ .Values.ingress.controller.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.ingress.controller.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.ingress.controller.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" containers: - name: nginx-node-health-check image: docker.io/library/nginx:{{ .Values.ingress.controller.nginxVersion }} diff --git a/deployment/helm/ditto/templates/policies-deployment.yaml b/deployment/helm/ditto/templates/policies-deployment.yaml index ea372d7915..d97ffb0694 100644 --- a/deployment/helm/ditto/templates/policies-deployment.yaml +++ b/deployment/helm/ditto/templates/policies-deployment.yaml @@ -68,6 +68,13 @@ spec: - name: ditto-log-files-directory mountPath: /var/log/ditto {{- end }} + topologySpreadConstraints: + - maxSkew: {{ .Values.policies.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.policies.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.policies.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-policies containers: - name: {{ .Chart.Name }}-policies image: {{ printf "%s:%s" .Values.policies.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.policies.image.tag ) ) }} diff --git a/deployment/helm/ditto/templates/swaggerui-deployment.yaml b/deployment/helm/ditto/templates/swaggerui-deployment.yaml index 360e13fcb2..0e553d52ca 100644 --- a/deployment/helm/ditto/templates/swaggerui-deployment.yaml +++ b/deployment/helm/ditto/templates/swaggerui-deployment.yaml @@ -78,6 +78,13 @@ spec: volumeMounts: - name: swagger-ui-init-config mountPath: /init-config + topologySpreadConstraints: + - maxSkew: {{ .Values.swaggerui.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.swaggerui.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.swaggerui.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-swaggerui containers: - name: {{ .Chart.Name }}-swaggerui image: "{{ .Values.swaggerui.image.repository }}:{{ .Values.swaggerui.image.tag }}" diff --git a/deployment/helm/ditto/templates/things-deployment.yaml b/deployment/helm/ditto/templates/things-deployment.yaml index a535598555..f741898bfc 100644 --- a/deployment/helm/ditto/templates/things-deployment.yaml +++ b/deployment/helm/ditto/templates/things-deployment.yaml @@ -68,6 +68,13 @@ spec: - name: ditto-log-files-directory mountPath: /var/log/ditto {{- end }} + topologySpreadConstraints: + - maxSkew: {{ .Values.things.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.things.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.things.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-things containers: - name: {{ .Chart.Name }}-things image: {{ printf "%s:%s" .Values.things.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.things.image.tag ) ) }} diff --git a/deployment/helm/ditto/templates/thingssearch-deployment.yaml b/deployment/helm/ditto/templates/thingssearch-deployment.yaml index 7c32d457eb..6055855bab 100644 --- a/deployment/helm/ditto/templates/thingssearch-deployment.yaml +++ b/deployment/helm/ditto/templates/thingssearch-deployment.yaml @@ -68,6 +68,13 @@ spec: - name: ditto-log-files-directory mountPath: /var/log/ditto {{- end }} + topologySpreadConstraints: + - maxSkew: {{ .Values.thingsSearch.topologySpreadConstraints.maxSkew }} + topologyKey: {{ .Values.thingsSearch.topologySpreadConstraints.topologyKey }} + whenUnsatisfiable: {{ .Values.thingsSearch.topologySpreadConstraints.whenUnsatisfiable }} + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "ditto.name" . }}-thingssearch containers: - name: {{ .Chart.Name }}-thingssearch image: {{ printf "%s:%s" .Values.thingsSearch.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.thingsSearch.image.tag ) ) }} diff --git a/deployment/helm/ditto/values.yaml b/deployment/helm/ditto/values.yaml index fb6f628f45..7485eaa5a6 100644 --- a/deployment/helm/ditto/values.yaml +++ b/deployment/helm/ditto/values.yaml @@ -185,6 +185,12 @@ ingress: nginxIngressVersion: "v1.8.0" # Nginx Version. Check Supported Versions table from https://github.com/kubernetes/ingress-nginx to match k8s version. nginxVersion: "1.21.6" + # Pod topology spread constraints for nginx-ingress controller + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway annotations: nginx.ingress.kubernetes.io/service-upstream: "true" nginx.ingress.kubernetes.io/server-snippet: | @@ -469,6 +475,12 @@ policies: # affinity configuration for policies # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # Pod topology spread constraints for policies + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # podMonitor configuration for policies podMonitor: # enabled configures whether Pod Monitor is enabled, then a resource to scrape policies metrics will be created @@ -663,6 +675,12 @@ things: # affinity configuration for things # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # Pod topology spread constraints for things + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # podMonitor configuration for things podMonitor: # enabled configures whether Pod Monitor is enabled, then a resource to scrape things metrics will be created @@ -879,6 +897,12 @@ thingsSearch: # affinity configuration for things-search # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # Pod topology spread constraints for things-search + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # podMonitor configuration for things-search podMonitor: # enabled configures whether Pod Monitor is enabled, then a resource to scrape things search metrics will be created @@ -1055,6 +1079,12 @@ connectivity: # affinity configuration for connectivity # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # Pod topology spread constraints for connectivity + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # podMonitor configuration for connectivity podMonitor: # enabled configures whether Pod Monitor is enabled, then a resource to scrape connectivity metrics will be created @@ -1314,6 +1344,12 @@ gateway: # affinity configuration for gateway # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # Pod topology spread constraints for gateway + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # podMonitor configuration for gateway podMonitor: # enabled configures whether Pod Monitor is enabled, then a resource to scrape gateway metrics will be created @@ -1471,6 +1507,12 @@ nginx: # affinity configuration for nginx # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # Pod topology spread constraints for nginx + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # init containers for nginx initContainers: waitForGateway: @@ -1517,6 +1559,12 @@ dittoui: cpu: 0.1 # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s memoryMi: 64 + # Pod topology spread constraints for Ditto UI + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # podDisruptionBudget ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ podDisruptionBudget: # enabled controls whether Ditto UI related PodDisruptionBudget should be created @@ -1563,6 +1611,12 @@ swaggerui: cpu: 0.1 # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s memoryMi: 64 + # Pod topology spread constraints for the swagger ui + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway # podDisruptionBudget ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ podDisruptionBudget: # enabled controls whether swagger ui related PodDisruptionBudget should be created From c81c2b7153d1e88f9d0e20653d7ae515b573e69b Mon Sep 17 00:00:00 2001 From: Vladica Obojevic Date: Wed, 6 Sep 2023 16:52:50 +0200 Subject: [PATCH 2/3] extract to values.yaml file hard-coded nginx-ingress configuration values Signed-off-by: Vladica Obojevic --- .../helm/ditto/templates/nginx-ingress.yaml | 16 +++++++-------- deployment/helm/ditto/values.yaml | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/deployment/helm/ditto/templates/nginx-ingress.yaml b/deployment/helm/ditto/templates/nginx-ingress.yaml index 9f209c5105..c9e1121eea 100644 --- a/deployment/helm/ditto/templates/nginx-ingress.yaml +++ b/deployment/helm/ditto/templates/nginx-ingress.yaml @@ -539,18 +539,18 @@ metadata: name: nginx-ingress-controller namespace: "{{ .Values.ingress.controller.namespace }}" spec: - replicas: 1 + replicas: {{ .Values.ingress.controller.replicaCount }} selector: matchLabels: app.kubernetes.io/name: "{{ .Values.ingress.controller.namespace }}" app.kubernetes.io/part-of: "{{ .Values.ingress.controller.namespace }}" strategy: - type: RollingUpdate + type: {{ .Values.ingress.controller.updateStrategy.type }} rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - minReadySeconds: 10 - revisionHistoryLimit: 5 + maxSurge: {{ .Values.ingress.controller.updateStrategy.rollingUpdate.maxSurge }} + maxUnavailable: {{ .Values.ingress.controller.updateStrategy.rollingUpdate.maxUnavailable }} + minReadySeconds: {{ .Values.ingress.controller.minReadySeconds }} + revisionHistoryLimit: {{ .Values.ingress.controller.revisionHistoryLimit }} template: metadata: labels: @@ -692,8 +692,8 @@ spec: successThreshold: 1 resources: requests: - cpu: "0.75" - memory: "1024Mi" + cpu: {{ mulf .Values.ingress.controller.resources.cpu 1000 }}m + memory: {{ .Values.ingress.controller.resources.memoryMi }}Mi volumeMounts: - mountPath: /usr/local/certificates/ name: webhook-cert diff --git a/deployment/helm/ditto/values.yaml b/deployment/helm/ditto/values.yaml index 7485eaa5a6..3429faa967 100644 --- a/deployment/helm/ditto/values.yaml +++ b/deployment/helm/ditto/values.yaml @@ -179,6 +179,26 @@ ingress: controller: # enabled whether Ingress controller should be enabled enabled: false + # replicaCount configuration for the ingress controller + replicaCount: 1 + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + # minReadySeconds configures the minimum number of seconds for which a newly created Pod should be ready without any + # of its containers crashing, for it to be considered available + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds + minReadySeconds: 10 + # specify how many old ReplicaSets for ingress controller deployment will be retained + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#clean-up-policy + revisionHistoryLimit: 5 + # resources configures the resources available/to use for the policies service + resources: + # cpu defines the "required" CPU of a node so that the service is placed there + cpu: 0.75 + # memoryMi defines the memory in mebibyte (MiB) used as "required" and "limit" in k8s + memoryMi: 1024 # namespace for ingress controller, managed by helm, should not be created manually namespace: ingress-nginx # Ingress-NGINX version. Check Supported Versions table from https://github.com/kubernetes/ingress-nginx to match k8s version. From f34a465fbbe61c69b992d4c86e388a3f2f7cdc24 Mon Sep 17 00:00:00 2001 From: Vladica Obojevic Date: Wed, 6 Sep 2023 17:09:38 +0200 Subject: [PATCH 3/3] Bump Chart version Signed-off-by: Vladica Obojevic --- deployment/helm/ditto/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/helm/ditto/Chart.yaml b/deployment/helm/ditto/Chart.yaml index 8fcc0c6d92..09243ed44f 100644 --- a/deployment/helm/ditto/Chart.yaml +++ b/deployment/helm/ditto/Chart.yaml @@ -16,7 +16,7 @@ description: | A digital twin is a virtual, cloud based, representation of his real world counterpart (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc). type: application -version: 3.3.6 # chart version is effectively set by release-job +version: 3.3.7 # chart version is effectively set by release-job appVersion: 3.3.6 keywords: - iot-chart