From e139cdfaf8ee0dc95a743f1370c0e28ac741b470 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Wed, 29 Nov 2023 10:49:24 +0100 Subject: [PATCH] skipper: enable manual canary deployment requires dict function https://github.com/zalando-incubator/cluster-lifecycle-manager/pull/711 Signed-off-by: Alexander Yastrebov --- cluster/config-defaults.yaml | 8 +++ cluster/manifests/deletions.yaml | 6 +++ cluster/manifests/skipper/deployment.yaml | 64 ++++++++++++++++------- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 980c128731..fe0740e9e6 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -70,6 +70,14 @@ skipper_ingress_min_replicas: "2" {{end}} skipper_ingress_cpu: "1000m" skipper_ingress_memory: "1500Mi" + +# Non empty value enables deployment of specified canary version +skipper_ingress_canary_internal_version: "" + +# Optional canary arguments separated by "[cf724afc]" to allow whitespaces, e.g.: +# skipper_ingress_canary_args: "-foo=has a whitespace[cf724afc]-baz=qux" +skipper_ingress_canary_args: "" + # When set to true (and dedicated node pool for skipper is also true) the # daemonset overhead will be subtracted from the cpu settings such # that skipper will perfectly fit on the node. diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index 05bfd0b48d..5945a24348 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -59,6 +59,12 @@ post_apply: kind: ClusterRoleBinding {{- end }} +{{ if not .Cluster.ConfigItems.skipper_ingress_canary_internal_version }} +- name: skipper-ingress-canary + namespace: kube-system + kind: Deployment +{{ end }} + {{ if eq .Cluster.ConfigItems.skipper_routesrv_enabled "false" }} - name: skipper-ingress-routesrv namespace: kube-system diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index cda4a03267..9ceb216337 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -1,27 +1,49 @@ {{ $internal_version := "v0.18.42-691" }} {{ $version := index (split $internal_version "-") 0 }} +{{ template "skipper-ingress" dict + "ConfigItems" .ConfigItems + "name" "skipper-ingress" + "internal_version" $internal_version +}} + +{{ if .ConfigItems.skipper_ingress_canary_internal_version }} +{{ template "skipper-ingress" dict + "ConfigItems" .ConfigItems + "name" "skipper-ingress-canary" + "internal_version" .ConfigItems.skipper_ingress_canary_internal_version + "replicas" 1 + "args" (index .ConfigItems "skipper_ingress_canary_args") +}} +{{ end }} + +{{ define "skipper-ingress" }} +{{ $version := index (split .internal_version "-") 0 }} +--- apiVersion: apps/v1 kind: Deployment metadata: - name: skipper-ingress + name: "{{ .name }}" namespace: kube-system labels: application: skipper-ingress version: {{ $version }} component: ingress spec: +{{ if index . "replicas" }} + replicas: {{ .replicas }} +{{ end }} strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 0 selector: matchLabels: - deployment: skipper-ingress + deployment: "{{ .name }}" template: metadata: labels: - deployment: skipper-ingress + deployment: "{{ .name }}" application: skipper-ingress version: {{ $version }} component: ingress @@ -29,7 +51,7 @@ spec: kubernetes-log-watcher/scalyr-parser: | [{"container": "skipper-ingress", "parser": "skipper-access-log"}] config/hash: {{"secret.yaml" | manifestHash}} - logging/destination: "{{.Cluster.ConfigItems.log_destination_local}}" + logging/destination: "{{ .ConfigItems.log_destination_local }}" prometheus.io/path: /metrics prometheus.io/port: "9911" prometheus.io/scrape: "true" @@ -48,9 +70,9 @@ spec: # kube-system in our admitters, since we've never really had any issues with them. parent-resource-hash: 71556441059f2d033fb06b1e73df03598c7ecaa6 {{- end }} - priorityClassName: "{{ .Cluster.ConfigItems.system_priority_class }}" + priorityClassName: "{{ .ConfigItems.system_priority_class }}" serviceAccountName: skipper-ingress - terminationGracePeriodSeconds: {{ .Cluster.ConfigItems.skipper_termination_grace_period }} + terminationGracePeriodSeconds: {{ .ConfigItems.skipper_termination_grace_period }} dnsPolicy: ClusterFirstWithHostNet hostNetwork: true containers: @@ -114,7 +136,7 @@ spec: - "run.sh" - "skipper" - "-validate-query={{ .ConfigItems.skipper_validate_query }}" - - "-validate-query-log={{ .Cluster.ConfigItems.skipper_validate_query_log }}" + - "-validate-query-log={{ .ConfigItems.skipper_validate_query_log }}" {{ if eq .ConfigItems.skipper_routesrv_enabled "exec" }} - "-routes-urls=http://skipper-ingress-routesrv.kube-system.svc.cluster.local/routes" - "-normalize-host" @@ -129,7 +151,7 @@ spec: {{ end }} - "-address=:9999" - "-wait-first-route-load" - - "-wait-for-healthcheck-interval={{ .Cluster.ConfigItems.skipper_wait_for_healthcheck_interval }}" + - "-wait-for-healthcheck-interval={{ .ConfigItems.skipper_wait_for_healthcheck_interval }}" {{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_skipper_eastwest "true")}} - "-enable-kubernetes-east-west" - "-kubernetes-east-west-domain=.ingress.cluster.local" @@ -187,7 +209,7 @@ spec: -opentracing=lightstep component-name=skipper-ingress token=$(LIGHTSTEP_TOKEN) - collector={{ .Cluster.ConfigItems.tracing_collector_host }}:8444 + collector={{ .ConfigItems.tracing_collector_host }}:8444 cmd-line=skipper-ingress tag=application=skipper-ingress tag=account={{ .Cluster.Alias }} @@ -201,7 +223,7 @@ spec: max-log-value-len={{ .ConfigItems.skipper_ingress_lightstep_max_log_value_len }} max-logs-per-span={{ .ConfigItems.skipper_ingress_lightstep_max_logs_per_span }} propagators={{ .ConfigItems.skipper_ingress_lightstep_propagators }} - {{ .Cluster.ConfigItems.skipper_ingress_lightstep_log_events }} + {{ .ConfigItems.skipper_ingress_lightstep_log_events }} - "-opentracing-excluded-proxy-tags={{ .ConfigItems.skipper_ingress_opentracing_excluded_proxy_tags }}" {{ if eq .ConfigItems.skipper_ingress_opentracing_backend_name_tag "true" }} - "-opentracing-backend-name-tag" @@ -271,10 +293,15 @@ spec: {{ range $pattern := split .ConfigItems.skipper_ingress_refuse_payload "[cf724afc]" }} - "-refuse-payload={{ $pattern }}" {{ end }} +{{ end }} +{{ if index . "args" }} + {{ range $arg := split .args "[cf724afc]" }} + - "{{ $arg }}" + {{ end }} {{ end }} resources: -{{ if and (eq .Cluster.ConfigItems.enable_dedicate_nodepool_skipper "true") (eq .Cluster.ConfigItems.skipper_ingress_binpack "true") }} -{{ $cpu_requests := sumQuantities .Cluster.ConfigItems.skipper_ingress_cpu (printf "-%s" .Cluster.ConfigItems.teapot_admission_controller_daemonset_reserved_cpu) (printf "-%s" .Cluster.ConfigItems.kubelet_system_reserved_cpu) (printf "-%s" .Cluster.ConfigItems.kubelet_kube_reserved_cpu) }} +{{ if and (eq .ConfigItems.enable_dedicate_nodepool_skipper "true") (eq .ConfigItems.skipper_ingress_binpack "true") }} +{{ $cpu_requests := sumQuantities .ConfigItems.skipper_ingress_cpu (printf "-%s" .ConfigItems.teapot_admission_controller_daemonset_reserved_cpu) (printf "-%s" .ConfigItems.kubelet_system_reserved_cpu) (printf "-%s" .ConfigItems.kubelet_kube_reserved_cpu) }} limits: cpu: "{{ $cpu_requests }}" memory: "{{ .ConfigItems.skipper_ingress_memory }}" @@ -397,6 +424,7 @@ spec: key: dedicated value: skipper-ingress {{ end }} +{{ end }} {{ if ne .ConfigItems.skipper_routesrv_enabled "false" }} --- @@ -429,7 +457,7 @@ spec: config/hash: {{"secret.yaml" | manifestHash}} kubernetes-log-watcher/scalyr-parser: | [{"container": "routesrv", "parser": "keyValue"}] - logging/destination: "{{.Cluster.ConfigItems.log_destination_local}}" + logging/destination: "{{ .ConfigItems.log_destination_local }}" prometheus.io/path: /metrics prometheus.io/port: "9990" prometheus.io/scrape: "true" @@ -448,9 +476,9 @@ spec: # kube-system in our admitters, since we've never really had any issues with them. parent-resource-hash: abd943226b6885f66785592be28bdf303863fbac {{- end }} - priorityClassName: "{{ .Cluster.ConfigItems.system_priority_class }}" + priorityClassName: "{{ .ConfigItems.system_priority_class }}" serviceAccountName: skipper-ingress - terminationGracePeriodSeconds: {{ .Cluster.ConfigItems.skipper_termination_grace_period }} + terminationGracePeriodSeconds: {{ .ConfigItems.skipper_termination_grace_period }} {{- if eq .ConfigItems.skipper_routesrv_node_affinity_enabled "true" }} affinity: podAntiAffinity: @@ -485,7 +513,7 @@ spec: - "-kubernetes-disable-catchall-routes={{ .ConfigItems.skipper_ingress_disable_catchall_routes }}" - "-enable-kubernetes-endpointslices={{ .ConfigItems.skipper_endpointslices_enabled }}" - "-address=:9990" - - "-wait-for-healthcheck-interval={{ .Cluster.ConfigItems.skipper_wait_for_healthcheck_interval }}" + - "-wait-for-healthcheck-interval={{ .ConfigItems.skipper_wait_for_healthcheck_interval }}" {{ if eq .ConfigItems.enable_skipper_eastwest "true"}} - "-enable-kubernetes-east-west" - "-kubernetes-east-west-domain=.ingress.cluster.local" @@ -517,7 +545,7 @@ spec: -opentracing=lightstep component-name=routesrv token=$(LIGHTSTEP_TOKEN) - collector={{ .Cluster.ConfigItems.tracing_collector_host }}:8444 + collector={{ .ConfigItems.tracing_collector_host }}:8444 cmd-line=routesrv tag=application=skipper-ingress tag=component=routesrv @@ -532,7 +560,7 @@ spec: max-log-value-len={{ .ConfigItems.skipper_ingress_lightstep_max_log_value_len }} max-logs-per-span={{ .ConfigItems.skipper_ingress_lightstep_max_logs_per_span }} propagators={{ .ConfigItems.skipper_ingress_lightstep_propagators }} - {{ .Cluster.ConfigItems.skipper_ingress_lightstep_log_events }} + {{ .ConfigItems.skipper_ingress_lightstep_log_events }} env: - name: LIGHTSTEP_TOKEN valueFrom: