diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index d940dda514..d866f55884 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -484,6 +484,9 @@ kubernetes_lifecycle_metrics_mem_min: "120Mi" kube_node_ready_controller_cpu: "50m" kube_node_ready_controller_memory: "200Mi" +# Enable kube-node-ready ASG lifecycle hook feature. +kube_node_ready_enabled: "true" + # Enable deployment of aws-cloud-controller-manager aws_cloud_controller_manager_enabled: "true" aws_cloud_controller_manager_cpu: "125m" @@ -1146,3 +1149,9 @@ control_plane_graceful_shutdown: "true" # fs.aio-max-nr = 8388608 # fs.inotify.max_user_watches = 100000 sysctl_settings: "" + + + +# scheduling_controls +teapot_admission_controller_scheduling_controls_enabled: "false" +teapot_admission_controller_scheduling_controls_default_architecture: "amd64" diff --git a/cluster/manifests/01-admission-control/config.yaml b/cluster/manifests/01-admission-control/config.yaml index fe203df9c2..b07b8ef4a0 100644 --- a/cluster/manifests/01-admission-control/config.yaml +++ b/cluster/manifests/01-admission-control/config.yaml @@ -52,6 +52,8 @@ data: {{- end}} pod.env-inject.node-options.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_inject_node_options_environment_variable }}" + pod.scheduling-controls.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_scheduling_controls_enabled }}" + pod.scheduling-controls.default-architecture: "{{ .Cluster.ConfigItems.teapot_admission_controller_scheduling_controls_default_architecture }}" podfactory.container-resource-check.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_validate_pod_template_resources }}" podfactory.application-label-check.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_validate_application_label }}" diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index eaad055f02..401057c9a8 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -309,3 +309,14 @@ post_apply: kind: DaemonSet namespace: kube-system {{- end }} +{{- if ne .Cluster.ConfigItems.kube_node_ready_enabled "true" }} +- name: kube-node-ready + kind: DaemonSet + namespace: kube-system +- name: kube-node-ready + kind: ServiceAccount + namespace: kube-system +- name: kube-node-ready + kind: Service + namespace: kube-system +{{- end }} diff --git a/cluster/manifests/kube-node-ready/01-rbac.yaml b/cluster/manifests/kube-node-ready/01-rbac.yaml index e38976084d..e6d2d09cab 100644 --- a/cluster/manifests/kube-node-ready/01-rbac.yaml +++ b/cluster/manifests/kube-node-ready/01-rbac.yaml @@ -1,3 +1,4 @@ +# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -5,3 +6,4 @@ metadata: namespace: kube-system annotations: iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-kube-node-ready" +# {{ end }} diff --git a/cluster/manifests/kube-node-ready/daemonset.yaml b/cluster/manifests/kube-node-ready/daemonset.yaml index fc7594203b..65f2e4d12b 100644 --- a/cluster/manifests/kube-node-ready/daemonset.yaml +++ b/cluster/manifests/kube-node-ready/daemonset.yaml @@ -1,3 +1,4 @@ +# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }} # {{ $image := "container-registry.zalando.net/teapot/kube-node-ready:master-34" }} # {{ $version := index (split $image ":") 1 }} @@ -65,3 +66,4 @@ spec: runAsUser: 1000 securityContext: fsGroup: 65534 +# {{ end }} diff --git a/cluster/manifests/kube-node-ready/service.yaml b/cluster/manifests/kube-node-ready/service.yaml index 27681e9759..5a9fb3a838 100644 --- a/cluster/manifests/kube-node-ready/service.yaml +++ b/cluster/manifests/kube-node-ready/service.yaml @@ -1,3 +1,4 @@ +# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }} kind: Service apiVersion: v1 metadata: @@ -16,3 +17,4 @@ spec: protocol: TCP selector: component: kube-node-ready +# {{ end }} diff --git a/cluster/manifests/kube2iam/daemonset.yaml b/cluster/manifests/kube2iam/daemonset.yaml index 0063770dde..9ac5efd6ea 100644 --- a/cluster/manifests/kube2iam/daemonset.yaml +++ b/cluster/manifests/kube2iam/daemonset.yaml @@ -38,13 +38,15 @@ spec: effect: NoExecute hostNetwork: true containers: - - image: container-registry.zalando.net/teapot/kube2iam:0.11.2-master-18.patched + - image: container-registry.zalando.net/teapot/kube2iam:0.12.0-master-19.patched name: kube2iam args: - --auto-discover-base-arn - --verbose - --node=$(NODE_NAME) env: + - name: AWS_DEFAULT_REGION + value: "{{.Cluster.Region}}" - name: NODE_NAME valueFrom: fieldRef: diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index c00e5472b3..dbefb69c38 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -1,6 +1,6 @@ {{/* image-updater-bot detects *image variables so use print to disable it for main image */}} -{{ $main_image := print "container-registry.zalando.net/teapot/skipper-internal:" "v0.21.216-1038" }} +{{ $main_image := print "container-registry.zalando.net/teapot/skipper-internal:" "v0.21.220-1043" }} {{ $canary_image := "container-registry.zalando.net/teapot/skipper-internal:v0.21.220-1043" }} diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index 044dfb88d8..1384d90f91 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -247,7 +247,7 @@ write_files: name: admission-controller-kubeconfig readOnly: true - name: skipper-admission-webhook - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222 + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223 args: - webhook - --address=:9085 @@ -424,7 +424,7 @@ write_files: value: {{ .Cluster.ConfigItems.apiserver_business_partner_ids }} {{ end }} - name: skipper-proxy - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222 + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223 args: - skipper - -access-log-strip-query @@ -475,7 +475,7 @@ write_files: name: ssl-certs-kubernetes readOnly: true - name: skipper-metrics - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222 + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223 args: - skipper - -access-log-strip-query diff --git a/cluster/node-pools/worker-combined/stack.yaml b/cluster/node-pools/worker-combined/stack.yaml index 632dbaf667..38a0d1e285 100644 --- a/cluster/node-pools/worker-combined/stack.yaml +++ b/cluster/node-pools/worker-combined/stack.yaml @@ -174,6 +174,7 @@ Resources: Roles: - !ImportValue '{{ .Cluster.ID }}:worker-iam-role' Type: 'AWS::IAM::InstanceProfile' +# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }} AutoscalingLifecycleHook: Properties: AutoScalingGroupName: !Ref AutoScalingGroup @@ -182,3 +183,4 @@ Resources: HeartbeatTimeout: '600' LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING' Type: 'AWS::AutoScaling::LifecycleHook' +# {{ end }} diff --git a/cluster/node-pools/worker-karpenter/provisioners.yaml b/cluster/node-pools/worker-karpenter/provisioners.yaml index b75565a805..9fe8f0513a 100644 --- a/cluster/node-pools/worker-karpenter/provisioners.yaml +++ b/cluster/node-pools/worker-karpenter/provisioners.yaml @@ -155,6 +155,18 @@ spec: - "c7in" - "m7in" - "r7in" +#{{ else if (gt (len .NodePool.InstanceTypes) 0) }} + - key: "node.kubernetes.io/instance-type" + operator: In + values: +# {{ range $type := .NodePool.InstanceTypes }} + - "{{ $type }}" +# {{ end }} +#{{ end }} + +# safety guards to prevent the use of unwanted instance types in case the user has not specified any specific instance types +#{{ if or (eq .NodePool.KarpenterInstanceTypeStrategy "default-for-karpenter") (eq .NodePool.KarpenterInstanceTypeStrategy "not-specified") }} + # exclude unwanted sizes - key: "karpenter.k8s.aws/instance-size" operator: "NotIn" values: @@ -166,14 +178,19 @@ spec: - "c5d.large" - "m5d.large" - "r5d.large" -#{{ else }} - - key: "node.kubernetes.io/instance-type" - operator: In +#{{end}} + +#{{ if (index .NodePool.ConfigItems "requirements") }} +# {{ range $requirement := .NodePool.KarpenterRequirements }} + - key: "{{ $requirement.Key }}" + operator: "{{ $requirement.Operator }}" values: -# {{ range $type := .NodePool.InstanceTypes }} - - "{{ $type }}" -# {{ end }} +# {{ range $value := $requirement.Values }} + - "{{ $value}}" +# {{ end }} +# {{ end }} #{{ end }} + # other configuration - key: "karpenter.sh/capacity-type" operator: In values: diff --git a/cluster/node-pools/worker-splitaz/stack.yaml b/cluster/node-pools/worker-splitaz/stack.yaml index 43a20bc1ce..10dafb3811 100644 --- a/cluster/node-pools/worker-splitaz/stack.yaml +++ b/cluster/node-pools/worker-splitaz/stack.yaml @@ -125,6 +125,7 @@ Resources: VPCZoneIdentifier: - "{{ index $data.Values.subnets $az }}" Type: 'AWS::AutoScaling::AutoScalingGroup' +# {{ if eq $data.Cluster.ConfigItems.kube_node_ready_enabled "true" }} AutoscalingLifecycleHook{{$azID}}: Properties: AutoScalingGroupName: !Ref AutoScalingGroup{{$azID}} @@ -133,6 +134,7 @@ Resources: HeartbeatTimeout: '600' LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING' Type: 'AWS::AutoScaling::LifecycleHook' +# {{ end }} {{ end }} {{ end }} {{ end }}