From 3727effdac2a64c0153297cf0ea13020650a2118 Mon Sep 17 00:00:00 2001 From: Katyanna Moura Date: Thu, 11 Apr 2024 16:57:43 +0200 Subject: [PATCH 1/5] Enable Stack versioned PCS for Stackset Signed-off-by: Katyanna Moura --- cluster/config-defaults.yaml | 3 +++ .../stackset-controller/01-stack-crd.yaml | 20 +++++++++++++++++++ .../stackset-controller/01-stackset-crd.yaml | 20 +++++++++++++++++++ .../stackset-controller/deployment.yaml | 3 +++ .../manifests/stackset-controller/rbac.yaml | 13 ++++++++++++ 5 files changed, 59 insertions(+) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 33d7658bdc..9df0e89c8b 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -923,6 +923,9 @@ stackset_ingress_source_switch_ttl: "5m" # enable/disable inline configmap support for stackset stackset_inline_configmap_support_enabled: "false" +# enable/disable plstformCredentialsSet support for stackset +stackset_pcs_support_enabled: "false" + # Enable/Disable profiling for Kubernetes components enable_control_plane_profiling: "false" diff --git a/cluster/manifests/stackset-controller/01-stack-crd.yaml b/cluster/manifests/stackset-controller/01-stack-crd.yaml index 93fb46e637..38505fe841 100644 --- a/cluster/manifests/stackset-controller/01-stack-crd.yaml +++ b/cluster/manifests/stackset-controller/01-stack-crd.yaml @@ -370,6 +370,26 @@ spec: description: Name of the ConfigMap type: string type: object +{{ end }} +{{- if eq .Cluster.ConfigItems.stackset_pcs_support_enabled "true" }} + platformCredentialsSet: + description: PlatformCredentialsSet to be created and owned + by Stack + properties: + name: + type: string + tokens: + additionalProperties: + properties: + privileges: + items: + type: string + type: array + required: + - privileges + type: object + type: object + type: object {{ end }} configMapRef: description: ConfigMap to be owned by Stack diff --git a/cluster/manifests/stackset-controller/01-stackset-crd.yaml b/cluster/manifests/stackset-controller/01-stackset-crd.yaml index 09248a7d32..ed799f8123 100644 --- a/cluster/manifests/stackset-controller/01-stackset-crd.yaml +++ b/cluster/manifests/stackset-controller/01-stackset-crd.yaml @@ -617,6 +617,26 @@ spec: description: Name of the ConfigMap type: string type: object +{{ end }} +{{- if eq .Cluster.ConfigItems.stackset_pcs_support_enabled "true" }} + platformCredentialsSet: + description: PlatformCredentialsSet to be created and owned + by Stack + properties: + name: + type: string + tokens: + additionalProperties: + properties: + privileges: + items: + type: string + type: array + required: + - privileges + type: object + type: object + type: object {{ end }} configMapRef: description: ConfigMap to be owned by Stack diff --git a/cluster/manifests/stackset-controller/deployment.yaml b/cluster/manifests/stackset-controller/deployment.yaml index d5fc9bff1f..ee702e6a46 100644 --- a/cluster/manifests/stackset-controller/deployment.yaml +++ b/cluster/manifests/stackset-controller/deployment.yaml @@ -40,6 +40,9 @@ spec: - "--enable-configmap-support" - "--enable-secret-support" - "--enable-traffic-segments" +{{- if eq .Cluster.ConfigItems.stackset_pcs_support_enabled "true" }} + - "--enable-pcs-support" +{{- end }} {{if eq .Cluster.Environment "e2e"}} - "--sync-ingress-annotation=example.org/i-haz-synchronize" - "--sync-ingress-annotation=teapot.org/the-best" diff --git a/cluster/manifests/stackset-controller/rbac.yaml b/cluster/manifests/stackset-controller/rbac.yaml index 464a712b52..2f5b5ac121 100644 --- a/cluster/manifests/stackset-controller/rbac.yaml +++ b/cluster/manifests/stackset-controller/rbac.yaml @@ -66,6 +66,19 @@ rules: - patch - delete {{- end }} +{{- if eq .Cluster.ConfigItems.stackset_pcs_support_enabled "true" }} +- apiGroups: + - "zalando.org" + resources: + - platformcredentialssets + verbs: + - get + - list + - create + - update + - patch + - delete +{{- end }} - apiGroups: - "" resources: From a03a4baa7a174f7a43793a3258f08d6433dd863d Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Thu, 11 Apr 2024 17:30:06 +0200 Subject: [PATCH 2/5] Update to Karpenter v0.36.0 https://github.com/aws/karpenter-provider-aws/releases/tag/v0.36.0 Signed-off-by: Mikkel Oscar Lyderik Larsen --- .../manifests/z-karpenter/04-clusterrole.yaml | 12 +- .../z-karpenter/05-clusterrole-core.yaml | 2 +- .../07-karpenter.k8s.aws_ec2nodeclasses.yaml | 342 +++++++++++------- .../08-karpenter.sh_nodeclaims.yaml | 12 +- .../09-karpenter.sh_nodepools.yaml | 16 +- cluster/manifests/z-karpenter/deployment.yaml | 2 +- 6 files changed, 220 insertions(+), 166 deletions(-) diff --git a/cluster/manifests/z-karpenter/04-clusterrole.yaml b/cluster/manifests/z-karpenter/04-clusterrole.yaml index 1598b3a6bc..d252d81088 100644 --- a/cluster/manifests/z-karpenter/04-clusterrole.yaml +++ b/cluster/manifests/z-karpenter/04-clusterrole.yaml @@ -10,13 +10,13 @@ metadata: component: karpenter rules: # Read - - apiGroups: [ "karpenter.k8s.aws" ] - resources: [ "ec2nodeclasses" ] - verbs: [ "get", "list", "watch" ] + - apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses"] + verbs: ["get", "list", "watch"] # Write - - apiGroups: [ "karpenter.k8s.aws" ] - resources: [ "ec2nodeclasses", "ec2nodeclasses/status" ] - verbs: [ "patch", "update" ] + - apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses", "ec2nodeclasses/status"] + verbs: ["patch", "update"] --- # Source: karpenter/templates/04-clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/cluster/manifests/z-karpenter/05-clusterrole-core.yaml b/cluster/manifests/z-karpenter/05-clusterrole-core.yaml index ae078da2fe..b652f08d70 100644 --- a/cluster/manifests/z-karpenter/05-clusterrole-core.yaml +++ b/cluster/manifests/z-karpenter/05-clusterrole-core.yaml @@ -37,7 +37,7 @@ rules: verbs: ["create", "patch"] - apiGroups: [""] resources: ["nodes"] - verbs: ["create", "patch", "delete"] + verbs: ["patch", "delete"] - apiGroups: [""] resources: ["pods/eviction"] verbs: ["create"] diff --git a/cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml b/cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml index 9a75053b73..8a10f30502 100644 --- a/cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: ec2nodeclasses.karpenter.k8s.aws spec: group: karpenter.k8s.aws @@ -26,26 +26,32 @@ spec: description: EC2NodeClass is the Schema for the EC2NodeClass API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: EC2NodeClassSpec is the top level specification for the AWS - Karpenter Provider. This will contain configuration necessary to launch - instances in AWS. + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. properties: amiFamily: description: AMIFamily is the AMI family that instances use. enum: - AL2 + - AL2023 - Bottlerocket - Ubuntu - Custom @@ -56,28 +62,30 @@ spec: description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. items: - description: AMISelectorTerm defines selection logic for an ami - used by Karpenter to launch nodes. If multiple fields are used - for selection, the requirements are ANDed. + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: id: description: ID is the ami id in EC2 pattern: ami-[0-9a-z]+ type: string name: - description: Name is the ami name in EC2. This value is the - name field, which is different from the name tag. + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. type: string owner: - description: Owner is the owner for the ami. You can specify - a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string tags: additionalProperties: type: string - description: Tags is a map of key/value tags used to select - subnets Specifying '*' for a value selects all values for - a given tag key. + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -113,27 +121,38 @@ spec: volume is deleted on instance termination. type: boolean encrypted: - description: Encrypted indicates whether the EBS volume - is encrypted. Encrypted volumes can only be attached to - instances that support Amazon EBS encryption. If you are - creating a volume from a snapshot, you can't specify an - encryption value. + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. type: boolean iops: - description: "IOPS is the number of I/O operations per second - (IOPS). For gp3, io1, and io2 volumes, this represents - the number of IOPS that are provisioned for the volume. - For gp2 volumes, this represents the baseline performance - of the volume and the rate at which the volume accumulates - I/O credits for bursting. \n The following are the supported - values for each volume type: \n * gp3: 3,000-16,000 IOPS - \n * io1: 100-64,000 IOPS \n * io2: 100-64,000 IOPS \n - For io1 and io2 volumes, we guarantee 64,000 IOPS only - for Instances built on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 - IOPS. \n This parameter is supported for io1, io2, and - gp3 volumes only. This parameter is not supported for - gp2, st1, sc1, or standard volumes." + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + + The following are the supported values for each volume type: + + + * gp3: 3,000-16,000 IOPS + + + * io1: 100-64,000 IOPS + + + * io2: 100-64,000 IOPS + + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. format: int64 type: integer kmsKeyID: @@ -144,9 +163,9 @@ spec: description: SnapshotID is the ID of an EBS snapshot type: string throughput: - description: 'Throughput to provision for a gp3 volume, - with a maximum of 1,000 MiB/s. Valid Range: Minimum value - of 125. Maximum value of 1000.' + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. format: int64 type: integer volumeSize: @@ -156,15 +175,27 @@ spec: anyOf: - type: integer - type: string - description: "VolumeSize in `Gi`, `G`, `Ti`, or `T`. You - must specify either a snapshot ID or a volume size. The - following are the supported volumes sizes for each volume - type: \n * gp2 and gp3: 1-16,384 \n * io1 and io2: 4-16,384 - \n * st1 and sc1: 125-16,384 \n * standard: 1-1,024" + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + + * gp2 and gp3: 1-16,384 + + + * io1 and io2: 4-16,384 + + + * st1 and sc1: 125-16,384 + + + * standard: 1-1,024 x-kubernetes-int-or-string: true volumeType: - description: VolumeType of the block device. For more information, - see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) in the Amazon Elastic Compute Cloud User Guide. enum: - standard @@ -180,9 +211,9 @@ spec: - message: snapshotID or volumeSize must be defined rule: has(self.snapshotID) || has(self.volumeSize) rootVolume: - description: RootVolume is a flag indicating if this device - is mounted as kubelet root dir. You can configure at most - one root volume in BlockDeviceMappings. + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. type: boolean type: object maxItems: 50 @@ -192,98 +223,121 @@ spec: rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 context: - description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string detailedMonitoring: description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched type: boolean instanceProfile: - description: InstanceProfile is the AWS entity that instances use. - This field is mutually exclusive from role. The instance profile - should already have a role assigned to it that Karpenter has PassRole - permission on for instance launch using this instanceProfile to - succeed. + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. type: string x-kubernetes-validations: - message: instanceProfile cannot be empty rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store + disks. + enum: + - RAID0 + type: string metadataOptions: default: httpEndpoint: enabled httpProtocolIPv6: disabled httpPutResponseHopLimit: 2 httpTokens: required - description: "MetadataOptions for the generated launch template of - provisioned nodes. \n This specifies the exposure of the Instance - Metadata Service to provisioned EC2 nodes. For more information, - see Instance Metadata and User Data (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. \n Refer to recommended, - security best practices (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens required." + disabled, with httpPutResponseLimit of 2, and with httpTokens + required. properties: httpEndpoint: default: enabled - description: "HTTPEndpoint enables or disables the HTTP metadata - endpoint on provisioned nodes. If metadata options is non-nil, - but this parameter is not specified, the default state is \"enabled\". - \n If you specify a value of \"disabled\", instance metadata - will not be accessible on the node." + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. enum: - enabled - disabled type: string httpProtocolIPv6: default: disabled - description: HTTPProtocolIPv6 enables or disables the IPv6 endpoint - for the instance metadata service on provisioned nodes. If metadata - options is non-nil, but this parameter is not specified, the - default state is "disabled". + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". enum: - enabled - disabled type: string httpPutResponseHopLimit: default: 2 - description: HTTPPutResponseHopLimit is the desired HTTP PUT response - hop limit for instance metadata requests. The larger the number, - the further instance metadata requests can travel. Possible - values are integers from 1 to 64. If metadata options is non-nil, - but this parameter is not specified, the default value is 2. + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 2. format: int64 maximum: 64 minimum: 1 type: integer httpTokens: default: required - description: "HTTPTokens determines the state of token usage for - instance metadata requests. If metadata options is non-nil, - but this parameter is not specified, the default state is \"required\". - \n If the state is optional, one can choose to retrieve instance - metadata with or without a signed token header on the request. - If one retrieves the IAM role credentials without a token, the - version 1.0 role credentials are returned. If one retrieves - the IAM role credentials using a valid signed token, the version - 2.0 role credentials are returned. \n If the state is \"required\", - one must send a signed token header with any instance metadata - retrieval requests. In this state, retrieving the IAM role credentials - always returns the version 2.0 credentials; the version 1.0 - credentials are not available." + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. enum: - required - optional type: string type: object role: - description: Role is the AWS identity that nodes use. This field is - immutable. This field is mutually exclusive from instanceProfile. - Marking this field as immutable avoids concerns around terminating - managed instance profiles from running instances. This field may - be made mutable in the future, assuming the correct garbage collection - and drift handling is implemented for the old instance profiles - on an update. + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. type: string x-kubernetes-validations: - message: role cannot be empty @@ -294,24 +348,25 @@ spec: description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. items: - description: SecurityGroupSelectorTerm defines selection logic for - a security group used by Karpenter to launch nodes. If multiple - fields are used for selection, the requirements are ANDed. + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: id: description: ID is the security group id in EC2 pattern: sg-[0-9a-z]+ type: string name: - description: Name is the security group name in EC2. This value - is the name field, which is different from the name tag. + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. type: string tags: additionalProperties: type: string - description: Tags is a map of key/value tags used to select - subnets Specifying '*' for a value selects all values for - a given tag key. + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -335,9 +390,9 @@ spec: description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. items: - description: SubnetSelectorTerm defines selection logic for a subnet - used by Karpenter to launch nodes. If multiple fields are used - for selection, the requirements are ANDed. + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: id: description: ID is the subnet id in EC2 @@ -346,9 +401,9 @@ spec: tags: additionalProperties: type: string - description: Tags is a map of key/value tags used to select - subnets Specifying '*' for a value selects all values for - a given tag key. + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -376,17 +431,19 @@ spec: rule: self.all(k, k != '') - message: tag contains a restricted tag matching kubernetes.io/cluster/ rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) - - message: tag contains a restricted tag matching karpenter.sh/provisioner-name - rule: self.all(k, k != 'karpenter.sh/provisioner-name') - message: tag contains a restricted tag matching karpenter.sh/nodepool rule: self.all(k, k != 'karpenter.sh/nodepool') - message: tag contains a restricted tag matching karpenter.sh/managed-by rule: self.all(k, k !='karpenter.sh/managed-by') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') userData: - description: UserData to be applied to the provisioned nodes. It must - be in the appropriate format based on the AMIFamily in use. Karpenter - will merge certain fields into this UserData to ensure nodes are - being provisioned with the correct configuration. + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. type: string required: - amiFamily @@ -409,8 +466,9 @@ spec: description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass properties: amis: - description: AMI contains the current AMI values that are available - to the cluster under the AMI selectors. + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. items: description: AMI contains resolved AMI selector values utilized for node launch @@ -425,26 +483,32 @@ spec: description: Requirements of the AMI to be utilized on an instance type items: - description: A node selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. properties: key: description: The label key that the selector applies to. type: string + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer operator: - description: Represents a key's relationship to a set - of values. Valid operators are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator - is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. If the operator is Gt or Lt, the - values array must have a single element, which will - be interpreted as an integer. This array is replaced - during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -463,8 +527,9 @@ spec: for the role type: string securityGroups: - description: SecurityGroups contains the current Security Groups values - that are available to the cluster under the SecurityGroups selectors. + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. items: description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch @@ -480,8 +545,9 @@ spec: type: object type: array subnets: - description: Subnets contains the current Subnet values that are available - to the cluster under the subnet selectors. + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. items: description: Subnet contains resolved Subnet selector values utilized for node launch diff --git a/cluster/manifests/z-karpenter/08-karpenter.sh_nodeclaims.yaml b/cluster/manifests/z-karpenter/08-karpenter.sh_nodeclaims.yaml index bf72949365..a2c67e7286 100644 --- a/cluster/manifests/z-karpenter/08-karpenter.sh_nodeclaims.yaml +++ b/cluster/manifests/z-karpenter/08-karpenter.sh_nodeclaims.yaml @@ -141,11 +141,8 @@ spec: type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: KubeReserved contains resources reserved for Kubernetes system components. type: object x-kubernetes-validations: @@ -170,11 +167,8 @@ spec: type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object x-kubernetes-validations: @@ -227,7 +221,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml b/cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml index f7f6165f50..c272eb65b4 100644 --- a/cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml +++ b/cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml @@ -81,7 +81,7 @@ spec: This is required if Schedule is set. This regex has an optional 0s at the end since the duration.String() always adds a 0s at the end. - pattern: ^([0-9]+(m|h)+(0s)?)$ + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ type: string nodes: default: 10% @@ -191,7 +191,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) type: object spec: description: NodeClaimSpec describes the desired state of the NodeClaim @@ -267,11 +267,8 @@ spec: type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: KubeReserved contains resources reserved for Kubernetes system components. type: object x-kubernetes-validations: @@ -296,11 +293,8 @@ spec: type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object x-kubernetes-validations: @@ -355,7 +349,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/cluster/manifests/z-karpenter/deployment.yaml b/cluster/manifests/z-karpenter/deployment.yaml index c33adf846d..cf8a512bc9 100644 --- a/cluster/manifests/z-karpenter/deployment.yaml +++ b/cluster/manifests/z-karpenter/deployment.yaml @@ -50,7 +50,7 @@ spec: drop: - ALL readOnlyRootFilesystem: true - image: "container-registry.zalando.net/teapot/karpenter:0.35.4-main-20.custom" + image: "container-registry.zalando.net/teapot/karpenter:0.36.0-main-21.custom" imagePullPolicy: IfNotPresent env: - name: KUBERNETES_MIN_VERSION From 03f5ed5e5e470e365f5ce8ab47618990a717d113 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Fri, 12 Apr 2024 10:26:26 +0200 Subject: [PATCH 3/5] remove okta_auth_enabled config item because it's enabled by default --- cluster/config-defaults.yaml | 1 - cluster/manifests/deletions.yaml | 4 ---- cluster/manifests/roles/cluster-admin-binding.yaml | 2 -- cluster/manifests/roles/collaborator-roles.yaml | 2 -- cluster/manifests/roles/poweruser-binding.yaml | 2 -- cluster/manifests/roles/readonly-binding.yaml | 6 ------ cluster/node-pools/master-default/userdata.yaml | 2 -- 7 files changed, 19 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 33d7658bdc..2ff25a8440 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -926,7 +926,6 @@ stackset_inline_configmap_support_enabled: "false" # Enable/Disable profiling for Kubernetes components enable_control_plane_profiling: "false" -okta_auth_enabled: "true" okta_auth_issuer_url: "" okta_auth_client_id: "kubernetes.cluster.{{.Cluster.Alias}}" diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index e95567de34..13262e9356 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -41,10 +41,6 @@ post_apply: - name: spot-node-rescheduler kind: ClusterRoleBinding {{ end }} -{{- if ne .Cluster.ConfigItems.okta_auth_enabled "true" }} -- name: cluster-admin-okta - kind: ClusterRoleBinding -{{- end }} {{ if ne .Cluster.ConfigItems.skipper_ingress_canary_enabled "true" }} - name: skipper-ingress-canary diff --git a/cluster/manifests/roles/cluster-admin-binding.yaml b/cluster/manifests/roles/cluster-admin-binding.yaml index a81c2de9f9..cfd1e01f90 100644 --- a/cluster/manifests/roles/cluster-admin-binding.yaml +++ b/cluster/manifests/roles/cluster-admin-binding.yaml @@ -1,4 +1,3 @@ -{{- if eq .Cluster.ConfigItems.okta_auth_enabled "true" }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -11,4 +10,3 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: okta:common/administrator -{{- end }} diff --git a/cluster/manifests/roles/collaborator-roles.yaml b/cluster/manifests/roles/collaborator-roles.yaml index cb6d80c6d9..8355cc9f09 100644 --- a/cluster/manifests/roles/collaborator-roles.yaml +++ b/cluster/manifests/roles/collaborator-roles.yaml @@ -34,8 +34,6 @@ subjects: - kind: Group name: CollaboratorEmergency apiGroup: rbac.authorization.k8s.io -{{- if eq .Cluster.ConfigItems.okta_auth_enabled "true" }} - kind: Group name: "okta:common/collaborator" apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/cluster/manifests/roles/poweruser-binding.yaml b/cluster/manifests/roles/poweruser-binding.yaml index 5c9c398ff1..df276d4210 100644 --- a/cluster/manifests/roles/poweruser-binding.yaml +++ b/cluster/manifests/roles/poweruser-binding.yaml @@ -21,11 +21,9 @@ subjects: - kind: Group name: Emergency apiGroup: rbac.authorization.k8s.io -{{- if eq .Cluster.ConfigItems.okta_auth_enabled "true" }} - kind: Group name: "okta:common/engineer" apiGroup: rbac.authorization.k8s.io - kind: Group name: "okta:common/collaborator" apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/cluster/manifests/roles/readonly-binding.yaml b/cluster/manifests/roles/readonly-binding.yaml index 45ce8afc8d..708fcf8c93 100644 --- a/cluster/manifests/roles/readonly-binding.yaml +++ b/cluster/manifests/roles/readonly-binding.yaml @@ -10,7 +10,6 @@ subjects: - kind: Group name: ReadOnly apiGroup: rbac.authorization.k8s.io - # {{ if eq .Cluster.ConfigItems.okta_auth_enabled "true" }} - kind: Group name: "okta:common/engineer" apiGroup: rbac.authorization.k8s.io @@ -20,7 +19,6 @@ subjects: - kind: Group name: "okta:common/read-only" apiGroup: rbac.authorization.k8s.io - # {{ end }} --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -35,7 +33,6 @@ subjects: - kind: Group name: ReadOnly apiGroup: rbac.authorization.k8s.io - # {{ if eq .Cluster.ConfigItems.okta_auth_enabled "true" }} - kind: Group name: "okta:common/engineer" apiGroup: rbac.authorization.k8s.io @@ -45,7 +42,6 @@ subjects: - kind: Group name: "okta:common/read-only" apiGroup: rbac.authorization.k8s.io - # {{ end }} --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -60,7 +56,6 @@ subjects: - kind: Group name: ReadOnly apiGroup: rbac.authorization.k8s.io - # {{ if eq .Cluster.ConfigItems.okta_auth_enabled "true" }} - kind: Group name: "okta:common/engineer" apiGroup: rbac.authorization.k8s.io @@ -70,4 +65,3 @@ subjects: - kind: Group name: "okta:common/read-only" apiGroup: rbac.authorization.k8s.io - # {{ end }} diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index 401db2f992..60340f6340 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -135,14 +135,12 @@ write_files: - --authorization-webhook-config-file=/etc/kubernetes/config/authz.yaml - --authorization-webhook-version=v1 - --token-auth-file=/etc/kubernetes/config/tokenfile.csv -{{- if eq .Cluster.ConfigItems.okta_auth_enabled "true" }} - --oidc-issuer-url={{.Cluster.ConfigItems.okta_auth_issuer_url}} - --oidc-client-id={{.Cluster.ConfigItems.okta_auth_client_id}} - --oidc-username-claim=email - "--oidc-username-prefix=okta:" - --oidc-groups-claim=groups - "--oidc-groups-prefix=okta:" -{{- end }} - --feature-gates=HPAScaleToZero={{ .Cluster.ConfigItems.enable_hpa_scale_to_zero }},HPAContainerMetrics={{ .Cluster.ConfigItems.enable_hpa_container_metrics }},StatefulSetAutoDeletePVC={{ .Cluster.ConfigItems.enable_statefulset_autodelete_pvc }},TopologyAwareHints={{ .Cluster.ConfigItems.enable_topology_aware_hints }},MinDomainsInPodTopologySpread={{ .Cluster.ConfigItems.min_domains_in_pod_topology_spread_enabled }},CronJobTimeZone={{.Cluster.ConfigItems.cronjob_time_zone_enabled}},MaxUnavailableStatefulSet={{.Cluster.ConfigItems.max_unavailable_statefulset_enabled}} - --service-account-key-file=/etc/kubernetes/ssl/service-account-public-key.pem - --service-account-signing-key-file=/etc/kubernetes/ssl/service-account-private-key.pem From 09c4920046eafe4ecaf6f9baf48d3ee131d0476c Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Fri, 12 Apr 2024 10:52:51 +0200 Subject: [PATCH 4/5] Apply suggestions from code review --- cluster/config-defaults.yaml | 2 +- .../stackset-controller/01-stackset-crd.yaml | 32 +++++++++---------- .../stackset-controller/deployment.yaml | 3 -- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 9df0e89c8b..0b35ae0c0e 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -923,7 +923,7 @@ stackset_ingress_source_switch_ttl: "5m" # enable/disable inline configmap support for stackset stackset_inline_configmap_support_enabled: "false" -# enable/disable plstformCredentialsSet support for stackset +# enable/disable platformCredentialsSet support for stackset stackset_pcs_support_enabled: "false" # Enable/Disable profiling for Kubernetes components diff --git a/cluster/manifests/stackset-controller/01-stackset-crd.yaml b/cluster/manifests/stackset-controller/01-stackset-crd.yaml index ed799f8123..0295d1d9bf 100644 --- a/cluster/manifests/stackset-controller/01-stackset-crd.yaml +++ b/cluster/manifests/stackset-controller/01-stackset-crd.yaml @@ -620,23 +620,23 @@ spec: {{ end }} {{- if eq .Cluster.ConfigItems.stackset_pcs_support_enabled "true" }} platformCredentialsSet: - description: PlatformCredentialsSet to be created and owned - by Stack - properties: - name: - type: string - tokens: - additionalProperties: - properties: - privileges: - items: - type: string - type: array - required: - - privileges - type: object + description: PlatformCredentialsSet to be created and owned + by Stack + properties: + name: + type: string + tokens: + additionalProperties: + properties: + privileges: + items: + type: string + type: array + required: + - privileges type: object - type: object + type: object + type: object {{ end }} configMapRef: description: ConfigMap to be owned by Stack diff --git a/cluster/manifests/stackset-controller/deployment.yaml b/cluster/manifests/stackset-controller/deployment.yaml index ee702e6a46..d5fc9bff1f 100644 --- a/cluster/manifests/stackset-controller/deployment.yaml +++ b/cluster/manifests/stackset-controller/deployment.yaml @@ -40,9 +40,6 @@ spec: - "--enable-configmap-support" - "--enable-secret-support" - "--enable-traffic-segments" -{{- if eq .Cluster.ConfigItems.stackset_pcs_support_enabled "true" }} - - "--enable-pcs-support" -{{- end }} {{if eq .Cluster.Environment "e2e"}} - "--sync-ingress-annotation=example.org/i-haz-synchronize" - "--sync-ingress-annotation=teapot.org/the-best" From a2b46d7cc57f21f3ff32e3f12e5a2a1f48922808 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Fri, 12 Apr 2024 10:56:36 +0200 Subject: [PATCH 5/5] Let's just do the minimal thing for now. --- cluster/manifests/stackset-controller/rbac.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cluster/manifests/stackset-controller/rbac.yaml b/cluster/manifests/stackset-controller/rbac.yaml index 2f5b5ac121..464a712b52 100644 --- a/cluster/manifests/stackset-controller/rbac.yaml +++ b/cluster/manifests/stackset-controller/rbac.yaml @@ -66,19 +66,6 @@ rules: - patch - delete {{- end }} -{{- if eq .Cluster.ConfigItems.stackset_pcs_support_enabled "true" }} -- apiGroups: - - "zalando.org" - resources: - - platformcredentialssets - verbs: - - get - - list - - create - - update - - patch - - delete -{{- end }} - apiGroups: - "" resources: