diff --git a/modules/karpenter/controller_iam.tf b/modules/karpenter/controller_iam.tf index 4f61a18..a389855 100644 --- a/modules/karpenter/controller_iam.tf +++ b/modules/karpenter/controller_iam.tf @@ -28,13 +28,19 @@ data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" { } } -resource "aws_iam_role_policy" "karpenter_controller" { - name = "KarpenterController" +resource "aws_iam_role_policy" "karpenter_controller_v1_alpha" { + count = var.v1alpha ? 1 : 0 + name = "KarpenterController-v1alpha" role = aws_iam_role.karpenter_controller.id - policy = data.aws_iam_policy_document.karpenter_controller.json + policy = data.aws_iam_policy_document.karpenter_controller_v1_alpha.json } -data "aws_iam_policy_document" "karpenter_controller" { +moved { + from = aws_iam_role_policy.karpenter_controller + to = aws_iam_role_policy.karpenter_controller_v1_alpha +} + +data "aws_iam_policy_document" "karpenter_controller_v1_alpha" { statement { sid = "AllowScopedEC2InstanceActions" effect = "Allow" @@ -282,3 +288,360 @@ data "aws_iam_policy_document" "karpenter_controller" { actions = ["eks:DescribeCluster"] } } + +resource "aws_iam_role_policy" "karpenter_controller_v1_beta" { + count = var.v1beta ? 1 : 0 + name = "KarpenterController-v1beta" + role = aws_iam_role.karpenter_controller.id + policy = data.aws_iam_policy_document.karpenter_controller_v1_beta.json +} + +data "aws_iam_policy_document" "karpenter_controller_v1_beta" { + statement { + sid = "AllowScopedEC2InstanceAccessActions" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}::image/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}::snapshot/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:security-group/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:subnet/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + ] + } + + statement { + sid = "AllowScopedEC2LaunchTemplateAccessActions" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedEC2InstanceActionsWithTags" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:fleet/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:volume/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:network-interface/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:spot-instances-request/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceCreationTagging" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:fleet/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:volume/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:network-interface/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:spot-instances-request/*", + ] + + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + + values = [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate", + ] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceTagging" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = ["arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*"] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = ["karpenter.sh/nodeclaim", "Name"] + } + } + + + statement { + sid = "AllowScopedDeletion" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + ] + + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowRegionalReadActions" + effect = "Allow" + resources = ["*"] + + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [data.aws_region.current.name] + } + } + + statement { + sid = "AllowSSMReadActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:ssm:${data.aws_region.current.name}::parameter/aws/service/*"] + actions = ["ssm:GetParameter"] + } + + statement { + sid = "AllowPricingReadActions" + effect = "Allow" + resources = ["*"] + actions = ["pricing:GetProducts"] + } + + statement { + sid = "AllowInterruptionQueueActions" + effect = "Allow" + resources = [aws_sqs_queue.karpenter_interruption.arn] + + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + ] + } + + statement { + sid = "AllowPassingInstanceRole" + effect = "Allow" + resources = concat([aws_iam_role.karpenter_node.arn], var.additional_node_role_arns) + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ec2.amazonaws.com"] + } + } + + statement { + sid = "AllowScopedInstanceProfileCreationActions" + effect = "Allow" + resources = ["*"] + actions = ["iam:CreateInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileTagActions" + effect = "Allow" + resources = ["*"] + actions = ["iam:TagInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileActions" + effect = "Allow" + resources = ["*"] + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowInstanceProfileReadActions" + effect = "Allow" + resources = ["*"] + actions = ["iam:GetInstanceProfile"] + } + + statement { + sid = "AllowAPIServerEndpointDiscovery" + effect = "Allow" + resources = [var.cluster_config.arn] + actions = ["eks:DescribeCluster"] + } +} diff --git a/modules/karpenter/interruption_queue.tf b/modules/karpenter/interruption_queue.tf index bfd4bad..3b90610 100644 --- a/modules/karpenter/interruption_queue.tf +++ b/modules/karpenter/interruption_queue.tf @@ -22,6 +22,18 @@ data "aws_iam_policy_document" "karpenter_interruption_queue_policy" { ] } } + + statement { + sid = "DenyHTTP" + effect = "Deny" + actions = ["sqs:*"] + resources = [aws_sqs_queue.karpenter_interruption.arn] + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = [false] + } + } } locals { diff --git a/modules/karpenter/node_iam.tf b/modules/karpenter/node_iam.tf index a1e1de4..e55d460 100644 --- a/modules/karpenter/node_iam.tf +++ b/modules/karpenter/node_iam.tf @@ -19,10 +19,10 @@ data "aws_iam_policy_document" "karpenter_node_assume_role_policy" { resource "aws_iam_role_policy_attachment" "karpenter_node_managed_policies" { for_each = toset([ - "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", - "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", - "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEKS_CNI_Policy", + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore", ]) role = aws_iam_role.karpenter_node.id diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 9b55d28..df8c81a 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -17,6 +17,18 @@ variable "oidc_config" { }) } +variable "v1alpha" { + description = "Enable controller policy for v1alpha resources (Karpenter <= 0.32.*)" + type = bool + default = true +} + +variable "v1beta" { + description = "Enable controller policy for v1beta resources (Karpenter >= 0.32.*)" + type = bool + default = true +} + variable "additional_node_role_arns" { description = <<-EOF Additional Node Role ARNS that karpenter should manage diff --git a/test/cluster_test.go b/test/cluster_test.go index 7704ee0..7b33b23 100644 --- a/test/cluster_test.go +++ b/test/cluster_test.go @@ -75,62 +75,68 @@ func installKarpenter(t *testing.T, kubeconfig, clusterName, sgName string) { helmOptions := helm.Options{ KubectlOptions: kubectlOptions, ExtraArgs: map[string][]string{ - "upgrade": []string{"--create-namespace", "--version", "v0.31.0", "--force"}, + "upgrade": []string{"--create-namespace", "--version", "0.37.0", "--force"}, }, } helm.Upgrade(t, &helmOptions, "oci://public.ecr.aws/karpenter/karpenter-crd", "karpenter-crd") helmOptions = helm.Options{ SetValues: map[string]string{ "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn": "arn:aws:iam::214219211678:role/Karpenter-" + clusterName, - "settings.aws.clusterName": clusterName, - "settings.aws.defaultInstanceProfile": "KarpenterNode-" + clusterName, - "settings.aws.interruptionQueueName": "Karpenter-" + clusterName, - "controller.resources.requests.cpu": "1", - "controller.resources.requests.memory": "1Gi", - "controller.resources.limits.cpu": "1", - "controller.resources.limits.memory": "1Gi", + "settings.clusterName": clusterName, + "settings.interruptionQueueName": "Karpenter-" + clusterName, + "controller.resources.requests.cpu": "1", + "controller.resources.requests.memory": "1Gi", + "controller.resources.limits.cpu": "1", + "controller.resources.limits.memory": "1Gi", }, KubectlOptions: kubectlOptions, ExtraArgs: map[string][]string{ - "upgrade": []string{"--create-namespace", "--version", "v0.31.0"}, + "upgrade": []string{"--create-namespace", "--version", "0.37.0"}, }, } helm.Upgrade(t, &helmOptions, "oci://public.ecr.aws/karpenter/karpenter", "karpenter") WaitUntilPodsAvailable(t, kubectlOptions, metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=karpenter"}, 2, 30, 6*time.Second) - provisionerManifest := fmt.Sprintf(KARPENTER_PROVISIONER, sgName) + provisionerManifest := fmt.Sprintf(KARPENTER_PROVISIONER, sgName, clusterName) k8s.KubectlApplyFromString(t, kubectlOptions, provisionerManifest) } const KARPENTER_PROVISIONER = `--- -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: default spec: - requirements: - - key: karpenter.k8s.aws/instance-family - operator: In - values: [t3] - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - - key: karpenter.k8s.aws/instance-size - operator: In - values: [small, medium, large] - providerRef: - name: default - ttlSecondsAfterEmpty: 30 + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + requirements: + - key: karpenter.k8s.aws/instance-family + operator: In + values: [t3] + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + - key: karpenter.k8s.aws/instance-size + operator: In + values: [small, medium, large] --- -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass metadata: name: default spec: amiFamily: Bottlerocket - subnetSelector: - Name: terraform-aws-eks-test-environment-private* - securityGroupSelector: - Name: %s + subnetSelectorTerms: + - tags: + Name: terraform-aws-eks-test-environment-private* + securityGroupSelectorTerms: + - tags: + Name: %s + instanceProfile: + KarpenterNode-%s ` func validateAdminRole(t *testing.T, kubeconfig string) {