From ac47f305429ec18b026b24398f8dd1c72dcebeac Mon Sep 17 00:00:00 2001 From: Alan Tyson Date: Thu, 10 Oct 2024 12:04:24 -0700 Subject: [PATCH 1/6] update karpenter chart to 1.0.0 --- analytics/terraform/spark-k8s-operator/addons.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/terraform/spark-k8s-operator/addons.tf b/analytics/terraform/spark-k8s-operator/addons.tf index 4689005d9..40524dd7b 100644 --- a/analytics/terraform/spark-k8s-operator/addons.tf +++ b/analytics/terraform/spark-k8s-operator/addons.tf @@ -417,7 +417,7 @@ module "eks_blueprints_addons" { } } karpenter = { - chart_version = "v0.34.0" + chart_version = "1.0.0" repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password } From c86ee2b6cd33ef459bcea1e4e868056c1469b831 Mon Sep 17 00:00:00 2001 From: Alan Tyson Date: Thu, 10 Oct 2024 13:11:36 -0700 Subject: [PATCH 2/6] bump karpenter to 1.0.0 --- analytics/terraform/spark-eks-ipv6/addons.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/terraform/spark-eks-ipv6/addons.tf b/analytics/terraform/spark-eks-ipv6/addons.tf index 46451e56a..ade6101af 100644 --- a/analytics/terraform/spark-eks-ipv6/addons.tf +++ b/analytics/terraform/spark-eks-ipv6/addons.tf @@ -103,7 +103,7 @@ module "eks_blueprints_addons" { } } karpenter = { - chart_version = "0.37.0" + chart_version = "1.0.0" repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password } From 65512d9f15b900996d2c8a45c97b60198d1158d5 Mon Sep 17 00:00:00 2001 From: Alan Tyson Date: Thu, 10 Oct 2024 13:11:49 -0700 Subject: [PATCH 3/6] fix install script --- analytics/terraform/spark-eks-ipv6/install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/analytics/terraform/spark-eks-ipv6/install.sh b/analytics/terraform/spark-eks-ipv6/install.sh index 40df43ba4..18f2a94d3 100755 --- a/analytics/terraform/spark-eks-ipv6/install.sh +++ b/analytics/terraform/spark-eks-ipv6/install.sh @@ -1,7 +1,6 @@ #!/bin/bash read -p "Enter the region: " region -read -p " export AWS_DEFAULT_REGION=$region # List of Terraform modules to apply in sequence From 3725b8865db4aa2d674aac9121f700d82664a7ed Mon Sep 17 00:00:00 2001 From: Alan Tyson Date: Fri, 11 Oct 2024 10:56:24 -0700 Subject: [PATCH 4/6] bump data addons to match karpenter v1 minimum --- analytics/terraform/spark-k8s-operator/addons.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/terraform/spark-k8s-operator/addons.tf b/analytics/terraform/spark-k8s-operator/addons.tf index 40524dd7b..48cd59b97 100644 --- a/analytics/terraform/spark-k8s-operator/addons.tf +++ b/analytics/terraform/spark-k8s-operator/addons.tf @@ -3,7 +3,7 @@ #--------------------------------------------------------------- module "eks_data_addons" { source = "aws-ia/eks-data-addons/aws" - version = "~> 1.30" # ensure to update this to the latest/desired version + version = "~> 1.34" # ensure to update this to the latest/desired version oidc_provider_arn = module.eks.oidc_provider_arn From c9a85c306f6de33cd451d43bd2f76288636597f0 Mon Sep 17 00:00:00 2001 From: Alan Tyson Date: Fri, 11 Oct 2024 15:19:35 -0700 Subject: [PATCH 5/6] ping data addons version to 1.34.0, karpenter to 1.0.6 --- analytics/terraform/spark-eks-ipv6/addons.tf | 4 ++-- analytics/terraform/spark-k8s-operator/addons.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/analytics/terraform/spark-eks-ipv6/addons.tf b/analytics/terraform/spark-eks-ipv6/addons.tf index ade6101af..f3ea0c19a 100644 --- a/analytics/terraform/spark-eks-ipv6/addons.tf +++ b/analytics/terraform/spark-eks-ipv6/addons.tf @@ -103,7 +103,7 @@ module "eks_blueprints_addons" { } } karpenter = { - chart_version = "1.0.0" + chart_version = "1.0.6" repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password } @@ -166,7 +166,7 @@ module "eks_blueprints_addons" { #--------------------------------------------------------------- module "eks_data_addons" { source = "aws-ia/eks-data-addons/aws" - version = "~> 1.30" # ensure to update this to the latest/desired version + version = "1.34.0" # ensure to update this to the latest/desired version oidc_provider_arn = module.eks.oidc_provider_arn diff --git a/analytics/terraform/spark-k8s-operator/addons.tf b/analytics/terraform/spark-k8s-operator/addons.tf index 48cd59b97..659acafc6 100644 --- a/analytics/terraform/spark-k8s-operator/addons.tf +++ b/analytics/terraform/spark-k8s-operator/addons.tf @@ -3,7 +3,7 @@ #--------------------------------------------------------------- module "eks_data_addons" { source = "aws-ia/eks-data-addons/aws" - version = "~> 1.34" # ensure to update this to the latest/desired version + version = "1.34.0" # ensure to update this to the latest/desired version oidc_provider_arn = module.eks.oidc_provider_arn @@ -417,7 +417,7 @@ module "eks_blueprints_addons" { } } karpenter = { - chart_version = "1.0.0" + chart_version = "1.0.6" repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password } From 36bebbc068776c949343b8abcca16e82c3b14ed2 Mon Sep 17 00:00:00 2001 From: Alan Tyson Date: Wed, 16 Oct 2024 10:26:07 -0700 Subject: [PATCH 6/6] pre-commit cleanup --- analytics/terraform/spark-eks-ipv6/README.md | 2 +- analytics/terraform/spark-eks-ipv6/spark-team.tf | 2 +- analytics/terraform/spark-k8s-operator/README.md | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/analytics/terraform/spark-eks-ipv6/README.md b/analytics/terraform/spark-eks-ipv6/README.md index d2e2f9ac3..e289c2886 100644 --- a/analytics/terraform/spark-eks-ipv6/README.md +++ b/analytics/terraform/spark-eks-ipv6/README.md @@ -27,7 +27,7 @@ | [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.34 | | [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.15 | | [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.2 | -| [eks\_data\_addons](#module\_eks\_data\_addons) | aws-ia/eks-data-addons/aws | 1.33.0 | +| [eks\_data\_addons](#module\_eks\_data\_addons) | aws-ia/eks-data-addons/aws | 1.34.0 | | [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.0 | | [spark\_team\_a\_irsa](#module\_spark\_team\_a\_irsa) | aws-ia/eks-blueprints-addon/aws | ~> 1.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | diff --git a/analytics/terraform/spark-eks-ipv6/spark-team.tf b/analytics/terraform/spark-eks-ipv6/spark-team.tf index beeddc9ba..b4c0bab3c 100644 --- a/analytics/terraform/spark-eks-ipv6/spark-team.tf +++ b/analytics/terraform/spark-eks-ipv6/spark-team.tf @@ -66,7 +66,7 @@ module "spark_team_a_irsa" { #--------------------------------------------------------------- resource "aws_iam_policy" "spark" { description = "IAM role policy for Spark Job execution" - name_prefix = "${local.name}-spark-irsa" + name_prefix = "${local.name}-spark-irsa" policy = data.aws_iam_policy_document.spark_operator.json } diff --git a/analytics/terraform/spark-k8s-operator/README.md b/analytics/terraform/spark-k8s-operator/README.md index 305425918..a47ab6857 100644 --- a/analytics/terraform/spark-k8s-operator/README.md +++ b/analytics/terraform/spark-k8s-operator/README.md @@ -30,7 +30,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.34 | | [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.15 | | [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.2 | -| [eks\_data\_addons](#module\_eks\_data\_addons) | aws-ia/eks-data-addons/aws | 1.33.0 | +| [eks\_data\_addons](#module\_eks\_data\_addons) | aws-ia/eks-data-addons/aws | 1.34.0 | | [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.0 | | [spark\_team\_irsa](#module\_spark\_team\_irsa) | aws-ia/eks-blueprints-addon/aws | ~> 1.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | @@ -70,18 +70,18 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [aws\_auth\_roles](#input\_aws\_auth\_roles) | additional aws auth roles | 
list(
    object(
      {
        rolearn  = string
        username = string
        groups = list(string
        )
      }
    )
  )
| `[]` | no | +| [aws\_auth\_roles](#input\_aws\_auth\_roles) | additional aws auth roles | 
list(
    object(
      {
        rolearn  = string
        username = string
        groups = list(string
        )
      }
    )
  )
| `[]` | no | | [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.30"` | no | -| [eks\_data\_plane\_subnet\_secondary\_cidr](#input\_eks\_data\_plane\_subnet\_secondary\_cidr) | Secondary CIDR blocks. 32766 IPs per Subnet per Subnet/AZ for EKS Node and Pods | `list(string)` | 
[
  "100.64.0.0/17",
  "100.64.128.0/17"
]
| no | +| [eks\_data\_plane\_subnet\_secondary\_cidr](#input\_eks\_data\_plane\_subnet\_secondary\_cidr) | Secondary CIDR blocks. 32766 IPs per Subnet per Subnet/AZ for EKS Node and Pods | `list(string)` | 
[
  "100.64.0.0/17",
  "100.64.128.0/17"
]
| no | | [enable\_amazon\_prometheus](#input\_enable\_amazon\_prometheus) | Enable AWS Managed Prometheus service | `bool` | `true` | no | | [enable\_vpc\_endpoints](#input\_enable\_vpc\_endpoints) | Enable VPC Endpoints | `bool` | `false` | no | | [enable\_yunikorn](#input\_enable\_yunikorn) | Enable Apache YuniKorn Scheduler | `bool` | `true` | no | | [kms\_key\_admin\_roles](#input\_kms\_key\_admin\_roles) | list of role ARNs to add to the KMS policy | `list(string)` | `[]` | no | | [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"spark-operator-doeks"` | no | -| [private\_subnets](#input\_private\_subnets) | Private Subnets CIDRs. 254 IPs per Subnet/AZ for Private NAT + NLB + Airflow + EC2 Jumphost etc. | `list(string)` | 
[
  "10.1.1.0/24",
  "10.1.2.0/24"
]
| no | -| [public\_subnets](#input\_public\_subnets) | Public Subnets CIDRs. 62 IPs per Subnet/AZ | `list(string)` | 
[
  "10.1.0.0/26",
  "10.1.0.64/26"
]
| no | +| [private\_subnets](#input\_private\_subnets) | Private Subnets CIDRs. 254 IPs per Subnet/AZ for Private NAT + NLB + Airflow + EC2 Jumphost etc. | `list(string)` | 
[
  "10.1.1.0/24",
  "10.1.2.0/24"
]
| no | +| [public\_subnets](#input\_public\_subnets) | Public Subnets CIDRs. 62 IPs per Subnet/AZ | `list(string)` | 
[
  "10.1.0.0/26",
  "10.1.0.64/26"
]
| no | | [region](#input\_region) | Region | `string` | `"us-west-2"` | no | -| [secondary\_cidr\_blocks](#input\_secondary\_cidr\_blocks) | Secondary CIDR blocks to be attached to VPC | `list(string)` | 
[
  "100.64.0.0/16"
]
| no | +| [secondary\_cidr\_blocks](#input\_secondary\_cidr\_blocks) | Secondary CIDR blocks to be attached to VPC | `list(string)` | 
[
  "100.64.0.0/16"
]
| no | | [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR. This should be a valid private (RFC 1918) CIDR range | `string` | `"10.1.0.0/16"` | no | ## Outputs