Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Update Spark v4 addons #692

Merged
merged 7 commits into from
Oct 31, 2024
Merged

chore: Update Spark v4 addons #692

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e2baa10
update eks module and access entries
alanty Oct 30, 2024
7de1b22
pre-commit cleanup
alanty Oct 30, 2024
07b1e42
Merge branch 'main' into spark-updates-v4
alanty Oct 30, 2024
f02aa09
bump addon helm charts to latest
alanty Oct 31, 2024
3b3156d
include kubecost ecr images from upstream values
alanty Oct 31, 2024
4c6b5e6
Merge remote-tracking branch 'origin/main' into spark-updates-v4
alanty Oct 31, 2024
e5c4e6f
pre-commit cleanup
alanty Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions analytics/terraform/spark-k8s-operator/addons.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,7 @@ module "eks_data_addons" {
# Spark history server is required only when EMR Spark Operator is enabled
enable_spark_history_server = true
spark_history_server_helm_config = {
chart_version = "1.2.0"
values = [
<<-EOT
sparkHistoryOpts: "-Dspark.history.fs.logDirectory=s3a://${module.s3_bucket.s3_bucket_id}/${aws_s3_object.this.key}"
Expand All @@ -417,6 +418,7 @@ module "eks_data_addons" {
#---------------------------------------------------------------
enable_kubecost = true
kubecost_helm_config = {
chart_version = "2.3.3"
values = [templatefile("${path.module}/helm-values/kubecost-values.yaml", {})]
repository_username = data.aws_ecrpublic_authorization_token.token.user_name
repository_password = data.aws_ecrpublic_authorization_token.token.password
Expand Down Expand Up @@ -476,14 +478,16 @@ module "eks_blueprints_addons" {
#---------------------------------------
enable_metrics_server = true
metrics_server = {
values = [templatefile("${path.module}/helm-values/metrics-server-values.yaml", {})]
chart_version = "3.12.2"
values = [templatefile("${path.module}/helm-values/metrics-server-values.yaml", {})]
}

#---------------------------------------
# Cluster Autoscaler
#---------------------------------------
enable_cluster_autoscaler = true
cluster_autoscaler = {
chart_version = "9.43.1"
values = [templatefile("${path.module}/helm-values/cluster-autoscaler-values.yaml", {
aws_region = var.region,
eks_cluster_id = module.eks.cluster_name
Expand Down Expand Up @@ -511,7 +515,8 @@ module "eks_blueprints_addons" {
#---------------------------------------
enable_aws_cloudwatch_metrics = true
aws_cloudwatch_metrics = {
values = [templatefile("${path.module}/helm-values/aws-cloudwatch-metrics-values.yaml", {})]
chart_version = "0.0.11"
values = [templatefile("${path.module}/helm-values/aws-cloudwatch-metrics-values.yaml", {})]
}

#---------------------------------------
Expand All @@ -524,6 +529,7 @@ module "eks_blueprints_addons" {
retention_in_days = 30
}
aws_for_fluentbit = {
chart_version = "0.1.34"
s3_bucket_arns = [
module.s3_bucket.s3_bucket_arn,
"${module.s3_bucket.s3_bucket_arn}/*"
Expand All @@ -538,7 +544,7 @@ module "eks_blueprints_addons" {

enable_aws_load_balancer_controller = true
aws_load_balancer_controller = {
chart_version = "1.5.4"
chart_version = "1.9.2"
set = [{
name = "enableServiceMutatorWebhook"
value = "false"
Expand All @@ -547,7 +553,7 @@ module "eks_blueprints_addons" {

enable_ingress_nginx = true
ingress_nginx = {
version = "4.5.2"
version = "4.11.3"
values = [templatefile("${path.module}/helm-values/nginx-values.yaml", {})]
}

Expand All @@ -571,7 +577,7 @@ module "eks_blueprints_addons" {
amp_url = "https://aps-workspaces.${local.region}.amazonaws.com/workspaces/${aws_prometheus_workspace.amp[0].id}"
}) : templatefile("${path.module}/helm-values/kube-prometheus.yaml", {})
]
chart_version = "48.1.1"
chart_version = "65.5.1"
set_sensitive = [
{
name = "grafana.adminPassword"
Expand Down
24 changes: 22 additions & 2 deletions analytics/terraform/spark-k8s-operator/helm-values/kubecost-values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,13 @@ prometheus:
create: false
kubeStateMetrics:
enabled: false

#imageVersion: prod-1.96.0 # commented to use the latest
server:
image:
repository: public.ecr.aws/kubecost/prometheus
configmapReload:
prometheus:
image:
repository: public.ecr.aws/kubecost/prometheus-config-reloader

kubecostFrontend:
image: public.ecr.aws/kubecost/frontend
Expand All @@ -46,6 +51,18 @@ kubecostModel:
cpu: "500m"
memory: "512Mi"

forecasting:
fullImageName: public.ecr.aws/kubecost/kubecost-modeling:v0.1.18

networkCosts:
image:
repository: public.ecr.aws/kubecost/kubecost-network-costs
tag: v0.17.6

clusterController:
image:
repository: public.ecr.aws/kubecost/cluster-controller

# Set this to false if you're bringing your own service account.
#serviceAccount:
# create: false
Expand All @@ -60,3 +77,6 @@ persistentVolume:
enabled: true # Note that setting this to false means configurations will be wiped out on pod restart.
storageClass: gp3
# existingClaim: kubecost-cost-analyzer # a claim in the same namespace as kubecost

reporting:
productAnalytics: false
Loading