Merge branch 'develop' into pydantic2

RELEASE.md

@@ -9,6 +9,27 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ---
 
+## Release 2024.3.3 - March 27, 2024
+
+### What's Changed
+* get default variable value when following a terraform variable by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2322
+* Upgrade Actions versions by @isumitjha in https://github.com/nebari-dev/nebari/pull/2291
+* Cleanup spawner logs by @krassowski in https://github.com/nebari-dev/nebari/pull/2328
+* Fix loki gateway url when deployed on non-dev namespace by @aktech in https://github.com/nebari-dev/nebari/pull/2327
+* Dmcandrew update ruamel.yaml by @dcmcand in https://github.com/nebari-dev/nebari/pull/2315
+* upgrade auth0-python version to ultimately resolve CVE-2024-26130 by @tylergraff in https://github.com/nebari-dev/nebari/pull/2314
+* remove deprecated code paths by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2349
+* Create SECURITY.md by @dcmcand in https://github.com/nebari-dev/nebari/pull/2354
+* Set node affinity for more pods to ensure they run on general node pool by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2353
+* Deduplicate conda-store in JupyterLab main menu by @krassowski in https://github.com/nebari-dev/nebari/pull/2347
+* Pass current namespace to argo via environment variable by @krassowski in https://github.com/nebari-dev/nebari/pull/2317
+* PVC for Traefik Ingress (prevent LetsEncrypt throttling) by @kenafoster in https://github.com/nebari-dev/nebari/pull/2352
+
+### New Contributors
+* @isumitjha made their first contribution in https://github.com/nebari-dev/nebari/pull/2291
+* @tylergraff made their first contribution in https://github.com/nebari-dev/nebari/pull/2314
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.3.2...2024.3.3
 
 ## Release 2024.3.2 - March 14, 2024
 

SECURITY.md

@@ -0,0 +1,11 @@
+# Security Policy
+
+## Supported Versions
+
+We support only the latest version, and we use [CalVer](https://calver.org/) for versioning.
+
+You should feel comfortable upgrading if you're using our documented public APIs and pay attention to `DeprecationWarnings`. Whenever there is a need to break compatibility, it is announced in the [Changelog](https://www.nebari.dev/docs/references/RELEASE) and will raise a `DeprecationWarning` before it's finally really broken.
+
+## Reporting a Vulnerability
+
+If you think you found a vulnerability, please report it at [nebari/security](https://github.com/nebari-dev/nebari/security/new). Please do not report security vulnerabilities on our public issue tracker. Exposing vulnerabilities publicly without giving maintainers a chance to release a fix puts users at risk.

src/_nebari/constants.py

@@ -1,4 +1,4 @@
-CURRENT_RELEASE = "2024.3.2"
+CURRENT_RELEASE = "2024.3.3"
 
 # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes
 # implemented in August 2023.

src/_nebari/render.py

@@ -8,7 +8,6 @@
 from rich.table import Table
 
 from _nebari.deprecate import DEPRECATED_FILE_PATHS
-from _nebari.utils import is_relative_to
 from nebari import hookspecs, schema
 
 
@@ -89,7 +88,7 @@ def render_template(
         for path in deleted:
             abs_path = (output_directory / path).resolve()
 
-            if not is_relative_to(abs_path, output_directory):
+            if not abs_path.is_relative_to(output_directory):
                 raise Exception(
                     f"[ERROR] SHOULD NOT HAPPEN filename was about to be deleted but path={abs_path} is outside of output_directory"
                 )

src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf

@@ -57,7 +57,6 @@ resource "aws_eks_node_group" "main" {
   ]
 
   tags = merge({
-    #    "kubernetes.io/cluster/${var.name}"                       = "shared"
     "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
     propagate_at_launch                                       = true
   }, var.tags)
@@ -69,9 +68,36 @@ data "aws_eks_cluster_auth" "main" {
 
 resource "aws_eks_addon" "aws-ebs-csi-driver" {
   # required for Kubernetes v1.23+ on AWS
-  addon_name        = "aws-ebs-csi-driver"
-  cluster_name      = aws_eks_cluster.main.name
-  resolve_conflicts = "OVERWRITE"
+  addon_name                  = "aws-ebs-csi-driver"
+  cluster_name                = aws_eks_cluster.main.name
+  resolve_conflicts_on_create = "OVERWRITE"
+  resolve_conflicts_on_update = "OVERWRITE"
+
+  configuration_values = jsonencode({
+    controller = {
+      nodeSelector = {
+        "eks.amazonaws.com/nodegroup" = "general"
+      }
+    }
+  })
+
+  # Ensure cluster and node groups are created
+  depends_on = [
+    aws_eks_cluster.main,
+    aws_eks_node_group.main,
+  ]
+}
+
+resource "aws_eks_addon" "coredns" {
+  addon_name   = "coredns"
+  cluster_name = aws_eks_cluster.main.name
+
+  configuration_values = jsonencode({
+    nodeSelector = {
+      "eks.amazonaws.com/nodegroup" = "general"
+    }
+  })
+
   # Ensure cluster and node groups are created
   depends_on = [
     aws_eks_cluster.main,

src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf

@@ -3,7 +3,6 @@ resource "aws_vpc" "main" {
 
   enable_dns_support   = true
   enable_dns_hostnames = true
-  enable_classiclink   = false
 
   tags = merge({ Name = var.name }, var.tags, var.vpc_tags)
 }

src/_nebari/stages/infrastructure/template/aws/versions.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "3.73.0"
+      version = "5.33.0"
     }
   }
   required_version = ">= 1.0"

src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf

@@ -9,7 +9,7 @@ locals {
       "--entrypoints.minio.http.tls.certResolver=letsencrypt",
       "--certificatesresolvers.letsencrypt.acme.tlschallenge",
       "--certificatesresolvers.letsencrypt.acme.email=${var.acme-email}",
-      "--certificatesresolvers.letsencrypt.acme.storage=acme.json",
+      "--certificatesresolvers.letsencrypt.acme.storage=/mnt/acme-certificates/acme.json",
       "--certificatesresolvers.letsencrypt.acme.caserver=${var.acme-server}",
     ]
     self-signed = local.default_cert
@@ -27,6 +27,22 @@ resource "kubernetes_service_account" "main" {
   }
 }
 
+resource "kubernetes_persistent_volume_claim" "traefik_certs_pvc" {
+  metadata {
+    name      = "traefik-ingress-certs"
+    namespace = var.namespace
+  }
+  spec {
+    access_modes = ["ReadWriteOnce"]
+    resources {
+      requests = {
+        storage = "5Gi"
+      }
+    }
+  }
+  wait_until_bound = false
+}
+
 
 resource "kubernetes_cluster_role" "main" {
   metadata {
@@ -215,6 +231,10 @@ resource "kubernetes_deployment" "main" {
           image = "${var.traefik-image.image}:${var.traefik-image.tag}"
           name  = var.name
 
+          volume_mount {
+            mount_path = "/mnt/acme-certificates"
+            name       = "acme-certificates"
+          }
           security_context {
             capabilities {
               drop = ["ALL"]
@@ -326,6 +346,12 @@ resource "kubernetes_deployment" "main" {
             success_threshold     = 1
           }
         }
+        volume {
+          name = "acme-certificates"
+          persistent_volume_claim {
+            claim_name = kubernetes_persistent_volume_claim.traefik_certs_pvc.metadata.0.name
+          }
+        }
       }
     }
   }

src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf

@@ -19,6 +19,24 @@ resource "helm_release" "autoscaler" {
         clusterName = var.cluster-name
         enabled     = true
       }
+
+      affinity = {
+        nodeAffinity = {
+          requiredDuringSchedulingIgnoredDuringExecution = {
+            nodeSelectorTerms = [
+              {
+                matchExpressions = [
+                  {
+                    key      = "eks.amazonaws.com/nodegroup"
+                    operator = "In"
+                    values   = ["general"]
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      }
     })
   ], var.overrides)
 }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf

@@ -557,6 +557,23 @@ resource "kubernetes_manifest" "deployment_admission_controller" {
               }
             },
           ]
+          affinity = {
+            nodeAffinity = {
+              requiredDuringSchedulingIgnoredDuringExecution = {
+                nodeSelectorTerms = [
+                  {
+                    matchExpressions = [
+                      {
+                        key      = var.node-group.key
+                        operator = "In"
+                        values   = [var.node-group.value]
+                      }
+                    ]
+                  }
+                ]
+              }
+            }
+          }
         }
       }
     }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py

@@ -455,6 +455,14 @@ def profile_conda_store_viewer_token():
                 }
             }
         },
+        "CONDA_STORE_SERVICE_NAMESPACE": {
+            "valueFrom": {
+                "secretKeyRef": {
+                    "name": "argo-workflows-conda-store-token",
+                    "key": "conda-store-service-namespace",
+                }
+            }
+        },
     }
 
 

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json

@@ -6,7 +6,8 @@
         "apiUrl": "/conda-store/",
         "authMethod": "cookie",
         "loginUrl": "/conda-store/login?next=",
-        "authToken": ""
+        "authToken": "",
+        "addMainMenuItem": false
     },
     "@jupyterlab/apputils-extension:notification": {
         "checkForUpdates": false,
@@ -50,13 +51,24 @@
                 "rank": 1000,
                 "items": [
                     {
-                        "command": "help:open",
-                        "rank": 1,
+                        "command": "nebari:run-first-enabled",
                         "args": {
-                            "url": "/conda-store",
-                            "text": "Environment Management",
-                            "newBrowserTab": true
-                        }
+                            "commands": [
+                                {
+                                  "id": "condastore:open",
+                                  "label": "Environment Management"
+                                },
+                                {
+                                  "id": "help:open",
+                                  "args": {
+                                    "url": "/conda-store",
+                                    "text": "Environment Management",
+                                    "newBrowserTab": true
+                                  }
+                                }
+                            ]
+                        },
+                        "rank": 1
                     },
                     {
                         "command": "help:open",

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf

@@ -264,8 +264,9 @@ resource "kubernetes_secret" "argo-workflows-conda-store-token" {
   }
 
   data = {
-    "conda-store-api-token"    = var.conda-store-argo-workflows-jupyter-scheduler-token
-    "conda-store-service-name" = var.conda-store-service-name
+    "conda-store-api-token"         = var.conda-store-argo-workflows-jupyter-scheduler-token
+    "conda-store-service-name"      = var.conda-store-service-name
+    "conda-store-service-namespace" = var.namespace
   }
 
   type = "Opaque"

src/_nebari/upgrade.py

@@ -727,6 +727,17 @@ def _version_specific_upgrade(
         return config
 
 
+class Upgrade_2024_3_3(UpgradeStep):
+    version = "2024.3.3"
+
+    def _version_specific_upgrade(
+        self, config, start_version, config_filename: Path, *args, **kwargs
+    ):
+        rich.print("Ready to upgrade to Nebari version [green]2024.3.3[/green].")
+
+        return config
+
+
 __rounded_version__ = str(rounded_ver_parse(__version__))
 
 # Manually-added upgrade steps must go above this line

src/_nebari/utils.py

@@ -268,18 +268,6 @@ def random_secure_string(
     return "".join(secrets.choice(chars) for i in range(length))
 
 
-def is_relative_to(self: Path, other: Path, /) -> bool:
-    """Compatibility function to bring ``Path.is_relative_to`` to Python 3.8"""
-    if sys.version_info[:2] >= (3, 9):
-        return self.is_relative_to(other)
-
-    try:
-        self.relative_to(other)
-        return True
-    except ValueError:
-        return False
-
-
 def set_do_environment():
     os.environ["AWS_ACCESS_KEY_ID"] = os.environ["SPACES_ACCESS_KEY_ID"]
     os.environ["AWS_SECRET_ACCESS_KEY"] = os.environ["SPACES_SECRET_ACCESS_KEY"]