Merge branch 'nebari-dev:develop' into 2238-cert-manager

.github/workflows/test-provider.yaml

@@ -67,7 +67,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: "3.11"
 
       - name: Retrieve secret from Vault
         uses: hashicorp/[email protected]

.github/workflows/test.yaml

@@ -32,10 +32,9 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.8"
-          - "3.9"
           - "3.10"
           - "3.11"
+          - "3.12"
       fail-fast: false
     concurrency:
       group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ matrix.python-version }}

.github/workflows/test_conda_build.yaml

@@ -33,13 +33,13 @@ jobs:
         uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
-          python-version: 3.8
+          python-version: "3.10"
           channels: conda-forge
           activate-environment: nebari-dev
 
       - name: Install dependencies
         run: |
-          conda install build grayskull conda-build
+          conda install build grayskull conda-build conda-verify
 
       - name: Generate sdist
         run: |
@@ -52,3 +52,8 @@ jobs:
       - name: Build conda package
         run: |
           conda build nebari
+
+      - name: Test conda package
+        run: |
+          conda install --use-local nebari
+          nebari --version

.github/workflows/test_helm_charts.yaml

@@ -29,7 +29,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.11"
       - name: Install additional Python dependencies
         run: |
           pip install python-hcl2

.github/workflows/test_local_integration.yaml

@@ -62,7 +62,7 @@ jobs:
           CONDA: /home/runnerx/miniconda3
         with:
           auto-update-conda: true
-          python-version: 3.8
+          python-version: "3.11"
           miniconda-version: "latest"
 
       - name: Install Nebari and playwright

README.md

@@ -79,7 +79,7 @@ Amazon [AWS](https://aws.amazon.com/), [GCP](https://cloud.google.com/ "Google C
 
 - Operating System: Currently, Nebari supports development on macOS and Linux operating systems. Windows is NOT supported.
   However, we would welcome contributions that add and improve support for Windows.
-- You need Python >= 3.8 on your local machine or virtual environment to work on Nebari.
+- You need Python >= 3.10 on your local machine or virtual environment to work on Nebari.
 - Adopting virtual environments ([`conda`](https://docs.conda.io/en/latest/), [`pipenv`](https://github.com/pypa/pipenv) or
   [`venv`](https://docs.python.org/3/library/venv.html)) is also encouraged.
 

SECURITY.md

@@ -0,0 +1,11 @@
+# Security Policy
+
+## Supported Versions
+
+We support only the latest version, and we use [CalVer](https://calver.org/) for versioning.
+
+You should feel comfortable upgrading if you're using our documented public APIs and pay attention to `DeprecationWarnings`. Whenever there is a need to break compatibility, it is announced in the [Changelog](https://www.nebari.dev/docs/references/RELEASE) and will raise a `DeprecationWarning` before it's finally really broken.
+
+## Reporting a Vulnerability
+
+If you think you found a vulnerability, please report it at [nebari/security](https://github.com/nebari-dev/nebari/security/new). Please do not report security vulnerabilities on our public issue tracker. Exposing vulnerabilities publicly without giving maintainers a chance to release a fix puts users at risk.

pyproject.toml

@@ -23,7 +23,7 @@ name = "nebari"
 dynamic = ["version"]
 description = "A Jupyter and Dask-powered open source data science platform."
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 license = "BSD-3-Clause"
 authors = [
     { name = "Nebari development team", email = "[email protected]" },
@@ -42,8 +42,6 @@ classifiers = [
     "Intended Audience :: Developers",
     "Topic :: Software Development :: Build Tools",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Intended Audience :: Developers",
@@ -55,23 +53,23 @@ classifiers = [
 ]
 
 dependencies = [
-    "auth0-python==4.4.2",
+    "auth0-python==4.7.1",
     "azure-identity==1.12.0",
     "azure-mgmt-containerservice==26.0.0",
     "azure-mgmt-resource==23.0.1",
     "bcrypt==4.0.1",
-    "boto3==1.28.40",
+    "boto3==1.34.63",
     "cloudflare==2.11.7",
     "kubernetes==27.2.0",
     "pluggy==1.3.0",
     "prompt-toolkit==3.0.36",
     "pydantic==1.10.12",
     "pynacl==1.5.0",
-    "python-keycloak==3.3.0",
+    "python-keycloak>=3.9.0",
     "questionary==2.0.0",
     "requests-toolbelt==1.0.0",
     "rich==13.5.1",
-    "ruamel.yaml==0.17.32",
+    "ruamel.yaml==0.18.6",
     "typer==0.9.0",
     "packaging==23.2",
 ]

src/_nebari/render.py

@@ -8,7 +8,6 @@
 from rich.table import Table
 
 from _nebari.deprecate import DEPRECATED_FILE_PATHS
-from _nebari.utils import is_relative_to
 from nebari import hookspecs, schema
 
 
@@ -89,7 +88,7 @@ def render_template(
         for path in deleted:
             abs_path = (output_directory / path).resolve()
 
-            if not is_relative_to(abs_path, output_directory):
+            if not abs_path.is_relative_to(output_directory):
                 raise Exception(
                     f"[ERROR] SHOULD NOT HAPPEN filename was about to be deleted but path={abs_path} is outside of output_directory"
                 )

src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf

@@ -57,7 +57,6 @@ resource "aws_eks_node_group" "main" {
   ]
 
   tags = merge({
-    #    "kubernetes.io/cluster/${var.name}"                       = "shared"
     "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
     propagate_at_launch                                       = true
   }, var.tags)
@@ -69,9 +68,36 @@ data "aws_eks_cluster_auth" "main" {
 
 resource "aws_eks_addon" "aws-ebs-csi-driver" {
   # required for Kubernetes v1.23+ on AWS
-  addon_name        = "aws-ebs-csi-driver"
-  cluster_name      = aws_eks_cluster.main.name
-  resolve_conflicts = "OVERWRITE"
+  addon_name                  = "aws-ebs-csi-driver"
+  cluster_name                = aws_eks_cluster.main.name
+  resolve_conflicts_on_create = "OVERWRITE"
+  resolve_conflicts_on_update = "OVERWRITE"
+
+  configuration_values = jsonencode({
+    controller = {
+      nodeSelector = {
+        "eks.amazonaws.com/nodegroup" = "general"
+      }
+    }
+  })
+
+  # Ensure cluster and node groups are created
+  depends_on = [
+    aws_eks_cluster.main,
+    aws_eks_node_group.main,
+  ]
+}
+
+resource "aws_eks_addon" "coredns" {
+  addon_name   = "coredns"
+  cluster_name = aws_eks_cluster.main.name
+
+  configuration_values = jsonencode({
+    nodeSelector = {
+      "eks.amazonaws.com/nodegroup" = "general"
+    }
+  })
+
   # Ensure cluster and node groups are created
   depends_on = [
     aws_eks_cluster.main,

src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf

@@ -3,7 +3,6 @@ resource "aws_vpc" "main" {
 
   enable_dns_support   = true
   enable_dns_hostnames = true
-  enable_classiclink   = false
 
   tags = merge({ Name = var.name }, var.tags, var.vpc_tags)
 }

src/_nebari/stages/infrastructure/template/aws/versions.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "3.73.0"
+      version = "5.33.0"
     }
   }
   required_version = ">= 1.0"

src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf

@@ -19,6 +19,24 @@ resource "helm_release" "autoscaler" {
         clusterName = var.cluster-name
         enabled     = true
       }
+
+      affinity = {
+        nodeAffinity = {
+          requiredDuringSchedulingIgnoredDuringExecution = {
+            nodeSelectorTerms = [
+              {
+                matchExpressions = [
+                  {
+                    key      = "eks.amazonaws.com/nodegroup"
+                    operator = "In"
+                    values   = ["general"]
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      }
     })
   ], var.overrides)
 }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf

@@ -557,6 +557,23 @@ resource "kubernetes_manifest" "deployment_admission_controller" {
               }
             },
           ]
+          affinity = {
+            nodeAffinity = {
+              requiredDuringSchedulingIgnoredDuringExecution = {
+                nodeSelectorTerms = [
+                  {
+                    matchExpressions = [
+                      {
+                        key      = var.node-group.key
+                        operator = "In"
+                        values   = [var.node-group.value]
+                      }
+                    ]
+                  }
+                ]
+              }
+            }
+          }
         }
       }
     }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py

@@ -455,6 +455,14 @@ def profile_conda_store_viewer_token():
                 }
             }
         },
+        "CONDA_STORE_SERVICE_NAMESPACE": {
+            "valueFrom": {
+                "secretKeyRef": {
+                    "name": "argo-workflows-conda-store-token",
+                    "key": "conda-store-service-namespace",
+                }
+            }
+        },
     }
 
 

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json

@@ -6,7 +6,8 @@
         "apiUrl": "/conda-store/",
         "authMethod": "cookie",
         "loginUrl": "/conda-store/login?next=",
-        "authToken": ""
+        "authToken": "",
+        "addMainMenuItem": false
     },
     "@jupyterlab/apputils-extension:notification": {
         "checkForUpdates": false,
@@ -50,13 +51,24 @@
                 "rank": 1000,
                 "items": [
                     {
-                        "command": "help:open",
-                        "rank": 1,
+                        "command": "nebari:run-first-enabled",
                         "args": {
-                            "url": "/conda-store",
-                            "text": "Environment Management",
-                            "newBrowserTab": true
-                        }
+                            "commands": [
+                                {
+                                  "id": "condastore:open",
+                                  "label": "Environment Management"
+                                },
+                                {
+                                  "id": "help:open",
+                                  "args": {
+                                    "url": "/conda-store",
+                                    "text": "Environment Management",
+                                    "newBrowserTab": true
+                                  }
+                                }
+                            ]
+                        },
+                        "rank": 1
                     },
                     {
                         "command": "help:open",

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf

@@ -264,8 +264,9 @@ resource "kubernetes_secret" "argo-workflows-conda-store-token" {
   }
 
   data = {
-    "conda-store-api-token"    = var.conda-store-argo-workflows-jupyter-scheduler-token
-    "conda-store-service-name" = var.conda-store-service-name
+    "conda-store-api-token"         = var.conda-store-argo-workflows-jupyter-scheduler-token
+    "conda-store-service-name"      = var.conda-store-service-name
+    "conda-store-service-namespace" = var.namespace
   }
 
   type = "Opaque"

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/main.tf

@@ -181,6 +181,14 @@ resource "helm_release" "prometheus-grafana" {
           "${var.node-group.key}" = var.node-group.value
         }
 
+        additionalDataSources = [
+          {
+            name = "Loki"
+            type = "loki"
+            url  = "http://loki-gateway.${var.namespace}"
+          }
+        ]
+
         # Avoid using the default password, as that's a security risk
         adminPassword : random_password.grafana_admin_password.result
 

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml

@@ -1,7 +1 @@
 # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml
-
-grafana:
-  additionalDataSources:
-   - name: Loki
-     type: loki
-     url: http://loki-gateway.dev

src/_nebari/utils.py

@@ -268,18 +268,6 @@ def random_secure_string(
     return "".join(secrets.choice(chars) for i in range(length))
 
 
-def is_relative_to(self: Path, other: Path, /) -> bool:
-    """Compatibility function to bring ``Path.is_relative_to`` to Python 3.8"""
-    if sys.version_info[:2] >= (3, 9):
-        return self.is_relative_to(other)
-
-    try:
-        self.relative_to(other)
-        return True
-    except ValueError:
-        return False
-
-
 def set_do_environment():
     os.environ["AWS_ACCESS_KEY_ID"] = os.environ["SPACES_ACCESS_KEY_ID"]
     os.environ["AWS_SECRET_ACCESS_KEY"] = os.environ["SPACES_SECRET_ACCESS_KEY"]