merge main

.pre-commit-config.yaml

@@ -37,7 +37,7 @@ repos:
         exclude: "^src/_nebari/template/"
 
   - repo: https://github.com/crate-ci/typos
-    rev: v1.27.0
+    rev: typos-dict-v0.11.37
     hooks:
       - id: typos
 
@@ -61,7 +61,7 @@ repos:
         args: ["--line-length=88", "--exclude=/src/_nebari/template/"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.2
+    rev: v0.8.1
     hooks:
       - id: ruff
         args: ["--fix"]

RELEASE.md

@@ -9,6 +9,23 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ---
 
+## Release 2024.11.1 - November 21, 2024 (Hotfix Release)
+
+> NOTE: This hotfix addresses several major bugs identified in the 2024.9.1 release. For a detailed overview, please refer to the related discussion at #2798. Users should upgrade directly from 2024.7.1 to 2024.11.1.
+
+## What's Changed
+
+- fix `CHECK_URL` in kuberhealthy checks to respect namespaces by @dcmcand in https://github.com/nebari-dev/nebari/pull/2779
+- fix bug where `check_immutable_fields` throws error with old version of Nebari by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2796
+- Fix immutable field validation error when a sub-schema is not Pydantic by @kenafoster in https://github.com/nebari-dev/nebari/pull/2797
+- Address issue with AWS instance type schema by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2787
+- Add broken note by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2802
+- Refactor role creation for upgrade command path by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2795
+- Allow overriding of keycloak root credentials for 2024.11.1 upgrade path #2843
+- Disable AWS `launch_template` from nebari-config schema #2856
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.9.1...2024.11.1
+
 ## Release 2024.9.1 - September 27, 2024 (Broken Release)
 
 > WARNING: This release was later found to have unresolved issues described further in [issue 2798](https://github.com/nebari-dev/nebari/issues/2798). We have marked this release as broken on conda-forge and yanked it on PyPI. One of the bugs prevents any upgrade from 2024.9.1 to 2024.11.1. Users should skip this release entirely and upgrade directly from 2024.7.1 to 2024.11.1.

src/_nebari/constants.py

@@ -1,11 +1,8 @@
-CURRENT_RELEASE = "2024.9.1"
+CURRENT_RELEASE = "2024.11.1"
 
 HELM_VERSION = "v3.15.3"
 KUSTOMIZE_VERSION = "5.4.3"
-# NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes
-# implemented in August 2023.
-# https://www.hashicorp.com/license-faq
-TERRAFORM_VERSION = "1.5.7"
+OPENTOFU_VERSION = "1.8.3"
 
 KUBERHEALTHY_HELM_VERSION = "100"
 
@@ -25,7 +22,6 @@
 
 
 # DOCS
-DO_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-do"
 AZURE_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-azure"
 AWS_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-aws"
 GCP_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-gcp"
@@ -34,4 +30,3 @@
 AWS_DEFAULT_REGION = "us-east-1"
 AZURE_DEFAULT_REGION = "Central US"
 GCP_DEFAULT_REGION = "us-central1"
-DO_DEFAULT_REGION = "nyc3"

src/_nebari/initialize.py

@@ -11,18 +11,12 @@
 from _nebari import constants
 from _nebari.provider import git
 from _nebari.provider.cicd import github
-from _nebari.provider.cloud import (
-    amazon_web_services,
-    azure_cloud,
-    digital_ocean,
-    google_cloud,
-)
+from _nebari.provider.cloud import amazon_web_services, azure_cloud, google_cloud
 from _nebari.provider.oauth.auth0 import create_client
 from _nebari.stages.bootstrap import CiEnum
 from _nebari.stages.infrastructure import (
     DEFAULT_AWS_NODE_GROUPS,
     DEFAULT_AZURE_NODE_GROUPS,
-    DEFAULT_DO_NODE_GROUPS,
     DEFAULT_GCP_NODE_GROUPS,
     node_groups_to_dict,
 )
@@ -117,22 +111,7 @@ def render_config(
                 ),
             }
 
-    if cloud_provider == ProviderEnum.do:
-        do_region = region or constants.DO_DEFAULT_REGION
-        do_kubernetes_versions = kubernetes_version or get_latest_kubernetes_version(
-            digital_ocean.kubernetes_versions()
-        )
-        config["digital_ocean"] = {
-            "kubernetes_version": do_kubernetes_versions,
-            "region": do_region,
-            "node_groups": node_groups_to_dict(DEFAULT_DO_NODE_GROUPS),
-        }
-
-        config["theme"]["jupyterhub"][
-            "hub_subtitle"
-        ] = f"{WELCOME_HEADER_TEXT} on Digital Ocean"
-
-    elif cloud_provider == ProviderEnum.gcp:
+    if cloud_provider == ProviderEnum.gcp:
         gcp_region = region or constants.GCP_DEFAULT_REGION
         gcp_kubernetes_version = kubernetes_version or get_latest_kubernetes_version(
             google_cloud.kubernetes_versions(gcp_region)
@@ -245,16 +224,7 @@ def github_auto_provision(config: pydantic.BaseModel, owner: str, repo: str):
 
     try:
         # Secrets
-        if config.provider == ProviderEnum.do:
-            for name in {
-                "AWS_ACCESS_KEY_ID",
-                "AWS_SECRET_ACCESS_KEY",
-                "SPACES_ACCESS_KEY_ID",
-                "SPACES_SECRET_ACCESS_KEY",
-                "DIGITALOCEAN_TOKEN",
-            }:
-                github.update_secret(owner, repo, name, os.environ[name])
-        elif config.provider == ProviderEnum.aws:
+        if config.provider == ProviderEnum.aws:
             for name in {
                 "AWS_ACCESS_KEY_ID",
                 "AWS_SECRET_ACCESS_KEY",

src/_nebari/provider/cicd/github.py

@@ -117,12 +117,6 @@ def gha_env_vars(config: schema.Main):
         env_vars["ARM_CLIENT_SECRET"] = "${{ secrets.ARM_CLIENT_SECRET }}"
         env_vars["ARM_SUBSCRIPTION_ID"] = "${{ secrets.ARM_SUBSCRIPTION_ID }}"
         env_vars["ARM_TENANT_ID"] = "${{ secrets.ARM_TENANT_ID }}"
-    elif config.provider == schema.ProviderEnum.do:
-        env_vars["AWS_ACCESS_KEY_ID"] = "${{ secrets.AWS_ACCESS_KEY_ID }}"
-        env_vars["AWS_SECRET_ACCESS_KEY"] = "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
-        env_vars["SPACES_ACCESS_KEY_ID"] = "${{ secrets.SPACES_ACCESS_KEY_ID }}"
-        env_vars["SPACES_SECRET_ACCESS_KEY"] = "${{ secrets.SPACES_SECRET_ACCESS_KEY }}"
-        env_vars["DIGITALOCEAN_TOKEN"] = "${{ secrets.DIGITALOCEAN_TOKEN }}"
     elif config.provider == schema.ProviderEnum.gcp:
         env_vars["GOOGLE_CREDENTIALS"] = "${{ secrets.GOOGLE_CREDENTIALS }}"
         env_vars["PROJECT_ID"] = "${{ secrets.PROJECT_ID }}"

src/_nebari/provider/cloud/amazon_web_services.py

@@ -24,25 +24,19 @@ def check_credentials() -> None:
 
 @functools.lru_cache()
 def aws_session(
-    region: Optional[str] = None, digitalocean_region: Optional[str] = None
+    region: Optional[str] = None,
 ) -> boto3.Session:
     """Create a boto3 session."""
-    if digitalocean_region:
-        aws_access_key_id = os.environ["SPACES_ACCESS_KEY_ID"]
-        aws_secret_access_key = os.environ["SPACES_SECRET_ACCESS_KEY"]
-        region = digitalocean_region
-        aws_session_token = None
-    else:
-        check_credentials()
-        aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
-        aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
-        aws_session_token = os.environ.get("AWS_SESSION_TOKEN")
-
-        if not region:
-            raise ValueError(
-                "Please specify `region` in the nebari-config.yaml or if initializing the nebari-config, set the region via the "
-                "`--region` flag or via the AWS_DEFAULT_REGION environment variable.\n"
-            )
+    check_credentials()
+    aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
+    aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
+    aws_session_token = os.environ.get("AWS_SESSION_TOKEN")
+
+    if not region:
+        raise ValueError(
+            "Please specify `region` in the nebari-config.yaml or if initializing the nebari-config, set the region via the "
+            "`--region` flag or via the AWS_DEFAULT_REGION environment variable.\n"
+        )
 
     return boto3.Session(
         region_name=region,
@@ -712,21 +706,17 @@ def aws_delete_s3_objects(
     bucket_name: str,
     endpoint: Optional[str] = None,
     region: Optional[str] = None,
-    digitalocean_region: Optional[str] = None,
 ):
     """
     Delete all objects in the S3 bucket.
 
-    NOTE: This method is shared with Digital Ocean as their "Spaces" is S3 compatible and uses the same API.
-
     Parameters:
         bucket_name (str): S3 bucket name
-        endpoint (str): S3 endpoint URL (required for Digital Ocean spaces)
+        endpoint (str): S3 endpoint URL
         region (str): AWS region
-        digitalocean_region (str): Digital Ocean region
 
     """
-    session = aws_session(region=region, digitalocean_region=digitalocean_region)
+    session = aws_session(region=region)
     s3 = session.client("s3", endpoint_url=endpoint)
 
     try:
@@ -779,22 +769,18 @@ def aws_delete_s3_bucket(
     bucket_name: str,
     endpoint: Optional[str] = None,
     region: Optional[str] = None,
-    digitalocean_region: Optional[str] = None,
 ):
     """
     Delete S3 bucket.
 
-    NOTE: This method is shared with Digital Ocean as their "Spaces" is S3 compatible and uses the same API.
-
     Parameters:
         bucket_name (str): S3 bucket name
-        endpoint (str): S3 endpoint URL (required for Digital Ocean spaces)
+        endpoint (str): S3 endpoint URL
         region (str): AWS region
-        digitalocean_region (str): Digital Ocean region
     """
-    aws_delete_s3_objects(bucket_name, endpoint, region, digitalocean_region)
+    aws_delete_s3_objects(bucket_name, endpoint, region)
 
-    session = aws_session(region=region, digitalocean_region=digitalocean_region)
+    session = aws_session(region=region)
     s3 = session.client("s3", endpoint_url=endpoint)
 
     try:

src/_nebari/provider/cloud/google_cloud.py

@@ -51,6 +51,24 @@ def regions() -> Set[str]:
     return {region.name for region in response}
 
 
+@functools.lru_cache()
+def instances(region: str) -> set[str]:
+    """Return a set of available compute instances in a region."""
+    credentials, project_id = load_credentials()
+    zones_client = compute_v1.services.region_zones.RegionZonesClient(
+        credentials=credentials
+    )
+    instances_client = compute_v1.MachineTypesClient(credentials=credentials)
+    zone_list = zones_client.list(project=project_id, region=region)
+    zones = [zone for zone in zone_list]
+    instance_set: set[str] = set()
+    for zone in zones:
+        instance_list = instances_client.list(project=project_id, zone=zone.name)
+        for instance in instance_list:
+            instance_set.add(instance.name)
+    return instance_set
+
+
 @functools.lru_cache()
 def kubernetes_versions(region: str) -> List[str]:
     """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest."""