From d6986307263603d435d747077576d679d9c308d2 Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:33:49 -0500 Subject: [PATCH 01/10] remove depracated code paths (#2349) --- src/_nebari/render.py | 3 +-- src/_nebari/utils.py | 12 ------------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index d46a66852f..daf2fa506d 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -8,7 +8,6 @@ from rich.table import Table from _nebari.deprecate import DEPRECATED_FILE_PATHS -from _nebari.utils import is_relative_to from nebari import hookspecs, schema @@ -89,7 +88,7 @@ def render_template( for path in deleted: abs_path = (output_directory / path).resolve() - if not is_relative_to(abs_path, output_directory): + if not abs_path.is_relative_to(output_directory): raise Exception( f"[ERROR] SHOULD NOT HAPPEN filename was about to be deleted but path={abs_path} is outside of output_directory" ) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 3378116a1d..3488c6f0e8 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -268,18 +268,6 @@ def random_secure_string( return "".join(secrets.choice(chars) for i in range(length)) -def is_relative_to(self: Path, other: Path, /) -> bool: - """Compatibility function to bring ``Path.is_relative_to`` to Python 3.8""" - if sys.version_info[:2] >= (3, 9): - return self.is_relative_to(other) - - try: - self.relative_to(other) - return True - except ValueError: - return False - - def set_do_environment(): os.environ["AWS_ACCESS_KEY_ID"] = os.environ["SPACES_ACCESS_KEY_ID"] os.environ["AWS_SECRET_ACCESS_KEY"] = os.environ["SPACES_SECRET_ACCESS_KEY"] From c4dc7d0781b4307feab810fd765cc14fc2731f89 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 20 Mar 2024 20:48:49 -0400 Subject: [PATCH 02/10] Create SECURITY.md --- SECURITY.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..b89ecfe58b --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,11 @@ +# Security Policy + +## Supported Versions + +We support only the latest version, and we use [CalVer](https://calver.org/) for versioning. + +You should feel comfortable upgrading if you're using our documented public APIs and pay attention to `DeprecationWarnings`. Whenever there is a need to break compatibility, it is announced in the [Changelog](./CHANGELOG.md) and will raise a `DeprecationWarning` before it's finally really broken. + +## Reporting a Vulnerability + +If you think you found a vulnerability, please report it at [nebari/security](https://github.com/nebari-dev/nebari/security). From 72ff6f96d869d69020b3b699d943ca8bab25066f Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Thu, 21 Mar 2024 08:36:10 -0400 Subject: [PATCH 03/10] Update SECURITY.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Krassowski <5832902+krassowski@users.noreply.github.com> --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index b89ecfe58b..0a637b1b67 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ We support only the latest version, and we use [CalVer](https://calver.org/) for versioning. -You should feel comfortable upgrading if you're using our documented public APIs and pay attention to `DeprecationWarnings`. Whenever there is a need to break compatibility, it is announced in the [Changelog](./CHANGELOG.md) and will raise a `DeprecationWarning` before it's finally really broken. +You should feel comfortable upgrading if you're using our documented public APIs and pay attention to `DeprecationWarnings`. Whenever there is a need to break compatibility, it is announced in the [Changelog](https://www.nebari.dev/docs/references/RELEASE) and will raise a `DeprecationWarning` before it's finally really broken. ## Reporting a Vulnerability From 811806a2fb3205fe1e575e0fd6128d57a47a92a7 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Thu, 21 Mar 2024 08:42:40 -0400 Subject: [PATCH 04/10] update security.md --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 0a637b1b67..76f80ef924 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,4 +8,4 @@ You should feel comfortable upgrading if you're using our documented public APIs ## Reporting a Vulnerability -If you think you found a vulnerability, please report it at [nebari/security](https://github.com/nebari-dev/nebari/security). +If you think you found a vulnerability, please report it at [nebari/security](https://github.com/nebari-dev/nebari/security/new). Please do not report security vulnerabilities on our public issue tracker. Exposing vulnerabilities publicly without giving maintainers a chance to release a fix puts users at risk. From 3b318a6bf934664343eae956fbcbe8a6cff62a62 Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:35:17 -0500 Subject: [PATCH 05/10] Set node affinity for more pods to ensure they run on general node pool (#2353) Co-authored-by: Scott Blair --- .../template/aws/modules/kubernetes/main.tf | 34 ++++++++++++++++--- .../template/aws/modules/network/main.tf | 1 - .../infrastructure/template/aws/versions.tf | 2 +- .../modules/cluster-autoscaler/main.tf | 18 ++++++++++ .../services/argo-workflows/main.tf | 17 ++++++++++ 5 files changed, 66 insertions(+), 6 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf index 848d1c0471..43e5538507 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf @@ -57,7 +57,6 @@ resource "aws_eks_node_group" "main" { ] tags = merge({ - # "kubernetes.io/cluster/${var.name}" = "shared" "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name propagate_at_launch = true }, var.tags) @@ -69,9 +68,36 @@ data "aws_eks_cluster_auth" "main" { resource "aws_eks_addon" "aws-ebs-csi-driver" { # required for Kubernetes v1.23+ on AWS - addon_name = "aws-ebs-csi-driver" - cluster_name = aws_eks_cluster.main.name - resolve_conflicts = "OVERWRITE" + addon_name = "aws-ebs-csi-driver" + cluster_name = aws_eks_cluster.main.name + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "OVERWRITE" + + configuration_values = jsonencode({ + controller = { + nodeSelector = { + "eks.amazonaws.com/nodegroup" = "general" + } + } + }) + + # Ensure cluster and node groups are created + depends_on = [ + aws_eks_cluster.main, + aws_eks_node_group.main, + ] +} + +resource "aws_eks_addon" "coredns" { + addon_name = "coredns" + cluster_name = aws_eks_cluster.main.name + + configuration_values = jsonencode({ + nodeSelector = { + "eks.amazonaws.com/nodegroup" = "general" + } + }) + # Ensure cluster and node groups are created depends_on = [ aws_eks_cluster.main, diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf index b3601a2a91..da42767976 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf @@ -3,7 +3,6 @@ resource "aws_vpc" "main" { enable_dns_support = true enable_dns_hostnames = true - enable_classiclink = false tags = merge({ Name = var.name }, var.tags, var.vpc_tags) } diff --git a/src/_nebari/stages/infrastructure/template/aws/versions.tf b/src/_nebari/stages/infrastructure/template/aws/versions.tf index 54fc973d6a..68c0faf27b 100644 --- a/src/_nebari/stages/infrastructure/template/aws/versions.tf +++ b/src/_nebari/stages/infrastructure/template/aws/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "3.73.0" + version = "5.33.0" } } required_version = ">= 1.0" diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf index 29f982c86a..c07edd70dd 100644 --- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf +++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf @@ -19,6 +19,24 @@ resource "helm_release" "autoscaler" { clusterName = var.cluster-name enabled = true } + + affinity = { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [ + { + matchExpressions = [ + { + key = "eks.amazonaws.com/nodegroup" + operator = "In" + values = ["general"] + } + ] + } + ] + } + } + } }) ], var.overrides) } diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf index 29f27da26a..92bcad6eae 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf @@ -557,6 +557,23 @@ resource "kubernetes_manifest" "deployment_admission_controller" { } }, ] + affinity = { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [ + { + matchExpressions = [ + { + key = var.node-group.key + operator = "In" + values = [var.node-group.value] + } + ] + } + ] + } + } + } } } } From 0ff7b7cab2bc53f1dce2bebd8a7df011636eab45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Fri, 22 Mar 2024 19:56:59 +0000 Subject: [PATCH 06/10] Deduplicate conda-store in JupyterLab main menu (#2347) --- .../files/jupyterlab/overrides.json | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json index fd6cafc624..02d6564455 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json @@ -6,7 +6,8 @@ "apiUrl": "/conda-store/", "authMethod": "cookie", "loginUrl": "/conda-store/login?next=", - "authToken": "" + "authToken": "", + "addMainMenuItem": false }, "@jupyterlab/apputils-extension:notification": { "checkForUpdates": false, @@ -50,13 +51,24 @@ "rank": 1000, "items": [ { - "command": "help:open", - "rank": 1, + "command": "nebari:run-first-enabled", "args": { - "url": "/conda-store", - "text": "Environment Management", - "newBrowserTab": true - } + "commands": [ + { + "id": "condastore:open", + "label": "Environment Management" + }, + { + "id": "help:open", + "args": { + "url": "/conda-store", + "text": "Environment Management", + "newBrowserTab": true + } + } + ] + }, + "rank": 1 }, { "command": "help:open", From 9b9b6bcac6801488bd167362b718df41ae8306c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Sat, 23 Mar 2024 07:08:08 +0000 Subject: [PATCH 07/10] Pass current namespace to argo via environment variable (#2317) Co-authored-by: Vinicius D. Cerutti <51954708+viniciusdc@users.noreply.github.com> --- .../services/jupyterhub/files/jupyterhub/03-profiles.py | 8 ++++++++ .../modules/kubernetes/services/jupyterhub/main.tf | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index 5c66ca72d5..ddbc3b6be2 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -455,6 +455,14 @@ def profile_conda_store_viewer_token(): } } }, + "CONDA_STORE_SERVICE_NAMESPACE": { + "valueFrom": { + "secretKeyRef": { + "name": "argo-workflows-conda-store-token", + "key": "conda-store-service-namespace", + } + } + }, } diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf index af690112f6..f99ef62eed 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf @@ -264,8 +264,9 @@ resource "kubernetes_secret" "argo-workflows-conda-store-token" { } data = { - "conda-store-api-token" = var.conda-store-argo-workflows-jupyter-scheduler-token - "conda-store-service-name" = var.conda-store-service-name + "conda-store-api-token" = var.conda-store-argo-workflows-jupyter-scheduler-token + "conda-store-service-name" = var.conda-store-service-name + "conda-store-service-namespace" = var.namespace } type = "Opaque" From ac9ecbcd40323d86c592de0b168487d66795cad9 Mon Sep 17 00:00:00 2001 From: kenafoster Date: Mon, 25 Mar 2024 14:16:24 -0400 Subject: [PATCH 08/10] PVC for Traefik Ingress (prevent LetsEncrypt throttling) (#2352) --- .../modules/kubernetes/ingress/main.tf | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index 08bb5b295d..217039f420 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -9,7 +9,7 @@ locals { "--entrypoints.minio.http.tls.certResolver=letsencrypt", "--certificatesresolvers.letsencrypt.acme.tlschallenge", "--certificatesresolvers.letsencrypt.acme.email=${var.acme-email}", - "--certificatesresolvers.letsencrypt.acme.storage=acme.json", + "--certificatesresolvers.letsencrypt.acme.storage=/mnt/acme-certificates/acme.json", "--certificatesresolvers.letsencrypt.acme.caserver=${var.acme-server}", ] self-signed = local.default_cert @@ -27,6 +27,22 @@ resource "kubernetes_service_account" "main" { } } +resource "kubernetes_persistent_volume_claim" "traefik_certs_pvc" { + metadata { + name = "traefik-ingress-certs" + namespace = var.namespace + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "5Gi" + } + } + } + wait_until_bound = false +} + resource "kubernetes_cluster_role" "main" { metadata { @@ -215,6 +231,10 @@ resource "kubernetes_deployment" "main" { image = "${var.traefik-image.image}:${var.traefik-image.tag}" name = var.name + volume_mount { + mount_path = "/mnt/acme-certificates" + name = "acme-certificates" + } security_context { capabilities { drop = ["ALL"] @@ -326,6 +346,12 @@ resource "kubernetes_deployment" "main" { success_threshold = 1 } } + volume { + name = "acme-certificates" + persistent_volume_claim { + claim_name = kubernetes_persistent_volume_claim.traefik_certs_pvc.metadata.0.name + } + } } } } From 3e1dfde9c56cfe6c151b36333713d6fbea30bf86 Mon Sep 17 00:00:00 2001 From: "Vinicius D. Cerutti" <51954708+viniciusdc@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:35:26 -0300 Subject: [PATCH 09/10] New upgrade step for 2024.3.3 (#2356) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- src/_nebari/upgrade.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index dcadc1a029..c60fcfa60e 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -727,6 +727,17 @@ def _version_specific_upgrade( return config +class Upgrade_2024_3_3(UpgradeStep): + version = "2024.3.3" + + def _version_specific_upgrade( + self, config, start_version, config_filename: Path, *args, **kwargs + ): + rich.print("Ready to upgrade to Nebari version [green]2024.3.3[/green].") + + return config + + __rounded_version__ = str(rounded_ver_parse(__version__)) # Manually-added upgrade steps must go above this line From bfa454852f50bfe4c0a91ae1d554eb2c4bb63ff5 Mon Sep 17 00:00:00 2001 From: "Vinicius D. Cerutti" <51954708+viniciusdc@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:43:46 -0300 Subject: [PATCH 10/10] Update release notes for 2024.3.3 (#2362) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- RELEASE.md | 21 +++++++++++++++++++++ src/_nebari/constants.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 2b249c6603..d8c9bd5b43 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,6 +9,27 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. --> --- +## Release 2024.3.3 - March 27, 2024 + +### What's Changed +* get default variable value when following a terraform variable by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2322 +* Upgrade Actions versions by @isumitjha in https://github.com/nebari-dev/nebari/pull/2291 +* Cleanup spawner logs by @krassowski in https://github.com/nebari-dev/nebari/pull/2328 +* Fix loki gateway url when deployed on non-dev namespace by @aktech in https://github.com/nebari-dev/nebari/pull/2327 +* Dmcandrew update ruamel.yaml by @dcmcand in https://github.com/nebari-dev/nebari/pull/2315 +* upgrade auth0-python version to ultimately resolve CVE-2024-26130 by @tylergraff in https://github.com/nebari-dev/nebari/pull/2314 +* remove deprecated code paths by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2349 +* Create SECURITY.md by @dcmcand in https://github.com/nebari-dev/nebari/pull/2354 +* Set node affinity for more pods to ensure they run on general node pool by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2353 +* Deduplicate conda-store in JupyterLab main menu by @krassowski in https://github.com/nebari-dev/nebari/pull/2347 +* Pass current namespace to argo via environment variable by @krassowski in https://github.com/nebari-dev/nebari/pull/2317 +* PVC for Traefik Ingress (prevent LetsEncrypt throttling) by @kenafoster in https://github.com/nebari-dev/nebari/pull/2352 + +### New Contributors +* @isumitjha made their first contribution in https://github.com/nebari-dev/nebari/pull/2291 +* @tylergraff made their first contribution in https://github.com/nebari-dev/nebari/pull/2314 + +**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.3.2...2024.3.3 ## Release 2024.3.2 - March 14, 2024 diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 1f4d9ef81e..46df430e78 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -1,4 +1,4 @@ -CURRENT_RELEASE = "2024.3.2" +CURRENT_RELEASE = "2024.3.3" # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes # implemented in August 2023.