From a7003935d71a7a568bc97874cfa6fde1f6eb2b4a Mon Sep 17 00:00:00 2001
From: Johnny Bouder <jbouder@metrostar.com>
Date: Mon, 29 Jan 2024 09:14:10 -0500
Subject: [PATCH 01/32] Add favicon to jupyterhub theme.

---
 src/_nebari/stages/kubernetes_services/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index bde7163744..0e87b51800 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -53,6 +53,7 @@ class JupyterHubTheme(schema.Base):
     hub_subtitle: str = "Your open source data science platform"
     welcome: str = """Welcome! Learn about Nebari's features and configurations in <a href="https://www.nebari.dev/docs">the documentation</a>. If you have any questions or feedback, reach the team on <a href="https://www.nebari.dev/docs/community#getting-support">Nebari's support forums</a>."""
     logo: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg"
+    favicon: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/symbol/favicon.ico"
     primary_color: str = "#4f4173"
     primary_color_dark: str = "#4f4173"
     secondary_color: str = "#957da6"

From fa4e18e64643e833da9b9aaa439079819ca137ea Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 2 Feb 2024 10:03:29 -0500
Subject: [PATCH 02/32] Modify Playwright test to account for changes in
 JupyterLab UI.

---
 tests/common/navigator.py                     | 12 ++++--------
 tests/common/run_notebook.py                  |  2 +-
 tests/tests_e2e/playwright/test_playwright.py |  2 +-
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/tests/common/navigator.py b/tests/common/navigator.py
index dc2adc9eba..ff56c8c5c2 100644
--- a/tests/common/navigator.py
+++ b/tests/common/navigator.py
@@ -320,15 +320,13 @@ def _set_environment_via_popup(self, kernel=None):
             # failure here indicates that the environment doesn't exist either
             # because of incorrect naming syntax or because the env is still
             # being built
-            self.page.get_by_role("combobox").nth(1).select_option(
-                f'{{"name":"{kernel}"}}'
-            )
+            self.page.get_by_role("combobox").nth(1).select_option(kernel)
             # click Select to close popup (deal with the two formats of this dialog)
             try:
-                self.page.get_by_role("button", name="Select", exact=True).click()
+                self.page.get_by_role("button", name="Select Kernel").click()
             except Exception:
                 self.page.locator("div").filter(has_text="No KernelSelect").get_by_role(
-                    "button", name="Select"
+                    "button", name="Select Kernel"
                 ).click()
 
     def set_environment(self, kernel):
@@ -360,10 +358,8 @@ def set_environment(self, kernel):
         self._set_environment_via_popup(kernel)
 
         # wait for the jupyter UI to catch up before moving forward
-        # extract conda env name
-        conda_env_label = re.search("conda-env-(.*)-py", kernel).group(1)
         # see if the jupyter notebook label for the conda env is visible
-        kernel_label_loc = self.page.get_by_role("button", name=conda_env_label)
+        kernel_label_loc = self.page.get_by_role("button", name=kernel)
         if not kernel_label_loc.is_visible():
             kernel_label_loc.wait_for(state="attached")
 
diff --git a/tests/common/run_notebook.py b/tests/common/run_notebook.py
index 03c383299a..10d28d6637 100644
--- a/tests/common/run_notebook.py
+++ b/tests/common/run_notebook.py
@@ -220,7 +220,7 @@ def _restart_run_all(self):
         # Restart dialog appears most, but not all of the time (e.g. set
         # No Kernel, then Restart Run All)
         restart_dialog_button = self.nav.page.get_by_role(
-            "button", name="Restart", exact=True
+            "button", name="Confirm Kernel Restart"
         )
         if restart_dialog_button.is_visible():
             restart_dialog_button.click()
diff --git a/tests/tests_e2e/playwright/test_playwright.py b/tests/tests_e2e/playwright/test_playwright.py
index 7f4dabac08..903af3f0dd 100644
--- a/tests/tests_e2e/playwright/test_playwright.py
+++ b/tests/tests_e2e/playwright/test_playwright.py
@@ -13,6 +13,6 @@ def test_notebook(navigator, test_data_root):
     test_app.run(
         path=notebook_name,
         expected_outputs=["success: 6"],
-        conda_env="conda-env-default-py",
+        conda_env="default *",
         timeout=500,
     )

From a605a248ad15ea2a33dbf9d54a5efda3409b7fe2 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 2 Feb 2024 12:53:16 -0500
Subject: [PATCH 03/32] Add header with xrsf token as API complains when not
 passed.

---
 tests/tests_deployment/utils.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/tests/tests_deployment/utils.py b/tests/tests_deployment/utils.py
index 327de53309..b2b2a707b6 100644
--- a/tests/tests_deployment/utils.py
+++ b/tests/tests_deployment/utils.py
@@ -28,16 +28,17 @@ def get_jupyterhub_session():
 
 def get_jupyterhub_token(note="jupyterhub-tests-deployment"):
     session = get_jupyterhub_session()
+    xsrf_token = session.cookies.get("_xsrf")
+    headers = {"Referer": f"https://{constants.NEBARI_HOSTNAME}/hub/token"}
+    if xsrf_token:
+        headers["X-XSRFToken"] = xsrf_token
+    data = {"note": note, "expires_in": None}
     r = session.post(
         f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{constants.KEYCLOAK_USERNAME}/tokens",
-        headers={
-            "Referer": f"https://{constants.NEBARI_HOSTNAME}/hub/token",
-        },
-        json={
-            "note": note,
-            "expires_in": None,
-        },
+        headers=headers,
+        json=data,
     )
+    print(xsrf_token, note, r.json())
     return r.json()["token"]
 
 

From 311b00d5cd9eedd62cb12e99fb73fa72c14721eb Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 2 Feb 2024 12:54:03 -0500
Subject: [PATCH 04/32] Extract api_token as a session fixture to avoid token
 conflicts when server starts.

---
 tests/tests_deployment/test_jupyterhub_ssh.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/tests_deployment/test_jupyterhub_ssh.py b/tests/tests_deployment/test_jupyterhub_ssh.py
index 0e90927a4c..fd6b0799d5 100644
--- a/tests/tests_deployment/test_jupyterhub_ssh.py
+++ b/tests/tests_deployment/test_jupyterhub_ssh.py
@@ -14,10 +14,14 @@
 TIMEOUT_SECS = 300
 
 
+@pytest.fixture(scope="session")
+def api_token():
+    return get_jupyterhub_token("jupyterhub-ssh")
+
+
 @pytest.fixture(scope="function")
-def paramiko_object():
+def paramiko_object(api_token):
     """Connects to JupyterHub ssh cluster from outside the cluster."""
-    api_token = get_jupyterhub_token("jupyterhub-ssh")
 
     try:
         client = paramiko.SSHClient()

From 4aa145724dbd20f15da5a5a1b81721a02bfc8f2d Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 2 Feb 2024 13:23:41 -0500
Subject: [PATCH 05/32] Remove print statement.

---
 tests/tests_deployment/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/tests_deployment/utils.py b/tests/tests_deployment/utils.py
index b2b2a707b6..d175a2dd05 100644
--- a/tests/tests_deployment/utils.py
+++ b/tests/tests_deployment/utils.py
@@ -38,7 +38,7 @@ def get_jupyterhub_token(note="jupyterhub-tests-deployment"):
         headers=headers,
         json=data,
     )
-    print(xsrf_token, note, r.json())
+
     return r.json()["token"]
 
 

From 874a2d83baf084a0342478b451a22f2fd6360c35 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Mon, 5 Feb 2024 13:30:23 -0500
Subject: [PATCH 06/32] Add function to stop server, making sure that it's not
 running when running deployment tests.

---
 tests/common/navigator.py           | 17 +++++++++++++++++
 tests/common/playwright_fixtures.py |  4 ++++
 2 files changed, 21 insertions(+)

diff --git a/tests/common/navigator.py b/tests/common/navigator.py
index ff56c8c5c2..50fc1837bc 100644
--- a/tests/common/navigator.py
+++ b/tests/common/navigator.py
@@ -407,3 +407,20 @@ def write_file(self, filepath, content):
         self.run_terminal_command(f"ls {filepath}")
         logger.debug(f"time to complete {dt.datetime.now() - start}")
         time.sleep(2)
+
+    def stop_server(self) -> None:
+        """Stops the JupyterHub server by navigating to the Hub Control Panel."""
+        self.page.get_by_text("File", exact=True).click()
+
+        with self.context.expect_page() as page_info:
+            self.page.get_by_role(
+                "menuitem", name="Hub Control Panel", exact=True
+            ).click()
+
+        home_page = page_info.value
+        home_page.wait_for_load_state()
+        stop_button = home_page.get_by_role("button", name="Stop My Server")
+        if not stop_button.is_visible():
+            stop_button.wait_for(state="visible")
+        stop_button.click()
+        stop_button.wait_for(state="hidden")
diff --git a/tests/common/playwright_fixtures.py b/tests/common/playwright_fixtures.py
index 388f6ef4b0..03e17a5065 100644
--- a/tests/common/playwright_fixtures.py
+++ b/tests/common/playwright_fixtures.py
@@ -48,6 +48,10 @@ def _navigator_session(request, browser_name, pytestconfig):
         logger.debug(e)
         raise
     finally:
+        try:
+            nav.stop_server()
+        except Exception as e:
+            logger.debug(e)
         nav.teardown()
 
 

From 446d790558266d5924a0eb6e6c8f1fbf7bf2ceaf Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Tue, 6 Feb 2024 11:56:45 -0500
Subject: [PATCH 07/32] Remove JupyterHub Notebook Tests as the same
 functionality is being tested with Playwright and Deployment tests.

---
 .github/workflows/test_local_integration.yaml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index 8ddc1f9690..8f8e47951f 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -167,22 +167,6 @@ jobs:
         run: |
           pytest tests/tests_deployment/ -v -s
 
-      - name: JupyterHub Notebook Tests
-        timeout-minutes: 2
-        # run jhub-client after pytest since jhubctl can cleanup
-        # the running server
-        env:
-          JUPYTERHUB_USERNAME: ${{ env.TEST_USERNAME }}
-          JUPYTERHUB_PASSWORD: ${{ env.TEST_PASSWORD }}
-        run: |
-          sleep 60
-          jhubctl --verbose run --hub=https://github-actions.nebari.dev\
-                                --auth-type=keycloak \
-                                --validate --no-verify-ssl \
-                                --kernel python3 \
-                                --stop-server \
-                                --notebook tests/tests_deployment/assets/notebook/simple.ipynb \
-
       ### CLEANUP AFTER TESTS
       - name: Cleanup nebari deployment
         if: always()

From 0d1a30d8c3c10dd1bf3a43a915bf688762777008 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <pt247@users.noreply.github.com>
Date: Sat, 10 Feb 2024 14:59:50 +0000
Subject: [PATCH 08/32] Set min nodes to 0 for worker and user. (#2168)

Co-authored-by: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Christopher Ostrouchov <chris.ostrouchov@gmail.com>
Co-authored-by: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com>
---
 .../provider/cloud/amazon_web_services.py     | 40 +++++++++++++++++++
 src/_nebari/stages/infrastructure/__init__.py | 25 +++++++++++-
 .../template/aws/modules/kubernetes/main.tf   |  8 +++-
 .../stages/kubernetes_services/__init__.py    |  3 ++
 .../template/jupyterhub.tf                    |  6 +++
 .../dask-gateway/files/gateway_config.py      |  6 ++-
 .../kubernetes/services/jupyterhub/main.tf    | 11 +++--
 .../services/jupyterhub/variables.tf          |  5 +++
 8 files changed, 96 insertions(+), 8 deletions(-)

diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py
index 576f72c1c6..2bf905bfcb 100644
--- a/src/_nebari/provider/cloud/amazon_web_services.py
+++ b/src/_nebari/provider/cloud/amazon_web_services.py
@@ -143,6 +143,46 @@ def aws_get_vpc_id(name: str, namespace: str, region: str) -> Optional[str]:
     return None
 
 
+def set_asg_tags(asg_node_group_map: Dict[str, str], region: str) -> None:
+    """Set tags for AWS node scaling from zero to work."""
+    session = aws_session(region=region)
+    autoscaling_client = session.client("autoscaling")
+    tags = []
+    for asg_name, node_group in asg_node_group_map.items():
+        tags.append(
+            {
+                "Key": "k8s.io/cluster-autoscaler/node-template/label/dedicated",
+                "Value": node_group,
+                "ResourceId": asg_name,
+                "ResourceType": "auto-scaling-group",
+                "PropagateAtLaunch": True,
+            }
+        )
+    autoscaling_client.create_or_update_tags(Tags=tags)
+
+
+def aws_get_asg_node_group_mapping(
+    name: str, namespace: str, region: str
+) -> Dict[str, str]:
+    """Return a dictionary of autoscaling groups and their associated node groups."""
+    asg_node_group_mapping = {}
+    session = aws_session(region=region)
+    eks = session.client("eks")
+    node_groups_response = eks.list_nodegroups(
+        clusterName=f"{name}-{namespace}",
+    )
+    node_groups = node_groups_response.get("nodegroups", [])
+    for nodegroup in node_groups:
+        response = eks.describe_nodegroup(
+            clusterName=f"{name}-{namespace}", nodegroupName=nodegroup
+        )
+        node_group_name = response["nodegroup"]["nodegroupName"]
+        auto_scaling_groups = response["nodegroup"]["resources"]["autoScalingGroups"]
+        for auto_scaling_group in auto_scaling_groups:
+            asg_node_group_mapping[auto_scaling_group["name"]] = node_group_name
+    return asg_node_group_mapping
+
+
 def aws_get_subnet_ids(name: str, namespace: str, region: str) -> List[str]:
     """Return list of subnet IDs for the EKS cluster named `{name}-{namespace}`."""
     session = aws_session(region=region)
diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 1e34cb05ef..5c1aa77f77 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -145,6 +145,17 @@ class AWSInputVars(schema.Base):
     tags: Dict[str, str] = {}
 
 
+def _calculate_asg_node_group_map(config: schema.Main):
+    if config.provider == schema.ProviderEnum.aws:
+        return amazon_web_services.aws_get_asg_node_group_mapping(
+            config.project_name,
+            config.namespace,
+            config.amazon_web_services.region,
+        )
+    else:
+        return {}
+
+
 def _calculate_node_groups(config: schema.Main):
     if config.provider == schema.ProviderEnum.aws:
         return {
@@ -438,10 +449,10 @@ class AmazonWebServicesProvider(schema.Base):
     node_groups: Dict[str, AWSNodeGroup] = {
         "general": AWSNodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1),
         "user": AWSNodeGroup(
-            instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False
+            instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False
         ),
         "worker": AWSNodeGroup(
-            instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False
+            instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False
         ),
     }
     existing_subnet_ids: List[str] = None
@@ -814,6 +825,16 @@ def set_outputs(
         outputs["node_selectors"] = _calculate_node_groups(self.config)
         super().set_outputs(stage_outputs, outputs)
 
+    @contextlib.contextmanager
+    def post_deploy(
+        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+    ):
+        asg_node_group_map = _calculate_asg_node_group_map(self.config)
+        if asg_node_group_map:
+            amazon_web_services.set_asg_tags(
+                asg_node_group_map, self.config.amazon_web_services.region
+            )
+
     @contextlib.contextmanager
     def deploy(
         self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index fd53a636c2..848d1c0471 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -39,6 +39,10 @@ resource "aws_eks_node_group" "main" {
     max_size     = var.node_groups[count.index].max_size
   }
 
+  labels = {
+    "dedicated" = var.node_groups[count.index].name
+  }
+
   lifecycle {
     ignore_changes = [
       scaling_config[0].desired_size,
@@ -53,7 +57,9 @@ resource "aws_eks_node_group" "main" {
   ]
 
   tags = merge({
-    "kubernetes.io/cluster/${var.name}" = "shared"
+    #    "kubernetes.io/cluster/${var.name}"                       = "shared"
+    "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
+    propagate_at_launch                                       = true
   }, var.tags)
 }
 
diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index 0e87b51800..8542ceb9d5 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -362,6 +362,7 @@ class JupyterhubInputVars(schema.Base):
     idle_culler_settings: Dict[str, Any] = Field(alias="idle-culler-settings")
     argo_workflows_enabled: bool = Field(alias="argo-workflows-enabled")
     jhub_apps_enabled: bool = Field(alias="jhub-apps-enabled")
+    cloud_provider: str = Field(alias="cloud-provider")
 
 
 class DaskGatewayInputVars(schema.Base):
@@ -411,6 +412,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
         realm_id = stage_outputs["stages/06-kubernetes-keycloak-configuration"][
             "realm_id"
         ]["value"]
+        cloud_provider = self.config.provider.value
         jupyterhub_shared_endpoint = (
             stage_outputs["stages/02-infrastructure"]
             .get("nfs_endpoint", {})
@@ -486,6 +488,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             ),
             jupyterhub_stared_storage=self.config.storage.shared_filesystem,
             jupyterhub_shared_endpoint=jupyterhub_shared_endpoint,
+            cloud_provider=cloud_provider,
             jupyterhub_profiles=self.config.profiles.dict()["jupyterlab"],
             jupyterhub_image=_split_docker_image_name(
                 self.config.default_images.jupyterhub
diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
index ca4176670e..76bc13e475 100644
--- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
+++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
@@ -55,6 +55,10 @@ variable "idle-culler-settings" {
   type        = any
 }
 
+variable "cloud-provider" {
+  description = "Name of cloud provider."
+  type        = string
+}
 
 module "kubernetes-nfs-server" {
   count = var.jupyterhub-shared-endpoint == null ? 1 : 0
@@ -88,6 +92,8 @@ module "jupyterhub" {
   name      = var.name
   namespace = var.environment
 
+  cloud-provider = var.cloud-provider
+
   external-url = var.endpoint
   realm_id     = var.realm_id
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index b1499efe44..12416e2527 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -114,8 +114,12 @@ def list_dask_environments():
 
 
 def base_node_group(options):
+    key = config["worker-node-group"]["key"]
+    if config.provider.value == "aws":
+        key = "dedicated"
     default_node_group = {
-        config["worker-node-group"]["key"]: config["worker-node-group"]["value"]
+        key: config["worker-node-group"]["value"],
+        # config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
     }
 
     # check `worker_extra_pod_config` first
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 7eed29caef..af690112f6 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -16,8 +16,11 @@ resource "random_password" "jhub_apps_jwt_secret" {
 }
 
 locals {
-  jhub_apps_secrets_name = "jhub-apps-secrets"
-  jhub_apps_env_var_name = "JHUB_APP_JWT_SECRET_KEY"
+  jhub_apps_secrets_name           = "jhub-apps-secrets"
+  jhub_apps_env_var_name           = "JHUB_APP_JWT_SECRET_KEY"
+  singleuser_nodeselector_key      = var.cloud-provider == "aws" ? "dedicated" : var.user-node-group.key
+  userscheduler_nodeselector_key   = var.cloud-provider == "aws" ? "dedicated" : var.user-node-group.key
+  userscheduler_nodeselector_value = var.cloud-provider == "aws" ? var.general-node-group.value : var.user-node-group.value
 }
 
 resource "kubernetes_secret" "jhub_apps_secrets" {
@@ -174,14 +177,14 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          "${var.user-node-group.key}" = var.user-node-group.value
+          "${local.singleuser_nodeselector_key}" = var.user-node-group.value
         }
       }
 
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "${var.user-node-group.key}" = var.user-node-group.value
+            "${local.userscheduler_nodeselector_key}" = local.userscheduler_nodeselector_value
           }
         }
       }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
index d43a2ab9c7..4b3ed0f0aa 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
@@ -168,6 +168,11 @@ variable "jupyterlab-pioneer-log-format" {
   type        = string
 }
 
+variable "cloud-provider" {
+  description = "Name of cloud provider."
+  type        = string
+}
+
 variable "initial-repositories" {
   description = "Map of folder location and git repo url to clone"
   type        = string

From 858ca5f477fdfeb207ecc496fb5c135f58c2b61c Mon Sep 17 00:00:00 2001
From: Pavithra Eswaramoorthy <pavithraes@outlook.com>
Date: Wed, 14 Feb 2024 17:25:55 +0530
Subject: [PATCH 09/32] Remove `jhub-client` from pyproject.toml (#2242)

---
 pyproject.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 20d2cc6c5a..cb90bc52d0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -84,7 +84,6 @@ dev = [
     "diagrams",
     "escapism",
     "importlib-metadata<5.0",
-    "jhub-client",
     "jinja2",
     "mypy==1.6.1",
     "paramiko",

From f6035e69c8c44a3c532ea7a446a437ef4c4fe332 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 16 Feb 2024 11:24:31 -0500
Subject: [PATCH 10/32] Rename Nebari menu item label and move it before the
 Help menu item.

---
 .../services/jupyterhub/files/jupyterlab/overrides.json     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
index bdd69047c1..619f923e45 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
@@ -15,10 +15,10 @@
     "@jupyterlab/mainmenu-extension:plugin": {
         "menus": [
             {
-                "id": "jp-mainmenu-nebari",
+                "id": "jp-mainmenu-services",
                 "disabled": false,
-                "label": "Nebari",
-                "rank": 1001,
+                "label": "Services",
+                "rank": 999,
                 "items": [
                     {
                         "command": "hub:control-panel",

From 0f34ffdfb22be2ce55fc1c99a4c620b8cbd173f3 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 16 Feb 2024 12:00:36 -0500
Subject: [PATCH 11/32] Add Nebari documentation as an item in the Help menu.

---
 .../jupyterhub/files/jupyterlab/overrides.json | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
index 619f923e45..95a29c2c06 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
@@ -20,10 +20,6 @@
                 "label": "Services",
                 "rank": 999,
                 "items": [
-                    {
-                        "command": "hub:control-panel",
-                        "rank": 0
-                    },
                     {
                         "command": "help:open",
                         "rank": 1,
@@ -61,6 +57,20 @@
                         }
                     }
                 ]
+            },
+            {
+                "id": "jp-mainmenu-help",
+                "items": [
+                    {
+                        "command": "help:open",
+                        "rank": 1001,
+                        "args": {
+                            "url": "https://www.nebari.dev/docs/welcome",
+                            "text": "Nebari documentation",
+                            "newBrowserTab": true
+                        }
+                    }
+                ]
             }
         ]
     }

From fd069b6ed6a806360251a61a85221c343def8dd1 Mon Sep 17 00:00:00 2001
From: viniciusdc <vinivdc2009@hotmail.com>
Date: Fri, 16 Feb 2024 14:37:15 -0300
Subject: [PATCH 12/32] validate and apply user permissions to progam. added
 repos/dirs

---
 .../files/extras/git_clone_update.sh          | 21 ++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
index 5c012c01c1..255579fd1d 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
@@ -31,6 +31,22 @@ if [ "$#" -lt 1 ] || [ "$1" = "--help" ]; then
   [ "$1" = "--help" ] && exit 0 || exit 1
 fi
 
+fix_parent_dir_permissions() {
+  # Fix parent directory permissions to allow the JupyterLab user to access the cloned repository
+
+  local folder_path="$1"
+
+  # Retrieve the very first parent directory
+  local parent_dir=$(echo "$folder_path" | cut -d '/' -f1)
+
+  # Check if the parent directory has the correct permissions
+  if [ "$(stat -c "%u:%g" "$parent_dir")" != "1000:100" ]; then
+    echo "Fixing permissions for parent directory: $parent_dir"
+    chown -R 1000:100 "$parent_dir" || { echo "Error: Unable to set ownership for $parent_dir"; return 1; }
+    chmod -R 755 "$parent_dir" || { echo "Error: Unable to set permissions for $parent_dir"; return 1; }
+  fi
+}
+
 clone_update_repository() {
   # Clone or update a Git repository into a specified folder,
   # and create a `.firstrun` file to mark the script's execution.
@@ -47,6 +63,9 @@ clone_update_repository() {
       mkdir -p "$folder_path"
     fi
 
+    # Fix parent directory permissions
+    fix_parent_dir_permissions "$folder_path" || return 1
+
     if [ -d "$folder_path/.git" ]; then
       echo -e "Updating Git repository in ${folder_path}..."
       (cd "$folder_path" && git pull)
@@ -72,7 +91,7 @@ for pair in "$@"; do
     echo -e "${RED}Invalid argument format: \"${pair}\". Please provide folder path and Git repository URL in the correct order.${NC}" >> "$ERROR_LOG"
   else
     clone_update_repository "$folder_path" "$git_repo_url" || echo -e "${RED}Error executing for ${folder_path}.${NC}" >> "$ERROR_LOG"
-    chown -R 1000:100 "$folder_path" # User permissions for JupyterLab user
+    # chown -R 1000:100 "$folder_path" # User permissions for JupyterLab user
   fi
 done
 

From 2ef0863aab6687a2c15aa78d2f9d566dba41ce13 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 16 Feb 2024 12:51:15 -0500
Subject: [PATCH 13/32] Disable Hub Control Panel and Log out items and add
 Home item  in the File menu.

---
 .../files/jupyterlab/overrides.json           | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
index 95a29c2c06..1bf9723934 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
@@ -14,6 +14,28 @@
     },
     "@jupyterlab/mainmenu-extension:plugin": {
         "menus": [
+            {
+                "id": "jp-mainmenu-file",
+                "items": [
+                    {
+                        "command": "hub:control-panel",
+                        "disabled": true
+                    },
+                    {
+                        "command": "hub:logout",
+                        "disabled": true
+                    },
+                    {
+                        "command": "help:open",
+                        "rank": 101,
+                        "args": {
+                            "url": "/hub/home",
+                            "text": "Home",
+                            "newBrowserTab": true
+                        }
+                    }
+                ]
+            },
             {
                 "id": "jp-mainmenu-services",
                 "disabled": false,

From 81616687c8465670f112f096d75b4675a41e3585 Mon Sep 17 00:00:00 2001
From: viniciusdc <vinivdc2009@hotmail.com>
Date: Fri, 16 Feb 2024 15:13:14 -0300
Subject: [PATCH 14/32] mistakenly removed the permission check for cloned
 files

---
 .../services/jupyterhub/files/extras/git_clone_update.sh   | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
index 255579fd1d..5c2f6d2d4c 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
@@ -74,7 +74,13 @@ clone_update_repository() {
       (git clone "$git_repo_url" "$folder_path")
     fi
 
+    echo -e "Creating .firstrun file in ${folder_path}..."
     touch "$firstrun_file"
+
+    # User permissions for JupyterLab user to newly created git folders
+    echo -e "Setting permissions for ${folder_path}..."
+    chown -R 1000:100 "$folder_path" || { echo "Error: Unable to set ownership for $folder_path"; return 1; }
+
     echo -e "Execution for ${folder_path} completed. ${GREEN}✅${NC}"
   fi
 }
@@ -91,7 +97,6 @@ for pair in "$@"; do
     echo -e "${RED}Invalid argument format: \"${pair}\". Please provide folder path and Git repository URL in the correct order.${NC}" >> "$ERROR_LOG"
   else
     clone_update_repository "$folder_path" "$git_repo_url" || echo -e "${RED}Error executing for ${folder_path}.${NC}" >> "$ERROR_LOG"
-    # chown -R 1000:100 "$folder_path" # User permissions for JupyterLab user
   fi
 done
 

From c3dce6171177fb4bdcb2b4d32eba21fed49bfc94 Mon Sep 17 00:00:00 2001
From: "Vinicius D. Cerutti" <51954708+viniciusdc@users.noreply.github.com>
Date: Fri, 16 Feb 2024 15:22:17 -0300
Subject: [PATCH 15/32] rm redundant comment

---
 .../services/jupyterhub/files/extras/git_clone_update.sh         | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
index 5c2f6d2d4c..bca1734ea2 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/extras/git_clone_update.sh
@@ -63,7 +63,6 @@ clone_update_repository() {
       mkdir -p "$folder_path"
     fi
 
-    # Fix parent directory permissions
     fix_parent_dir_permissions "$folder_path" || return 1
 
     if [ -d "$folder_path/.git" ]; then

From e8b91a11558abb3cad11e0d16ffed6a6a1e449b3 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 16 Feb 2024 13:54:43 -0500
Subject: [PATCH 16/32] Move Home item to the top.

---
 .../files/jupyterlab/overrides.json           | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
index 1bf9723934..00a221c885 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
@@ -17,22 +17,29 @@
             {
                 "id": "jp-mainmenu-file",
                 "items": [
-                    {
-                        "command": "hub:control-panel",
-                        "disabled": true
-                    },
-                    {
-                        "command": "hub:logout",
-                        "disabled": true
-                    },
                     {
                         "command": "help:open",
-                        "rank": 101,
+                        "rank": 0,
                         "args": {
                             "url": "/hub/home",
                             "text": "Home",
                             "newBrowserTab": true
                         }
+                    },
+                    {
+                        "type": "submenu",
+                        "submenu": {
+                            "id": "jp-mainmenu-file-new"
+                        },
+                        "rank": 0.5
+                    },
+                    {
+                        "command": "hub:control-panel",
+                        "disabled": true
+                    },
+                    {
+                        "command": "hub:logout",
+                        "disabled": true
                     }
                 ]
             },

From cb9e1ac8e03878c05908f13f9d8313e4c0e55c1b Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Fri, 16 Feb 2024 14:55:51 -0500
Subject: [PATCH 17/32] Make sure that the Services menu item is left from the
 Help menu.

---
 .../services/jupyterhub/files/jupyterlab/overrides.json        | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
index 00a221c885..5c9b8eb8cb 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
@@ -47,7 +47,7 @@
                 "id": "jp-mainmenu-services",
                 "disabled": false,
                 "label": "Services",
-                "rank": 999,
+                "rank": 1000,
                 "items": [
                     {
                         "command": "help:open",
@@ -89,6 +89,7 @@
             },
             {
                 "id": "jp-mainmenu-help",
+                "rank": 1001,
                 "items": [
                     {
                         "command": "help:open",

From 790e7afc319c70803928d53e943c537bd367f7be Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Sun, 18 Feb 2024 15:26:08 -0500
Subject: [PATCH 18/32] Update menu item to Home because it changed in the UI.

---
 tests/common/navigator.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/common/navigator.py b/tests/common/navigator.py
index 50fc1837bc..12a1445bd5 100644
--- a/tests/common/navigator.py
+++ b/tests/common/navigator.py
@@ -413,9 +413,7 @@ def stop_server(self) -> None:
         self.page.get_by_text("File", exact=True).click()
 
         with self.context.expect_page() as page_info:
-            self.page.get_by_role(
-                "menuitem", name="Hub Control Panel", exact=True
-            ).click()
+            self.page.get_by_role("menuitem", name="Home", exact=True).click()
 
         home_page = page_info.value
         home_page.wait_for_load_state()

From 32f62f8dbcb498727e964e1da8260c46c7966a45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Krassowski?=
 <5832902+krassowski@users.noreply.github.com>
Date: Mon, 19 Feb 2024 09:45:39 +0000
Subject: [PATCH 19/32] Expose jupyter's preferred dir as a config option
 (#2251)

---
 src/_nebari/stages/kubernetes_services/__init__.py         | 5 +++++
 .../stages/kubernetes_services/template/jupyterhub.tf      | 7 +++++++
 .../modules/kubernetes/services/jupyterhub/configmaps.tf   | 1 +
 .../jupyterhub/files/jupyter/jupyter_server_config.py.tpl  | 4 ++++
 .../modules/kubernetes/services/jupyterhub/variables.tf    | 5 +++++
 5 files changed, 22 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index 8542ceb9d5..69ae5b4b1c 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -229,6 +229,7 @@ class IdleCuller(schema.Base):
 class JupyterLab(schema.Base):
     idle_culler: IdleCuller = IdleCuller()
     initial_repositories: typing.List[typing.Dict[str, str]] = []
+    preferred_dir: typing.Optional[str] = None
 
 
 class InputSchema(schema.Base):
@@ -363,6 +364,9 @@ class JupyterhubInputVars(schema.Base):
     argo_workflows_enabled: bool = Field(alias="argo-workflows-enabled")
     jhub_apps_enabled: bool = Field(alias="jhub-apps-enabled")
     cloud_provider: str = Field(alias="cloud-provider")
+    jupyterlab_preferred_dir: typing.Optional[str] = Field(
+        alias="jupyterlab-preferred-dir"
+    )
 
 
 class DaskGatewayInputVars(schema.Base):
@@ -501,6 +505,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             argo_workflows_enabled=self.config.argo_workflows.enabled,
             jhub_apps_enabled=self.config.jhub_apps.enabled,
             initial_repositories=str(self.config.jupyterlab.initial_repositories),
+            jupyterlab_preferred_dir=self.config.jupyterlab.preferred_dir,
         )
 
         dask_gateway_vars = DaskGatewayInputVars(
diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
index 76bc13e475..d4f60352f7 100644
--- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
+++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
@@ -39,6 +39,11 @@ variable "jupyterlab-profiles" {
   description = "JupyterHub profiles to expose to user"
 }
 
+variable "jupyterlab-preferred-dir" {
+  description = "Directory in which the JupyterLab should open the file browser"
+  type        = string
+}
+
 variable "initial-repositories" {
   description = "Map of folder location and git repo url to clone"
   type        = string
@@ -144,4 +149,6 @@ module "jupyterhub" {
 
   jupyterlab-pioneer-enabled    = var.jupyterlab-pioneer-enabled
   jupyterlab-pioneer-log-format = var.jupyterlab-pioneer-log-format
+
+  jupyterlab-preferred-dir = var.jupyterlab-preferred-dir
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
index 4b6561129e..daae8da2f5 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
@@ -7,6 +7,7 @@ locals {
     kernel_cull_connected               = var.idle-culler-settings.kernel_cull_connected ? "True" : "False" # for Python compatible boolean values
     kernel_cull_busy                    = var.idle-culler-settings.kernel_cull_busy ? "True" : "False"      # for Python compatible boolean values
     server_shutdown_no_activity_timeout = var.idle-culler-settings.server_shutdown_no_activity_timeout
+    jupyterlab_preferred_dir            = var.jupyterlab-preferred-dir != null ? var.jupyterlab-preferred-dir : ""
     }
   )
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl
index 79e3ec37d2..ab5117ee1b 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl
@@ -9,6 +9,10 @@
 c.ContentsManager.allow_hidden = True
 c.FileContentsManager.allow_hidden = True
 
+# Set the preferred path for the frontend to start in
+preferred_dir = ${jupyterlab_preferred_dir}
+c.FileContentsManager.preferred_dir = preferred_dir if preferred_dir else None
+
 # Timeout (in seconds) in which a terminal has been inactive and ready to
 # be culled.
 c.TerminalManager.cull_inactive_timeout = ${terminal_cull_inactive_timeout} * 60
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
index 4b3ed0f0aa..24d1543182 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
@@ -168,6 +168,11 @@ variable "jupyterlab-pioneer-log-format" {
   type        = string
 }
 
+variable "jupyterlab-preferred-dir" {
+  description = "Directory in which the JupyterLab should open the file browser"
+  type        = string
+}
+
 variable "cloud-provider" {
   description = "Name of cloud provider."
   type        = string

From 872f7bfc893d365676266a4815b6d427c69c9d28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Krassowski?=
 <5832902+krassowski@users.noreply.github.com>
Date: Mon, 19 Feb 2024 11:05:47 +0000
Subject: [PATCH 20/32] Allow to configure default settings for JupyterLab
 (`overrides.json`) (#2249)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Amit Kumar <dtu.amit@gmail.com>
---
 .github/workflows/test_local_integration.yaml |  9 ++++++
 .../stages/kubernetes_services/__init__.py    |  5 ++++
 .../template/jupyterhub.tf                    |  7 +++++
 .../services/jupyterhub/configmaps.tf         | 28 ++++++++++++++++---
 .../services/jupyterhub/variables.tf          |  5 ++++
 tests/tests_e2e/cypress/integration/main.js   |  3 ++
 ...min.happy.jupyterlab.default_settings.yaml |  5 ++++
 7 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 tests/tests_unit/cli_validate/min.happy.jupyterlab.default_settings.yaml

diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index 8f8e47951f..ac5ff87b46 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -96,6 +96,15 @@ jobs:
           sed -i -E 's/(cpu_guarantee):\s+[0-9\.]+/\1: 0.25/g' "nebari-config.yaml"
           sed -i -E 's/(mem_guarantee):\s+[A-Za-z0-9\.]+/\1: 0.25G/g' "nebari-config.yaml"
 
+
+          # Change default JupyterLab theme
+          cat >> nebari-config.yaml <<- EOM
+          jupyterlab:
+            default_settings:
+              "@jupyterlab/apputils-extension:themes":
+                theme: JupyterLab Dark
+          EOM
+
           cat nebari-config.yaml
 
       - name: Deploy Nebari
diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index 69ae5b4b1c..6e2f1c0e9f 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -227,6 +227,7 @@ class IdleCuller(schema.Base):
 
 
 class JupyterLab(schema.Base):
+    default_settings: typing.Dict[str, typing.Any] = {}
     idle_culler: IdleCuller = IdleCuller()
     initial_repositories: typing.List[typing.Dict[str, str]] = []
     preferred_dir: typing.Optional[str] = None
@@ -353,6 +354,9 @@ class CondaStoreInputVars(schema.Base):
 class JupyterhubInputVars(schema.Base):
     jupyterhub_theme: Dict[str, Any] = Field(alias="jupyterhub-theme")
     jupyterlab_image: ImageNameTag = Field(alias="jupyterlab-image")
+    jupyterlab_default_settings: Dict[str, Any] = Field(
+        alias="jupyterlab-default-settings"
+    )
     initial_repositories: str = Field(alias="initial-repositories")
     jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides")
     jupyterhub_stared_storage: str = Field(alias="jupyterhub-shared-storage")
@@ -505,6 +509,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             argo_workflows_enabled=self.config.argo_workflows.enabled,
             jhub_apps_enabled=self.config.jhub_apps.enabled,
             initial_repositories=str(self.config.jupyterlab.initial_repositories),
+            jupyterlab_default_settings=self.config.jupyterlab.default_settings,
             jupyterlab_preferred_dir=self.config.jupyterlab.preferred_dir,
         )
 
diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
index d4f60352f7..481ebc74cd 100644
--- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
+++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
@@ -49,6 +49,11 @@ variable "initial-repositories" {
   type        = string
 }
 
+variable "jupyterlab-default-settings" {
+  description = "Default settings for JupyterLab to be placed in overrides.json"
+  type        = map(any)
+}
+
 variable "jupyterhub-hub-extraEnv" {
   description = "Extracted overrides to merge with jupyterhub.hub.extraEnv"
   type        = string
@@ -147,6 +152,8 @@ module "jupyterhub" {
   idle-culler-settings = var.idle-culler-settings
   initial-repositories = var.initial-repositories
 
+  jupyterlab-default-settings = var.jupyterlab-default-settings
+
   jupyterlab-pioneer-enabled    = var.jupyterlab-pioneer-enabled
   jupyterlab-pioneer-log-format = var.jupyterlab-pioneer-log-format
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
index daae8da2f5..6fe6cd0e5e 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
@@ -12,6 +12,13 @@ locals {
   )
 }
 
+locals {
+  jupyterlab-overrides-json-object = merge(
+    jsondecode(file("${path.module}/files/jupyterlab/overrides.json")),
+    var.jupyterlab-default-settings
+  )
+}
+
 locals {
   jupyter-pioneer-config-py-template = templatefile("${path.module}/files/jupyter/jupyter_jupyterlab_pioneer_config.py.tpl", {
     log_format = var.jupyterlab-pioneer-log-format != null ? var.jupyterlab-pioneer-log-format : ""
@@ -31,6 +38,12 @@ resource "local_file" "jupyter_jupyterlab_pioneer_config_py" {
 }
 
 
+resource "local_file" "overrides_json" {
+  content  = jsonencode(local.jupyterlab-overrides-json-object)
+  filename = "${path.module}/files/jupyterlab/overrides.json"
+}
+
+
 resource "kubernetes_config_map" "etc-ipython" {
   metadata {
     name      = "etc-ipython"
@@ -57,6 +70,12 @@ locals {
   )
 }
 
+locals {
+  etc-jupyterlab-settings = {
+    "overrides.json" = local_file.overrides_json.content
+  }
+}
+
 resource "kubernetes_config_map" "etc-jupyter" {
   depends_on = [
     local_file.jupyter_server_config_py,
@@ -86,15 +105,16 @@ resource "kubernetes_config_map" "etc-skel" {
 
 
 resource "kubernetes_config_map" "jupyterlab-settings" {
+  depends_on = [
+    local_file.overrides_json
+  ]
+
   metadata {
     name      = "jupyterlab-settings"
     namespace = var.namespace
   }
 
-  data = {
-    for filename in fileset("${path.module}/files/jupyterlab", "*") :
-    filename => file("${path.module}/files/jupyterlab/${filename}")
-  }
+  data = local.etc-jupyterlab-settings
 }
 
 resource "kubernetes_config_map" "git_clone_update" {
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
index 24d1543182..577dedc8ef 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
@@ -158,6 +158,11 @@ variable "argo-workflows-enabled" {
   type        = bool
 }
 
+variable "jupyterlab-default-settings" {
+  description = "Default settings for JupyterLab to be placed in overrides.json"
+  type        = map(any)
+}
+
 variable "jupyterlab-pioneer-enabled" {
   description = "Enable JupyterLab Pioneer for telemetry"
   type        = bool
diff --git a/tests/tests_e2e/cypress/integration/main.js b/tests/tests_e2e/cypress/integration/main.js
index 1e1fbf206f..1184ba76d6 100644
--- a/tests/tests_e2e/cypress/integration/main.js
+++ b/tests/tests_e2e/cypress/integration/main.js
@@ -61,6 +61,9 @@ describe('First Test', () => {
       // Click VS Code Launcher exists
       cy.get('div.jp-LauncherCard[title="VS Code [↗]"]').should('exist');
 
+      // Should reflect theme set by default_settings
+      cy.get('body[data-jp-theme-name="JupyterLab Dark"]').should('exist');
+
       // Stop my Jupyter server - must do this so PVC can be destroyed on Minikube
       cy.visit('/hub/home');
 
diff --git a/tests/tests_unit/cli_validate/min.happy.jupyterlab.default_settings.yaml b/tests/tests_unit/cli_validate/min.happy.jupyterlab.default_settings.yaml
new file mode 100644
index 0000000000..4b29a53c1c
--- /dev/null
+++ b/tests/tests_unit/cli_validate/min.happy.jupyterlab.default_settings.yaml
@@ -0,0 +1,5 @@
+project_name: test
+jupyterlab:
+  default_settings:
+    "@jupyterlab/apputils-extension:themes":
+      theme: JupyterLab Dark

From d11993e85544c78a286bcdb29da696ccac20a8cd Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Mon, 19 Feb 2024 09:31:13 -0500
Subject: [PATCH 21/32] Update
 src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Michał Krassowski <5832902+krassowski@users.noreply.github.com>
---
 .../services/jupyterhub/files/jupyterlab/overrides.json         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
index 5c9b8eb8cb..357ab1af0c 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
@@ -95,7 +95,7 @@
                         "command": "help:open",
                         "rank": 1001,
                         "args": {
-                            "url": "https://www.nebari.dev/docs/welcome",
+                            "url": "https://www.nebari.dev/docs/welcome/",
                             "text": "Nebari documentation",
                             "newBrowserTab": true
                         }

From 642c8948e01b2b12286a61cc0f05380a84ed048f Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Tue, 20 Feb 2024 12:18:10 -0500
Subject: [PATCH 22/32] Pass cloud provider to the dask gateway config.json
 file and fix access to it when determining the default node group.

---
 src/_nebari/stages/kubernetes_services/__init__.py         | 2 ++
 .../stages/kubernetes_services/template/dask_gateway.tf    | 7 +++++++
 .../services/dask-gateway/files/gateway_config.py          | 7 ++-----
 .../modules/kubernetes/services/dask-gateway/gateway.tf    | 1 +
 .../modules/kubernetes/services/dask-gateway/variables.tf  | 5 +++++
 5 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index 6e2f1c0e9f..0702a27c52 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -376,6 +376,7 @@ class JupyterhubInputVars(schema.Base):
 class DaskGatewayInputVars(schema.Base):
     dask_worker_image: ImageNameTag = Field(alias="dask-worker-image")
     dask_gateway_profiles: Dict[str, Any] = Field(alias="dask-gateway-profiles")
+    cloud_provider: str = Field(alias="cloud-provider")
 
 
 class MonitoringInputVars(schema.Base):
@@ -518,6 +519,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 self.config.default_images.dask_worker
             ),
             dask_gateway_profiles=self.config.profiles.dict()["dask_worker"],
+            cloud_provider=cloud_provider,
         )
 
         monitoring_vars = MonitoringInputVars(
diff --git a/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
index 765be2753a..cb35bf60c8 100644
--- a/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
+++ b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
@@ -11,6 +11,11 @@ variable "dask-gateway-profiles" {
   description = "Dask Gateway profiles to expose to user"
 }
 
+variable "cloud-provider" {
+  description = "Name of cloud provider."
+  type        = string
+}
+
 
 # =================== RESOURCES =====================
 module "dask-gateway" {
@@ -39,4 +44,6 @@ module "dask-gateway" {
 
   # profiles
   profiles = var.dask-gateway-profiles
+
+  cloud-provider = var.cloud-provider
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index 12416e2527..2219d14e56 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -115,12 +115,9 @@ def list_dask_environments():
 
 def base_node_group(options):
     key = config["worker-node-group"]["key"]
-    if config.provider.value == "aws":
+    if config.get("provider", "") == "aws":
         key = "dedicated"
-    default_node_group = {
-        key: config["worker-node-group"]["value"],
-        # config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
-    }
+    default_node_group = {key: config["worker-node-group"]["value"]}
 
     # check `worker_extra_pod_config` first
     worker_node_group = (
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/gateway.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/gateway.tf
index eb99f75d54..62265b350b 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/gateway.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/gateway.tf
@@ -24,6 +24,7 @@ resource "kubernetes_secret" "gateway" {
       conda-store-api-token                = var.conda-store-api-token
       conda-store-service-name             = var.conda-store-service-name
       conda-store-namespace                = var.namespace
+      provider                             = var.cloud-provider
     })
   }
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf
index 5feb72d167..7f8a4aa978 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf
@@ -199,3 +199,8 @@ variable "conda-store-service-name" {
   description = "internal service-name:port where conda-store can be reached"
   type        = string
 }
+
+variable "cloud-provider" {
+  description = "Name of the cloud provider to deploy to."
+  type        = string
+}

From 8f7d933da6b9aaf1f71daa58d67525e4d5592282 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Tue, 20 Feb 2024 12:29:25 -0500
Subject: [PATCH 23/32] Move duplicate variable to variables.tf

---
 .../stages/kubernetes_services/template/dask_gateway.tf     | 6 ------
 .../stages/kubernetes_services/template/jupyterhub.tf       | 5 -----
 .../stages/kubernetes_services/template/variables.tf        | 6 ++++++
 3 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
index cb35bf60c8..b9b0a9c6c3 100644
--- a/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
+++ b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
@@ -11,12 +11,6 @@ variable "dask-gateway-profiles" {
   description = "Dask Gateway profiles to expose to user"
 }
 
-variable "cloud-provider" {
-  description = "Name of cloud provider."
-  type        = string
-}
-
-
 # =================== RESOURCES =====================
 module "dask-gateway" {
   source = "./modules/kubernetes/services/dask-gateway"
diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
index 481ebc74cd..4f8bebb9e4 100644
--- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
+++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
@@ -65,11 +65,6 @@ variable "idle-culler-settings" {
   type        = any
 }
 
-variable "cloud-provider" {
-  description = "Name of cloud provider."
-  type        = string
-}
-
 module "kubernetes-nfs-server" {
   count = var.jupyterhub-shared-endpoint == null ? 1 : 0
 
diff --git a/src/_nebari/stages/kubernetes_services/template/variables.tf b/src/_nebari/stages/kubernetes_services/template/variables.tf
index 7a125f1c69..4b78f5994e 100644
--- a/src/_nebari/stages/kubernetes_services/template/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/variables.tf
@@ -53,7 +53,13 @@ variable "jupyterlab-pioneer-log-format" {
   description = "Logging format for JupyterLab Pioneer"
   type        = string
 }
+
 variable "jhub-apps-enabled" {
   description = "Enable JupyterHub Apps"
   type        = bool
 }
+
+variable "cloud-provider" {
+  description = "Name of cloud provider."
+  type        = string
+}

From 04edd533e5014557a548b48ffbb0fe79bed6a2a8 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Wed, 21 Feb 2024 13:38:42 +0000
Subject: [PATCH 24/32] Add "Open VS Code" entry in services

---
 .../services/jupyterhub/files/jupyterlab/overrides.json    | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
index 357ab1af0c..fd6cafc624 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json
@@ -84,6 +84,13 @@
                             "text": "Argo Workflows",
                             "newBrowserTab": true
                         }
+                    },
+                    {
+                        "command": "nebari:open-proxy",
+                        "rank": 5,
+                        "args": {
+                            "name": "vscode"
+                        }
                     }
                 ]
             },

From 529adc113c9e9e70af0469aef907210838844785 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Thu, 22 Feb 2024 14:46:31 -0500
Subject: [PATCH 25/32] Change nebari and nebari workflow controller image
 tags.

---
 src/_nebari/constants.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py
index 19ed5ce7e8..229d7957a4 100644
--- a/src/_nebari/constants.py
+++ b/src/_nebari/constants.py
@@ -12,8 +12,8 @@
 DEFAULT_GKE_RELEASE_CHANNEL = "UNSPECIFIED"
 
 DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE
-DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE
-DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = "2024.1.1"
+DEFAULT_NEBARI_IMAGE_TAG = "2024.2.1rc2"
+DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = "2024.2.1rc2"
 
 DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.1.1"
 

From a1e5fd1e7262d5193e9c34a55b1c26a8a6fcca8f Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Thu, 22 Feb 2024 14:46:42 -0500
Subject: [PATCH 26/32] Add upgrade message for new version.

---
 src/_nebari/upgrade.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index ef933f48ea..17cb4ef9f0 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -705,6 +705,20 @@ def _version_specific_upgrade(
         return config
 
 
+class Upgrade_2024_2_1(UpgradeStep):
+    version = "2024.2.1"
+
+    def _version_specific_upgrade(
+        self, config, start_version, config_filename: Path, *args, **kwargs
+    ):
+        rich.print("\n ⚠️  Warning ⚠️")
+        rich.print(
+            "-> Please run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment."
+        )
+
+        return config
+
+
 __rounded_version__ = str(rounded_ver_parse(__version__))
 
 # Manually-added upgrade steps must go above this line

From 292df05ad9de616a4b316989a20737e8065cfccf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Krassowski?=
 <5832902+krassowski@users.noreply.github.com>
Date: Tue, 5 Mar 2024 14:50:34 +0000
Subject: [PATCH 27/32] Fix syntax error in jupyter-server-config Python file
 (#2286)

---
 .../kubernetes/services/jupyterhub/configmaps.tf       | 10 ++++++++++
 .../jupyter/jupyter_jupyterlab_pioneer_config.py.tpl   |  2 +-
 .../files/jupyter/jupyter_server_config.py.tpl         |  4 +---
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
index 6fe6cd0e5e..4b8f9145b9 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
@@ -30,11 +30,21 @@ locals {
 resource "local_file" "jupyter_server_config_py" {
   content  = local.jupyter-notebook-config-py-template
   filename = "${path.module}/files/jupyter/jupyter_server_config.py"
+
+  provisioner "local-exec" {
+    # check the syntax of the config file without running it
+    command = "python -m py_compile ${self.filename}"
+  }
 }
 
 resource "local_file" "jupyter_jupyterlab_pioneer_config_py" {
   content  = local.jupyter-pioneer-config-py-template
   filename = "${path.module}/files/jupyter/jupyter_jupyterlab_pioneer_config.py"
+
+  provisioner "local-exec" {
+    # check the syntax of the config file without running it
+    command = "python -m py_compile ${self.filename}"
+  }
 }
 
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_jupyterlab_pioneer_config.py.tpl b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_jupyterlab_pioneer_config.py.tpl
index 2149d298f8..66b653b894 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_jupyterlab_pioneer_config.py.tpl
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_jupyterlab_pioneer_config.py.tpl
@@ -3,7 +3,7 @@ import json
 
 
 default_log_format = "%(asctime)s %(levelname)9s %(lineno)4s %(module)s: %(message)s"
-log_format = ${log_format}
+log_format = "${log_format}"
 
 logging.basicConfig(
     level=logging.INFO,
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl
index ab5117ee1b..d5e089dfa3 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl
@@ -4,14 +4,12 @@
 
 # Extra config available at:
 # https://zero-to-jupyterhub.readthedocs.io/en/1.x/jupyterhub/customizing/user-management.html#culling-user-pods
-
 # Enable Show Hidden Files menu option in View menu
 c.ContentsManager.allow_hidden = True
 c.FileContentsManager.allow_hidden = True
 
 # Set the preferred path for the frontend to start in
-preferred_dir = ${jupyterlab_preferred_dir}
-c.FileContentsManager.preferred_dir = preferred_dir if preferred_dir else None
+c.FileContentsManager.preferred_dir = "${jupyterlab_preferred_dir}"
 
 # Timeout (in seconds) in which a terminal has been inactive and ready to
 # be culled.

From 3bd47cb11b6ea440ee34a604305cfb77f9190b01 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Tue, 5 Mar 2024 15:10:06 -0500
Subject: [PATCH 28/32] Update version and warning message.

---
 src/_nebari/upgrade.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 17cb4ef9f0..ec0b1227ad 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -705,15 +705,15 @@ def _version_specific_upgrade(
         return config
 
 
-class Upgrade_2024_2_1(UpgradeStep):
-    version = "2024.2.1"
+class Upgrade_2024_3_1(UpgradeStep):
+    version = "2024.3.1"
 
     def _version_specific_upgrade(
         self, config, start_version, config_filename: Path, *args, **kwargs
     ):
         rich.print("\n ⚠️  Warning ⚠️")
         rich.print(
-            "-> Please run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment."
+            "-> Please run [green]mv stages stages.old[/green] so that we can keep a backup of the original Terraform scripts and regenerate an updated set of them for your deployment."
         )
 
         return config

From 2abed564104f13447be063a4c56db93107fe00ea Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Tue, 5 Mar 2024 17:12:13 -0500
Subject: [PATCH 29/32] Update upgrade message.

---
 src/_nebari/upgrade.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index ec0b1227ad..5c095f04a2 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -711,10 +711,7 @@ class Upgrade_2024_3_1(UpgradeStep):
     def _version_specific_upgrade(
         self, config, start_version, config_filename: Path, *args, **kwargs
     ):
-        rich.print("\n ⚠️  Warning ⚠️")
-        rich.print(
-            "-> Please run [green]mv stages stages.old[/green] so that we can keep a backup of the original Terraform scripts and regenerate an updated set of them for your deployment."
-        )
+        rich.print("Ready to upgrade to Nebari version [green]2024.3.1[/green].")
 
         return config
 

From 0210a47a1acbe4940e0f6ff18fb72f630e224230 Mon Sep 17 00:00:00 2001
From: Amit Kumar <dtu.amit@gmail.com>
Date: Fri, 8 Mar 2024 19:15:52 +0000
Subject: [PATCH 30/32] Add Grafana Loki integration (#2156)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Vinicius D. Cerutti <51954708+viniciusdc@users.noreply.github.com>
---
 .cirun.yml                                    |   4 +-
 .github/workflows/test_local_integration.yaml |  13 +-
 RELEASE.md                                    |   2 +
 .../stages/kubernetes_services/__init__.py    |  22 +++
 .../services/monitoring/loki/main.tf          | 103 ++++++++++++++
 .../services/monitoring/loki/values_loki.yaml |  78 +++++++++++
 .../monitoring/loki/values_minio.yaml         |   1 +
 .../monitoring/loki/values_promtail.yaml      |   1 +
 .../services/monitoring/loki/variables.tf     |  84 ++++++++++++
 .../services/monitoring/values.yaml           |   6 +
 .../template/monitoring.tf                    |  11 ++
 .../kubernetes_services/template/variables.tf |  24 ++++
 tests/common/kube_api.py                      |  40 ++++++
 .../tests_deployment/test_loki_deployment.py  | 126 ++++++++++++++++++
 .../min.happy.monitoring.overrides.yaml       |  10 ++
 15 files changed, 521 insertions(+), 4 deletions(-)
 create mode 100644 src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/main.tf
 create mode 100644 src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_loki.yaml
 create mode 100644 src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_minio.yaml
 create mode 100644 src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_promtail.yaml
 create mode 100644 src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/variables.tf
 create mode 100644 tests/common/kube_api.py
 create mode 100644 tests/tests_deployment/test_loki_deployment.py
 create mode 100644 tests/tests_unit/cli_validate/min.happy.monitoring.overrides.yaml

diff --git a/.cirun.yml b/.cirun.yml
index bdabe6500b..dcc829bb8b 100644
--- a/.cirun.yml
+++ b/.cirun.yml
@@ -4,8 +4,8 @@ runners:
   - name: run-k8s-tests
     # Cloud Provider: AWS
     cloud: aws
-    # Instance Type has 4 vcpu, 16 GiB memory, Up to 5 Gbps Network Performance
-    instance_type: t3a.xlarge
+    # Instance Type has 8 vcpu, 32 GiB memory, Up to 5 Gbps Network Performance
+    instance_type: t3a.2xlarge
     # Custom AMI with docker/cypress/hub pre-installed
     machine_image: ami-0a388df278199ff52
     # Region: Oregon
diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index ac5ff87b46..05dec384b0 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -96,7 +96,6 @@ jobs:
           sed -i -E 's/(cpu_guarantee):\s+[0-9\.]+/\1: 0.25/g' "nebari-config.yaml"
           sed -i -E 's/(mem_guarantee):\s+[A-Za-z0-9\.]+/\1: 0.25G/g' "nebari-config.yaml"
 
-
           # Change default JupyterLab theme
           cat >> nebari-config.yaml <<- EOM
           jupyterlab:
@@ -105,6 +104,16 @@ jobs:
                 theme: JupyterLab Dark
           EOM
 
+          # Change default value for minio persistence size
+          cat >> nebari-config.yaml <<- EOM
+          monitoring:
+            enabled: true
+            overrides:
+              minio:
+                persistence:
+                  size: 1Gi
+          EOM
+
           cat nebari-config.yaml
 
       - name: Deploy Nebari
@@ -115,7 +124,7 @@ jobs:
       - name: Basic kubectl checks after deployment
         if: always()
         run: |
-          kubectl get all,cm,secret,ing -A
+          kubectl get all,cm,secret,pv,pvc,ing -A
 
       - name: Check github-actions.nebari.dev resolves
         run: |
diff --git a/RELEASE.md b/RELEASE.md
index f3f93499ae..076754b3a6 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -11,6 +11,8 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ## Upcoming Release
 
+* Added Grafana Loki to aggregate, index and search logs
+
 ## Release 2024.1.1 - January 17, 2024
 
 ### Feature changes and enhancements
diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index 0702a27c52..a9124f41ac 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -199,8 +199,16 @@ class JHubApps(schema.Base):
     enabled: bool = False
 
 
+class MonitoringOverrides(schema.Base):
+    loki: typing.Dict = {}
+    promtail: typing.Dict = {}
+    minio: typing.Dict = {}
+
+
 class Monitoring(schema.Base):
     enabled: bool = True
+    overrides: MonitoringOverrides = MonitoringOverrides()
+    minio_enabled: bool = True
 
 
 class JupyterLabPioneer(schema.Base):
@@ -381,6 +389,12 @@ class DaskGatewayInputVars(schema.Base):
 
 class MonitoringInputVars(schema.Base):
     monitoring_enabled: bool = Field(alias="monitoring-enabled")
+    minio_enabled: bool = Field(alias="minio-enabled")
+    grafana_loki_overrides: List[str] = Field(alias="grafana-loki-overrides")
+    grafana_promtail_overrides: List[str] = Field(alias="grafana-promtail-overrides")
+    grafana_loki_minio_overrides: List[str] = Field(
+        alias="grafana-loki-minio-overrides"
+    )
 
 
 class TelemetryInputVars(schema.Base):
@@ -524,6 +538,14 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
 
         monitoring_vars = MonitoringInputVars(
             monitoring_enabled=self.config.monitoring.enabled,
+            minio_enabled=self.config.monitoring.minio_enabled,
+            grafana_loki_overrides=[json.dumps(self.config.monitoring.overrides.loki)],
+            grafana_promtail_overrides=[
+                json.dumps(self.config.monitoring.overrides.promtail)
+            ],
+            grafana_loki_minio_overrides=[
+                json.dumps(self.config.monitoring.overrides.minio)
+            ],
         )
 
         telemetry_vars = TelemetryInputVars(
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/main.tf
new file mode 100644
index 0000000000..8180d46fb8
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/main.tf
@@ -0,0 +1,103 @@
+resource "random_password" "minio_root_password" {
+  length  = 32
+  special = false
+}
+
+locals {
+  minio-url = "http://${var.minio-release-name}:${var.minio-port}"
+  node-selector = {
+    "${var.node-group.key}" = "${var.node-group.value}"
+  }
+}
+
+resource "helm_release" "loki-minio" {
+  count      = var.minio-enabled ? 1 : 0
+  name       = var.minio-release-name
+  namespace  = var.namespace
+  repository = "https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami"
+  chart      = "minio"
+  # last release that was Apache-2.0
+  version = var.minio-helm-chart-version
+
+  set {
+    name  = "accessKey.password"
+    value = "admin"
+  }
+
+  set {
+    name  = "secretKey.password"
+    value = random_password.minio_root_password.result
+  }
+
+  set {
+    name  = "defaultBuckets"
+    value = join(" ", var.buckets)
+  }
+
+  set {
+    name  = "persistence.size"
+    value = var.minio-storage
+  }
+
+  values = concat([
+    file("${path.module}/values_minio.yaml"),
+    jsonencode({
+      nodeSelector : local.node-selector
+    })
+  ], var.grafana-loki-minio-overrides)
+}
+
+
+resource "helm_release" "grafana-loki" {
+  name       = "nebari-loki"
+  namespace  = var.namespace
+  repository = "https://grafana.github.io/helm-charts"
+  chart      = "loki"
+  version    = var.loki-helm-chart-version
+
+  values = concat([
+    file("${path.module}/values_loki.yaml"),
+    jsonencode({
+      loki : {
+        storage : {
+          s3 : {
+            endpoint : local.minio-url,
+            accessKeyId : "admin"
+            secretAccessKey : random_password.minio_root_password.result,
+            s3ForcePathStyle : true
+          }
+        }
+      }
+      storageConfig : {
+        # We configure MinIO by using the AWS config because MinIO implements the S3 API
+        aws : {
+          s3 : local.minio-url
+          s3ForcePathStyle : true
+        }
+      }
+      write : { nodeSelector : local.node-selector }
+      read : { nodeSelector : local.node-selector }
+      backend : { nodeSelector : local.node-selector }
+      gateway : { nodeSelector : local.node-selector }
+    })
+  ], var.grafana-loki-overrides)
+
+  depends_on = [helm_release.loki-minio]
+}
+
+resource "helm_release" "grafana-promtail" {
+  # Promtail ships the contents of logs to Loki instance
+  name       = "nebari-promtail"
+  namespace  = var.namespace
+  repository = "https://grafana.github.io/helm-charts"
+  chart      = "promtail"
+  version    = var.promtail-helm-chart-version
+
+  values = concat([
+    file("${path.module}/values_promtail.yaml"),
+    jsonencode({
+    })
+  ], var.grafana-promtail-overrides)
+
+  depends_on = [helm_release.grafana-loki]
+}
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_loki.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_loki.yaml
new file mode 100644
index 0000000000..c11ebe5d1f
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_loki.yaml
@@ -0,0 +1,78 @@
+# https://github.com/grafana/loki/blob/4cae003ecedd474e4c15feab4ea2ef435afff83f/production/helm/loki/values.yaml
+
+loki:
+  storage:
+    type: s3
+  commonConfig:
+    replication_factor: 1
+  # Not required as it is inside cluster and not exposed to the public network
+  auth_enabled: false
+
+  # The Compactor deduplicates index entries and also apply granular retention.
+  compactor:
+    # is the directory where marked chunks and temporary tables will be saved.
+    working_directory: /var/loki/compactor/data/retention
+    # minio s3
+    shared_store: s3
+    # how often compaction will happen
+    compaction_interval: 1h
+    # should delete old logs after retention delete delay
+    # ideally we would want to do storage based retention, but this is not
+    # currently implemented in loki, that's why we're doing time based retention.
+    retention_enabled: true
+    # is the delay after which the Compactor will delete marked chunks.
+    retention_delete_delay: 1h
+    # specifies the maximum quantity of goroutine workers instantiated to delete chunks.
+    retention_delete_worker_count: 150
+
+  limits_config:
+    # The minimum retention period is 24h.
+    # This is reasonable in most cases, but if people would like to retain logs for longer
+    # then they can override this variable from nebari-config.yaml
+    retention_period: 60d
+
+  schema_config:
+    configs:
+      # list of period_configs
+      # The date of the first day that index buckets should be created.
+      - from: "2024-03-01"
+        index:
+            period: 24h
+            prefix: loki_index_
+        object_store: s3
+        schema: v11
+        store: boltdb-shipper
+  storage_config:
+    boltdb_shipper:
+        # Directory where ingesters would write index files which would then be
+        # uploaded by shipper to configured storage
+        active_index_directory: /var/loki/compactor/data/index
+        # Cache location for restoring index files from storage for queries
+        cache_location: /var/loki/compactor/data/boltdb-cache
+        # Shared store for keeping index files
+        shared_store: s3
+
+# Configuration for the write pod(s)
+write:
+  # -- Number of replicas for the write
+  # Keeping cost of running Nebari in mind
+  # We don't need so many replicas, if people need it
+  # they can always override from nebari-config.yaml
+  replicas: 1
+
+read:
+  # -- Number of replicas for the read
+  replicas: 1
+
+backend:
+  # -- Number of replicas for the backend
+  replicas: 1
+
+minio:
+  # We are deploying minio from bitnami chart separately
+  enabled: false
+
+monitoring:
+  selfMonitoring:
+    grafanaAgent:
+      installOperator: false
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_minio.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_minio.yaml
new file mode 100644
index 0000000000..666542bb45
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_minio.yaml
@@ -0,0 +1 @@
+# https://github.com/bitnami/charts/blob/440ec159c26e4ff0748b9e9866b345d98220c40a/bitnami/minio/values.yaml
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_promtail.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_promtail.yaml
new file mode 100644
index 0000000000..5a18a9bc09
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/values_promtail.yaml
@@ -0,0 +1 @@
+# https://github.com/grafana/helm-charts/blob/3831194ba2abd2a0ca7a14ca00e578f8e9d2abc6/charts/promtail/values.yaml
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/variables.tf
new file mode 100644
index 0000000000..a43695252c
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/loki/variables.tf
@@ -0,0 +1,84 @@
+variable "namespace" {
+  description = "deploy monitoring services on this namespace"
+  type        = string
+  default     = "dev"
+}
+
+variable "loki-helm-chart-version" {
+  description = "version to deploy for the loki helm chart"
+  type        = string
+  default     = "5.43.3"
+}
+
+variable "promtail-helm-chart-version" {
+  description = "version to deploy for the promtail helm chart"
+  type        = string
+  default     = "6.15.5"
+}
+
+variable "minio-helm-chart-version" {
+  description = "version to deploy for the minio helm chart"
+  type        = string
+  default     = "6.7.4"
+}
+
+variable "grafana-loki-overrides" {
+  description = "Grafana Loki helm chart overrides"
+  type        = list(string)
+  default     = []
+}
+
+variable "grafana-promtail-overrides" {
+  description = "Grafana Promtail helm chart overrides"
+  type        = list(string)
+  default     = []
+}
+
+variable "grafana-loki-minio-overrides" {
+  description = "Grafana Loki minio helm chart overrides"
+  type        = list(string)
+  default     = []
+}
+
+variable "minio-release-name" {
+  description = "Grafana Loki minio release name"
+  type        = string
+  default     = "nebari-loki-minio"
+}
+
+variable "minio-port" {
+  description = "Grafana Loki minio port"
+  type        = number
+  default     = 9000
+}
+
+variable "buckets" {
+  description = "Minio buckets"
+  type        = list(string)
+  default = [
+    "chunks",
+    "ruler",
+    "admin",
+    "loki"
+  ]
+}
+
+variable "minio-storage" {
+  description = "Minio storage"
+  type        = string
+  default     = "50Gi"
+}
+
+variable "minio-enabled" {
+  description = "Deploy minio along with loki or not"
+  type        = bool
+  default     = true
+}
+
+variable "node-group" {
+  description = "Node key value pair for bound resources"
+  type = object({
+    key   = string
+    value = string
+  })
+}
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml
index ada868882f..f3cf47c88d 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml
@@ -1 +1,7 @@
 # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml
+
+grafana:
+  additionalDataSources:
+   - name: Loki
+     type: loki
+     url: http://loki-gateway.dev
diff --git a/src/_nebari/stages/kubernetes_services/template/monitoring.tf b/src/_nebari/stages/kubernetes_services/template/monitoring.tf
index ec20a75ba7..39487c4bb1 100644
--- a/src/_nebari/stages/kubernetes_services/template/monitoring.tf
+++ b/src/_nebari/stages/kubernetes_services/template/monitoring.tf
@@ -14,3 +14,14 @@ module "monitoring" {
 
   node-group = var.node_groups.general
 }
+
+module "grafana-loki" {
+  count                        = var.monitoring-enabled ? 1 : 0
+  source                       = "./modules/kubernetes/services/monitoring/loki"
+  namespace                    = var.environment
+  grafana-loki-overrides       = var.grafana-loki-overrides
+  grafana-promtail-overrides   = var.grafana-promtail-overrides
+  grafana-loki-minio-overrides = var.grafana-loki-minio-overrides
+  node-group                   = var.node_groups.general
+  minio-enabled                = var.minio-enabled
+}
diff --git a/src/_nebari/stages/kubernetes_services/template/variables.tf b/src/_nebari/stages/kubernetes_services/template/variables.tf
index 4b78f5994e..9e36e65979 100644
--- a/src/_nebari/stages/kubernetes_services/template/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/variables.tf
@@ -63,3 +63,27 @@ variable "cloud-provider" {
   description = "Name of cloud provider."
   type        = string
 }
+
+variable "grafana-loki-overrides" {
+  description = "Helm chart overrides for loki"
+  type        = list(string)
+  default     = []
+}
+
+variable "grafana-promtail-overrides" {
+  description = "Helm chart overrides for promtail"
+  type        = list(string)
+  default     = []
+}
+
+variable "grafana-loki-minio-overrides" {
+  description = "Grafana Loki minio helm chart overrides"
+  type        = list(string)
+  default     = []
+}
+
+variable "minio-enabled" {
+  description = "Deploy minio along with loki or not"
+  type        = bool
+  default     = true
+}
diff --git a/tests/common/kube_api.py b/tests/common/kube_api.py
new file mode 100644
index 0000000000..eec1d05d7b
--- /dev/null
+++ b/tests/common/kube_api.py
@@ -0,0 +1,40 @@
+import socket
+import typing
+
+from kubernetes import config
+from kubernetes.client.api import core_v1_api
+from kubernetes.client.models import V1Pod
+from kubernetes.stream import portforward
+
+
+def kubernetes_port_forward(
+    pod_labels: typing.Dict[str, str], port: int, namespace: str = "dev"
+) -> V1Pod:
+    """Given pod labels and port, finds the pod name and port forwards to
+    the given port.
+    :param pod_labels: dict of labels, by which to search the pod
+    :param port: port number to forward
+    :param namespace: kubernetes namespace name
+    :return: kubernetes pod object
+    """
+    config.load_kube_config()
+    core_v1 = core_v1_api.CoreV1Api()
+    label_selector = ",".join([f"{k}={v}" for k, v in pod_labels.items()])
+    pods = core_v1.list_namespaced_pod(
+        namespace=namespace, label_selector=label_selector
+    )
+    assert pods.items
+    pod = pods.items[0]
+    pod_name = pod.metadata.name
+
+    def kubernetes_create_connection(address, *args, **kwargs):
+        pf = portforward(
+            core_v1.connect_get_namespaced_pod_portforward,
+            pod_name,
+            namespace,
+            ports=str(port),
+        )
+        return pf.socket(port)
+
+    socket.create_connection = kubernetes_create_connection
+    return pod
diff --git a/tests/tests_deployment/test_loki_deployment.py b/tests/tests_deployment/test_loki_deployment.py
new file mode 100644
index 0000000000..59210a8fc3
--- /dev/null
+++ b/tests/tests_deployment/test_loki_deployment.py
@@ -0,0 +1,126 @@
+import json
+import urllib.parse
+import urllib.request as urllib_request
+
+import pytest
+from kubernetes.client import V1Pod
+
+from tests.common.kube_api import kubernetes_port_forward
+
+LOKI_BACKEND_PORT = 3100
+LOKI_BACKEND_POD_LABELS = {
+    "app.kubernetes.io/instance": "nebari-loki",
+    "app.kubernetes.io/component": "backend",
+}
+
+MINIO_PORT = 9000
+MINIO_POD_LABELS = {
+    "app.kubernetes.io/instance": "nebari-loki-minio",
+    "app.kubernetes.io/name": "minio",
+}
+
+LOKI_GATEWAY_PORT = 8080
+LOKI_GATEWAY_POD_LABELS = {
+    "app.kubernetes.io/instance": "nebari-loki",
+    "app.kubernetes.io/component": "gateway",
+}
+
+
+@pytest.fixture(scope="module")
+def port_forward_fixture(request):
+    """Pytest fixture to port forward loki backend pod to make it accessible
+    on localhost so that we can run some tests on it.
+    """
+    return kubernetes_port_forward(
+        pod_labels=request.param["labels"], port=request.param["port"]
+    )
+
+
+def port_forward(labels, port):
+    params = {"labels": labels, "port": port}
+    return pytest.mark.parametrize("port_forward_fixture", [params], indirect=True)
+
+
+@pytest.mark.parametrize(
+    "endpoint_path",
+    (
+        "metrics",
+        "services",
+        "config",
+        "ready",
+        "log_level",
+    ),
+)
+@port_forward(labels=LOKI_BACKEND_POD_LABELS, port=LOKI_BACKEND_PORT)
+def test_loki_endpoint(endpoint_path: str, port_forward_fixture: V1Pod):
+    """This will hit some endpoints in the loki API and verify that we
+    get a 200 status code, to make sure Loki is working properly.
+    :param endpoint_path: a loki api endpoint path
+    :param port_forward_fixture: pytest fixture to port forward.
+    :return:
+    """
+    pod_name = port_forward_fixture.metadata.name
+    url = f"http://{pod_name}.pod.dev.kubernetes:{LOKI_BACKEND_PORT}/{endpoint_path}"
+    response = urllib_request.urlopen(url)
+    response.read().decode("utf-8")
+    assert response.code == 200
+    response.close()
+
+
+@port_forward(labels=MINIO_POD_LABELS, port=MINIO_PORT)
+def test_minio_accessible(port_forward_fixture: V1Pod):
+    """This will hit liveness endpoint of minio  API and verify that we
+    get a 200 status code, to make sure minio is up and running.
+    :param port_forward_fixture: pytest fixture to port forward.
+    :return:
+    """
+    pod_name = port_forward_fixture.metadata.name
+    url = f"http://{pod_name}.pod.dev.kubernetes:{MINIO_PORT}/minio/health/live"
+    response = urllib_request.urlopen(url)
+    response.read().decode("utf-8")
+    assert response.code == 200
+    response.close()
+
+
+@port_forward(labels=LOKI_GATEWAY_POD_LABELS, port=LOKI_GATEWAY_PORT)
+def test_loki_gateway(port_forward_fixture: V1Pod):
+    """This will hit an endpoint of loki gateway API and verify that we
+    get a 200 status code, to make sure minio is up and running.
+    :param port_forward_fixture: pytest fixture to port forward.
+    :return:
+    """
+    pod_name = port_forward_fixture.metadata.name
+    url = f"http://{pod_name}.pod.dev.kubernetes:{LOKI_BACKEND_PORT}/loki/api/v1/labels"
+    response = urllib_request.urlopen(url)
+    response_content = response.read().decode("utf-8")
+    response_json = json.loads(response_content)
+    assert response.code == 200
+    assert response_json["status"] == "success"
+    response.close()
+
+
+@port_forward(labels=LOKI_GATEWAY_POD_LABELS, port=LOKI_GATEWAY_PORT)
+def test_loki_gateway_fetch_logs(port_forward_fixture: V1Pod):
+    """This will hit an endpoint of loki gateway API to fetch some logs
+    and verify logs received.
+    :param port_forward_fixture: pytest fixture to port forward.
+    :return: None
+    """
+    pod_name = port_forward_fixture.metadata.name
+    query_params = {
+        "limit": "5",
+        # Fetch logs for jupyterhub app
+        "query": '{app="jupyterhub"}',
+    }
+
+    encoded_params = urllib.parse.urlencode(query_params)
+    path = f"/loki/api/v1/query_range?{encoded_params}"
+    url = f"http://{pod_name}.pod.dev.kubernetes:{LOKI_BACKEND_PORT}/{path}"
+    response = urllib_request.urlopen(url)
+    response_content = response.read().decode("utf-8")
+    response_json = json.loads(response_content)
+    assert response.code == 200
+    assert response_json["status"] == "success"
+    # Make sure log lines received
+    assert len(response_json["data"]["result"][0]["values"]) > 0
+    response.close()
diff --git a/tests/tests_unit/cli_validate/min.happy.monitoring.overrides.yaml b/tests/tests_unit/cli_validate/min.happy.monitoring.overrides.yaml
new file mode 100644
index 0000000000..587c0cf5cb
--- /dev/null
+++ b/tests/tests_unit/cli_validate/min.happy.monitoring.overrides.yaml
@@ -0,0 +1,10 @@
+project_name: test
+monitoring:
+  enabled: true
+  overrides:
+    loki:
+      loki: foobar
+    promtail:
+      promtail: foobar
+    minio:
+      minio: foobar

From 930707029cb12432f63a7f6206c41b0e7ae112ae Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Mon, 11 Mar 2024 08:50:55 -0500
Subject: [PATCH 31/32] Update curretn version and image tags.

---
 src/_nebari/constants.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py
index 229d7957a4..0d49bc6e1b 100644
--- a/src/_nebari/constants.py
+++ b/src/_nebari/constants.py
@@ -1,4 +1,4 @@
-CURRENT_RELEASE = "2024.1.1"
+CURRENT_RELEASE = "2024.3.1"
 
 # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes
 # implemented in August 2023.
@@ -12,8 +12,8 @@
 DEFAULT_GKE_RELEASE_CHANNEL = "UNSPECIFIED"
 
 DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE
-DEFAULT_NEBARI_IMAGE_TAG = "2024.2.1rc2"
-DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = "2024.2.1rc2"
+DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE
+DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = CURRENT_RELEASE
 
 DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.1.1"
 

From 47fa44d62fdfe7e721de844d427f6648b09d51b2 Mon Sep 17 00:00:00 2001
From: Marcelo Villa <mvilla@quansight.com>
Date: Mon, 11 Mar 2024 12:23:24 -0500
Subject: [PATCH 32/32] Add release notes.

---
 RELEASE.md | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/RELEASE.md b/RELEASE.md
index 076754b3a6..41433e9e13 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -11,7 +11,28 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ## Upcoming Release
 
-* Added Grafana Loki to aggregate, index and search logs
+## Release 2024.3.1 - March 11, 2024
+
+### What's Changed
+* Modify Playwright test to account for changes in JupyterLab UI. by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2232
+* Add favicon to jupyterhub theme. by @jbouder in https://github.com/nebari-dev/nebari/pull/2222
+* Set min nodes to 0 for worker and user. by @pt247 in https://github.com/nebari-dev/nebari/pull/2168
+* Remove `jhub-client` from pyproject.toml by @pavithraes in https://github.com/nebari-dev/nebari/pull/2242
+* Include permission validation step to programmatically cloned repos by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2258
+* Expose jupyter's preferred dir as a config option by @krassowski in https://github.com/nebari-dev/nebari/pull/2251
+* Allow to configure default settings for JupyterLab (`overrides.json`) by @krassowski in https://github.com/nebari-dev/nebari/pull/2249
+* Feature/jlab menu customization by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2259
+* Add cloud provider to the dask config.json file by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2266
+* Fix syntax error in jupyter-server-config Python file by @krassowski in https://github.com/nebari-dev/nebari/pull/2286
+* Add "Open VS Code" entry in services by @krassowski in https://github.com/nebari-dev/nebari/pull/2267
+* Add Grafana Loki integration by @aktech in https://github.com/nebari-dev/nebari/pull/2156
+
+### New Contributors
+* @jbouder made their first contribution in https://github.com/nebari-dev/nebari/pull/2222
+* @krassowski made their first contribution in https://github.com/nebari-dev/nebari/pull/2251
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.1.1...2024.3.1
+
 
 ## Release 2024.1.1 - January 17, 2024