diff --git a/.github/workflows/test_aws_integration.yaml b/.github/workflows/test_aws_integration.yaml
index 36112ccd50..b6c27c938d 100644
--- a/.github/workflows/test_aws_integration.yaml
+++ b/.github/workflows/test_aws_integration.yaml
@@ -1,4 +1,4 @@
-name: test-aws-integration
+name: AWS Deployment
 
 on:
   schedule:
diff --git a/.github/workflows/test_azure_integration.yaml b/.github/workflows/test_azure_integration.yaml
new file mode 100644
index 0000000000..4579fe8471
--- /dev/null
+++ b/.github/workflows/test_azure_integration.yaml
@@ -0,0 +1,91 @@
+name: Azure Delpoyment
+
+on:
+  schedule:
+    - cron: "0 0 * * MON"
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Nebari branch to deploy, test, destroy'
+        required: true
+        default: develop
+        type: string
+      image-tag:
+        description: 'Nebari image tag created by the nebari-docker-images repo'
+        required: true
+        default: main
+        type: string
+      tf-log-level:
+        description: 'Change Terraform log levels'
+        required: false
+        default: info
+        type: choice
+        options:
+        - info
+        - warn
+        - debug
+        - trace
+        - error
+
+env:
+  NEBARI_GH_BRANCH: ${{ github.event.inputs.branch || 'develop' }}
+  NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
+  TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}
+
+jobs:
+  test-azure-integration:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.NEBARI_GH_BRANCH }}
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install Nebari
+        run: |
+          pip install .[dev]
+          conda install --quiet --yes conda-build
+          playwright install
+
+      - name: Retrieve secret from Vault
+        uses: hashicorp/vault-action@v2.5.0
+        with:
+          method: jwt
+          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
+          namespace: "admin/quansight"
+          role: "repository-nebari-dev-nebari-role"
+          secrets: |
+            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci client_id | ARM_CLIENT_ID;
+            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID;
+            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID;
+            kv/data/repository/nebari-dev/nebari/cloudflare/internal-devops@quansight.com/nebari-dev-ci token | CLOUDFLARE_TOKEN;
+
+      - name: 'Azure login'
+        uses: azure/login@v2
+        with:
+          client-id: ${{ env.ARM_CLIENT_ID }}
+          tenant-id: ${{ env.ARM_TENANT_ID }}
+          subscription-id: ${{ env.ARM_SUBSCRIPTION_ID }}
+
+      - name: Integration Tests
+        run: |
+          pytest --version
+          pytest tests/tests_integration/ -vvv -s --cloud azure
+        env:
+          NEBARI_SECRET__default_images__jupyterhub: "quay.io/nebari/nebari-jupyterhub:${{ env.NEBARI_IMAGE_TAG }}"
+          NEBARI_SECRET__default_images__jupyterlab: "quay.io/nebari/nebari-jupyterlab:${{ env.NEBARI_IMAGE_TAG }}"
+          NEBARI_SECRET__default_images__dask_worker: "quay.io/nebari/nebari-dask-worker:${{ env.NEBARI_IMAGE_TAG }}"
+          ARM_CLIENT_ID: ${{ env.ARM_CLIENT_ID }}
+          ARM_TENANT_ID: ${{ env.ARM_TENANT_ID }}
+          ARM_SUBSCRIPTION_ID: ${{ env.ARM_SUBSCRIPTION_ID }}
+          ARM_USE_OIDC: "true"
+          CLOUDFLARE_TOKEN: ${{ env.CLOUDFLARE_TOKEN }}
diff --git a/.github/workflows/test_do_integration.yaml b/.github/workflows/test_do_integration.yaml
index dcfacf3175..ef0cbb2352 100644
--- a/.github/workflows/test_do_integration.yaml
+++ b/.github/workflows/test_do_integration.yaml
@@ -1,4 +1,4 @@
-name: test-gcp-integration
+name: Digital Ocean Deployment
 
 on:
   schedule:
diff --git a/.github/workflows/test_gcp_integration.yaml b/.github/workflows/test_gcp_integration.yaml
index 0418e0af40..6ba1921f41 100644
--- a/.github/workflows/test_gcp_integration.yaml
+++ b/.github/workflows/test_gcp_integration.yaml
@@ -1,4 +1,4 @@
-name: test-gcp-integration
+name: GCP Deployment
 
 on:
   schedule:
diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
new file mode 100644
index 0000000000..2a8bf120fa
--- /dev/null
+++ b/.github/workflows/trivy.yml
@@ -0,0 +1,46 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Code Scanning
+
+on:
+  push:
+    branches: [ "develop", "release/*" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "develop" ]
+  schedule:
+    - cron: '19 23 * * 6'
+
+permissions:
+  contents: read
+
+jobs:
+  SAST:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+    name: Trivy config Scan
+    runs-on: "ubuntu-20.04"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner in fs mode
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'config'
+          hide-progress: true
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          ignore-unfixed: true
+          severity: 'CRITICAL,HIGH'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 9e9dcd9147..e26093698a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -22,7 +22,7 @@ ci:
 repos:
   # general
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: end-of-file-fixer
         exclude: "^docs-sphinx/cli.html"
@@ -51,13 +51,13 @@ repos:
 
   # python
   - repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.4.2
     hooks:
       - id: black
         args: ["--line-length=88", "--exclude=/src/_nebari/template/"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.5
+    rev: v0.4.3
     hooks:
       - id: ruff
         args: ["--fix"]
@@ -73,7 +73,7 @@ repos:
 
   # terraform
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.88.4
+    rev: v1.89.1
     hooks:
       - id: terraform_fmt
         args:
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 0000000000..5294ea08d0
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,10 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - name: "Nebari Development Team"
+type: software
+title: "Nebari"
+version: 2024.4.1
+date-released: 2024-04-20
+url: "https://www.nebari.dev"
+repository-code: "https://github.com/nebari-dev/nebari"
diff --git a/README.md b/README.md
index c693dfb22f..c6a81a17c4 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,8 @@
 | :---------- | :-----|
 |   Project   | [![License](https://img.shields.io/badge/License-BSD%203--Clause-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Nebari documentation](https://img.shields.io/badge/%F0%9F%93%96%20Read-the%20docs-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://www.nebari.dev/docs/welcome) [![PyPI](https://img.shields.io/pypi/v/nebari)](https://badge.fury.io/py/nebari) [![conda version](https://img.shields.io/conda/vn/conda-forge/nebari)]((https://anaconda.org/conda-forge/nebari))  |
 |  Community  | [![GH discussions](https://img.shields.io/badge/%F0%9F%92%AC%20-Participate%20in%20discussions-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://github.com/nebari-dev/nebari/discussions) [![Open an issue](https://img.shields.io/badge/%F0%9F%93%9D%20Open-an%20issue-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://github.com/nebari-dev/nebari/issues/new/choose) [![Community guidelines](https://img.shields.io/badge/🤝%20Community-guidelines-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://www.nebari.dev/docs/community/) |
-|     CI      | [![Kubernetes Tests](https://github.com/nebari-dev/nebari/actions/workflows/test_local_integration.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/kubernetes_test.yaml) [![Tests](https://github.com/nebari-dev/nebari/actions/workflows/test.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test.yaml) [![Test Nebari Provider](https://github.com/nebari-dev/nebari/actions/workflows/test-provider.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test-provider.yaml) |
+|     CI      | [![Kubernetes Tests](https://github.com/nebari-dev/nebari/actions/workflows/test_local_integration.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/kubernetes_test.yaml) [![Tests](https://github.com/nebari-dev/nebari/actions/workflows/test.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test.yaml) [![Test Nebari Provider](https://github.com/nebari-dev/nebari/actions/workflows/test-provider.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test-provider.yaml)|
+| Cloud Providers | [![AWS Deployment Status](https://github.com/nebari-dev/nebari/actions/workflows/test_aws_integration.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test_aws_integration.yaml) [![Azure Deployment Status](https://github.com/nebari-dev/nebari/actions/workflows/test_azure_integration.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test_azure_integration.yaml) [![GCP Deployment Status](https://github.com/nebari-dev/nebari/actions/workflows/test_gcp_integration.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test_gcp_integration.yaml) [![Digital Ocean Deployment Status](https://github.com/nebari-dev/nebari/actions/workflows/test_do_integration.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test_do_integration.yaml)|
 
 ## Table of contents
 
diff --git a/RELEASE.md b/RELEASE.md
index c3ea55dda9..e1bac3acc9 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -9,6 +9,21 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ---
 
+### Release 2024.5.1 - May 13, 2024
+
+## What's Changed
+
+* make userscheduler run on general node group by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2415>
+* Upgrade to Pydantic V2 by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2348>
+* Pydantic2 PR fix by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2421>
+* remove redundant pydantic class, fix bug by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2426>
+* Update `python-keycloak` version pins constraints by @viniciusdc in <https://github.com/nebari-dev/nebari/pull/2435>
+* add HERA_TOKEN env var to user pods by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2438>
+* fix docs link by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2443>
+* Update allowed admin groups by @aktech in <https://github.com/nebari-dev/nebari/pull/2429>
+
+**Full Changelog**: <https://github.com/nebari-dev/nebari/compare/2024.4.1...2024.5.1>
+
 ## Release 2024.4.1 - April 20, 2024
 
 ### What's Changed
diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py
index 7ca8df28b4..d0e3f37444 100644
--- a/src/_nebari/constants.py
+++ b/src/_nebari/constants.py
@@ -1,4 +1,4 @@
-CURRENT_RELEASE = "2024.4.1"
+CURRENT_RELEASE = "2024.5.1"
 
 # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes
 # implemented in August 2023.
diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 0266f3aa14..cdc2fe88c7 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -112,6 +112,7 @@ class AzureInputVars(schema.Base):
     tags: Dict[str, str] = {}
     max_pods: Optional[int] = None
     network_profile: Optional[Dict[str, str]] = None
+    workload_identity_enabled: bool = False
 
 
 class AWSNodeGroupInputVars(schema.Base):
@@ -314,9 +315,9 @@ class GCPNodeGroup(schema.Base):
 
 
 DEFAULT_GCP_NODE_GROUPS = {
-    "general": GCPNodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1),
-    "user": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5),
-    "worker": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5),
+    "general": GCPNodeGroup(instance="e2-highmem-4", min_nodes=1, max_nodes=1),
+    "user": GCPNodeGroup(instance="e2-standard-4", min_nodes=0, max_nodes=5),
+    "worker": GCPNodeGroup(instance="e2-standard-4", min_nodes=0, max_nodes=5),
 }
 
 
@@ -380,6 +381,7 @@ class AzureProvider(schema.Base):
     tags: Optional[Dict[str, str]] = {}
     network_profile: Optional[Dict[str, str]] = None
     max_pods: Optional[int] = None
+    workload_identity_enabled: bool = False
 
     @model_validator(mode="before")
     @classmethod
@@ -788,6 +790,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 tags=self.config.azure.tags,
                 network_profile=self.config.azure.network_profile,
                 max_pods=self.config.azure.max_pods,
+                workload_identity_enabled=self.config.azure.workload_identity_enabled,
             ).model_dump()
         elif self.config.provider == schema.ProviderEnum.aws:
             return AWSInputVars(
diff --git a/src/_nebari/stages/infrastructure/template/azure/main.tf b/src/_nebari/stages/infrastructure/template/azure/main.tf
index 2ee687cc0f..2d6e2e2afa 100644
--- a/src/_nebari/stages/infrastructure/template/azure/main.tf
+++ b/src/_nebari/stages/infrastructure/template/azure/main.tf
@@ -40,6 +40,7 @@ module "kubernetes" {
       max_size      = config.max_nodes
     }
   ]
-  vnet_subnet_id          = var.vnet_subnet_id
-  private_cluster_enabled = var.private_cluster_enabled
+  vnet_subnet_id            = var.vnet_subnet_id
+  private_cluster_enabled   = var.private_cluster_enabled
+  workload_identity_enabled = var.workload_identity_enabled
 }
diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf
index 5f2bad6561..cd39488309 100644
--- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf
@@ -5,6 +5,10 @@ resource "azurerm_kubernetes_cluster" "main" {
   resource_group_name = var.resource_group_name
   tags                = var.tags
 
+  # To enable Azure AD Workload Identity oidc_issuer_enabled must be set to true.
+  oidc_issuer_enabled       = var.workload_identity_enabled
+  workload_identity_enabled = var.workload_identity_enabled
+
   # DNS prefix specified when creating the managed cluster. Changing this forces a new resource to be created.
   dns_prefix = "Nebari" # required
 
@@ -39,6 +43,9 @@ resource "azurerm_kubernetes_cluster" "main" {
       "azure-node-pool" = var.node_groups[0].name
     }
     tags = var.tags
+
+    # temparory_name_for_rotation must be <= 12 characters
+    temporary_name_for_rotation = "${substr(var.node_groups[0].name, 0, 9)}tmp"
   }
 
   sku_tier = "Free" # "Free" [Default] or "Paid"
diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf
index 35d7b048b9..e96187bcd6 100644
--- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf
+++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf
@@ -17,3 +17,13 @@ output "kubeconfig" {
   sensitive   = true
   value       = azurerm_kubernetes_cluster.main.kube_config_raw
 }
+
+output "cluster_oidc_issuer_url" {
+  description = "The OpenID Connect issuer URL that is associated with the AKS cluster"
+  value       = azurerm_kubernetes_cluster.main.oidc_issuer_url
+}
+
+output "resource_group_name" {
+  description = "The name of the resource group in which the AKS cluster is created"
+  value       = azurerm_kubernetes_cluster.main.resource_group_name
+}
diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf
index b7159dad9b..b93a9fae2d 100644
--- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf
+++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf
@@ -70,3 +70,9 @@ variable "max_pods" {
   type        = number
   default     = 60
 }
+
+variable "workload_identity_enabled" {
+  description = "Enable Workload Identity"
+  type        = bool
+  default     = false
+}
diff --git a/src/_nebari/stages/infrastructure/template/azure/outputs.tf b/src/_nebari/stages/infrastructure/template/azure/outputs.tf
index 352e52e3c5..d904e3ec1e 100644
--- a/src/_nebari/stages/infrastructure/template/azure/outputs.tf
+++ b/src/_nebari/stages/infrastructure/template/azure/outputs.tf
@@ -22,3 +22,13 @@ output "kubeconfig_filename" {
   description = "filename for nebari kubeconfig"
   value       = var.kubeconfig_filename
 }
+
+output "cluster_oidc_issuer_url" {
+  description = "The OpenID Connect issuer URL that is associated with the AKS cluster"
+  value       = module.kubernetes.cluster_oidc_issuer_url
+}
+
+output "resource_group_name" {
+  description = "The name of the resource group in which the AKS cluster is created"
+  value       = module.kubernetes.resource_group_name
+}
diff --git a/src/_nebari/stages/infrastructure/template/azure/variables.tf b/src/_nebari/stages/infrastructure/template/azure/variables.tf
index 4d9e6440eb..dcef2c97cb 100644
--- a/src/_nebari/stages/infrastructure/template/azure/variables.tf
+++ b/src/_nebari/stages/infrastructure/template/azure/variables.tf
@@ -76,3 +76,9 @@ variable "max_pods" {
   type        = number
   default     = 60
 }
+
+variable "workload_identity_enabled" {
+  description = "Enable Workload Identity"
+  type        = bool
+  default     = false
+}
diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index cdc1ae9151..fae8955de1 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -24,6 +24,9 @@
 TIMEOUT = 10  # seconds
 
 
+_forwardauth_middleware_name = "traefik-forward-auth"
+
+
 @schema.yaml_object(schema.yaml)
 class AccessEnum(str, enum.Enum):
     all = "all"
@@ -327,6 +330,8 @@ class KubernetesServicesInputVars(schema.Base):
     realm_id: str
     node_groups: Dict[str, Dict[str, str]]
     jupyterhub_logout_redirect_url: str = Field(alias="jupyterhub-logout-redirect-url")
+    forwardauth_middleware_name: str = _forwardauth_middleware_name
+    cert_secret_name: Optional[str] = None
 
 
 def _split_docker_image_name(image_name):
@@ -383,6 +388,7 @@ class DaskGatewayInputVars(schema.Base):
     dask_worker_image: ImageNameTag = Field(alias="dask-worker-image")
     dask_gateway_profiles: Dict[str, Any] = Field(alias="dask-gateway-profiles")
     cloud_provider: str = Field(alias="cloud-provider")
+    forwardauth_middleware_name: str = _forwardauth_middleware_name
 
 
 class MonitoringInputVars(schema.Base):
@@ -486,6 +492,11 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             realm_id=realm_id,
             node_groups=stage_outputs["stages/02-infrastructure"]["node_selectors"],
             jupyterhub_logout_redirect_url=final_logout_uri,
+            cert_secret_name=(
+                self.config.certificate.secret_name
+                if self.config.certificate.type == "existing"
+                else None
+            ),
         )
 
         conda_store_vars = CondaStoreInputVars(
diff --git a/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
index b9b0a9c6c3..fb2fdc71fc 100644
--- a/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
+++ b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf
@@ -40,4 +40,6 @@ module "dask-gateway" {
   profiles = var.dask-gateway-profiles
 
   cloud-provider = var.cloud-provider
+
+  forwardauth_middleware_name = var.forwardauth_middleware_name
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/forward-auth.tf b/src/_nebari/stages/kubernetes_services/template/forward-auth.tf
index 3cb4e827e2..2d98bf3e6a 100644
--- a/src/_nebari/stages/kubernetes_services/template/forward-auth.tf
+++ b/src/_nebari/stages/kubernetes_services/template/forward-auth.tf
@@ -5,5 +5,27 @@ module "forwardauth" {
   external-url = var.endpoint
   realm_id     = var.realm_id
 
-  node-group = var.node_groups.general
+  node-group                  = var.node_groups.general
+  forwardauth_middleware_name = var.forwardauth_middleware_name
+  cert_secret_name            = var.cert_secret_name
+}
+
+variable "forwardauth_middleware_name" {
+  description = "Name of the traefik forward auth middleware"
+  type        = string
+}
+
+variable "cert_secret_name" {
+  description = "Name of the secret containing the certificate"
+  type        = string
+}
+
+output "forward-auth-middleware" {
+  description = "middleware name for use with forward auth"
+  value       = module.forwardauth.forward-auth-middleware
+}
+
+output "forward-auth-service" {
+  description = "middleware name for use with forward auth"
+  value       = module.forwardauth.forward-auth-service
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/main.tf
index 6d9eb126ea..564d397d1a 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/main.tf
@@ -59,7 +59,19 @@ resource "kubernetes_deployment" "forwardauth-deployment" {
         node_selector = {
           "${var.node-group.key}" = var.node-group.value
         }
-
+        dynamic "volume" {
+          for_each = var.cert_secret_name == null ? [] : [1]
+          content {
+            name = "cert-volume"
+            secret {
+              secret_name = var.cert_secret_name
+              items {
+                key  = "tls.crt"
+                path = "tls.crt"
+              }
+            }
+          }
+        }
         container {
           # image = "thomseddon/traefik-forward-auth:2.2.0"
           # Use PR #159 https://github.com/thomseddon/traefik-forward-auth/pull/159
@@ -125,10 +137,26 @@ resource "kubernetes_deployment" "forwardauth-deployment" {
             value = var.external-url
           }
 
+          dynamic "env" {
+            for_each = var.cert_secret_name == null ? [] : [1]
+            content {
+              name  = "SSL_CERT_FILE"
+              value = "/config/tls.crt"
+            }
+          }
+
           port {
             container_port = 4181
           }
 
+          dynamic "volume_mount" {
+            for_each = var.cert_secret_name == null ? [] : [1]
+            content {
+              name       = "cert-volume"
+              mount_path = "/config"
+              read_only  = true
+            }
+          }
         }
 
       }
@@ -144,12 +172,12 @@ resource "kubernetes_manifest" "forwardauth-middleware" {
     apiVersion = "traefik.containo.us/v1alpha1"
     kind       = "Middleware"
     metadata = {
-      name      = "traefik-forward-auth"
+      name      = var.forwardauth_middleware_name
       namespace = var.namespace
     }
     spec = {
       forwardAuth = {
-        address = "http://forwardauth-service:4181"
+        address = "http://${kubernetes_service.forwardauth-service.metadata.0.name}:4181"
         authResponseHeaders = [
           "X-Forwarded-User"
         ]
@@ -175,7 +203,7 @@ resource "kubernetes_manifest" "forwardauth-ingressroute" {
 
           middlewares = [
             {
-              name      = "traefik-forward-auth"
+              name      = kubernetes_manifest.forwardauth-middleware.manifest.metadata.name
               namespace = var.namespace
             }
           ]
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/outputs.tf
new file mode 100644
index 0000000000..9280da29e9
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/outputs.tf
@@ -0,0 +1,13 @@
+output "forward-auth-middleware" {
+  description = "middleware name for use with forward auth"
+  value = {
+    name = kubernetes_manifest.forwardauth-middleware.manifest.metadata.name
+  }
+}
+
+output "forward-auth-service" {
+  description = "middleware name for use with forward auth"
+  value = {
+    name = kubernetes_service.forwardauth-service.metadata.0.name
+  }
+}
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/variables.tf
index 3674b1db75..ae53c5b3a1 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/variables.tf
@@ -26,3 +26,13 @@ variable "node-group" {
     value = string
   })
 }
+
+variable "forwardauth_middleware_name" {
+  description = "Name of the traefik forward auth middleware"
+  type        = string
+}
+
+variable "cert_secret_name" {
+  description = "Name of the secret containing the certificate"
+  type        = string
+}
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/middleware.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/middleware.tf
index 01680129b8..389127d06e 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/middleware.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/middleware.tf
@@ -32,7 +32,7 @@ resource "kubernetes_manifest" "chain-middleware" {
       chain = {
         middlewares = [
           {
-            name      = "traefik-forward-auth"
+            name      = var.forwardauth_middleware_name
             namespace = var.namespace
           },
           {
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf
index 7f8a4aa978..074e1214d0 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf
@@ -204,3 +204,7 @@ variable "cloud-provider" {
   description = "Name of the cloud provider to deploy to."
   type        = string
 }
+
+variable "forwardauth_middleware_name" {
+  type = string
+}
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
index c3934aad05..ea9511a4cc 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py
@@ -72,7 +72,6 @@ def service_for_jhub_apps(name, url):
                 "url": url,
                 "external": True,
             },
-            "oauth_no_confirm": True,
         }
 
     c.JupyterHub.services.extend(
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
new file mode 100644
index 0000000000..bc6fb6a721
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py
@@ -0,0 +1,238 @@
+import json
+import os
+import time
+import urllib
+from functools import reduce
+
+from jupyterhub import scopes
+from jupyterhub.traitlets import Callable
+from oauthenticator.generic import GenericOAuthenticator
+from traitlets import Bool, Unicode, Union
+
+
+class KeyCloakOAuthenticator(GenericOAuthenticator):
+    """
+    Since `oauthenticator` 16.3 `GenericOAuthenticator` supports group management.
+    This subclass adds role management on top of it, building on the new `manage_roles`
+    feature added in JupyterHub 5.0 (https://github.com/jupyterhub/jupyterhub/pull/4748).
+    """
+
+    claim_roles_key = Union(
+        [Unicode(os.environ.get("OAUTH2_ROLES_KEY", "groups")), Callable()],
+        config=True,
+        help="""As `claim_groups_key` but for roles.""",
+    )
+
+    realm_api_url = Unicode(
+        config=True, help="""The keycloak REST API URL for the realm."""
+    )
+
+    reset_managed_roles_on_startup = Bool(True)
+
+    async def update_auth_model(self, auth_model):
+        """Updates and returns the auth_model dict.
+        This function is called every time a user authenticates with JupyterHub, as in
+        every time a user login to Nebari.
+
+        It will fetch the roles and their corresponding scopes from keycloak
+        and return updated auth model which will updates roles/scopes for the
+        user. When a user's roles/scopes are updated, they take in-affect only
+        after they log in to Nebari.
+        """
+        start = time.time()
+        self.log.info("Updating user auth model")
+        auth_model = await super().update_auth_model(auth_model)
+        user_id = auth_model["auth_state"]["oauth_user"]["sub"]
+        token = await self._get_token()
+
+        jupyterhub_client_id = await self._get_jupyterhub_client_id(token=token)
+        user_info = auth_model["auth_state"][self.user_auth_state_key]
+        user_roles_from_claims = self._get_user_roles(user_info=user_info)
+        keycloak_api_call_start = time.time()
+        user_roles = await self._get_client_roles_for_user(
+            user_id=user_id, client_id=jupyterhub_client_id, token=token
+        )
+        user_roles_rich = await self._get_roles_with_attributes(
+            roles=user_roles, client_id=jupyterhub_client_id, token=token
+        )
+        keycloak_api_call_time_taken = time.time() - keycloak_api_call_start
+        user_roles_rich_names = {role["name"] for role in user_roles_rich}
+        user_roles_non_jhub_client = [
+            {"name": role}
+            for role in user_roles_from_claims
+            if role in (user_roles_from_claims - user_roles_rich_names)
+        ]
+        auth_model["roles"] = [
+            {
+                "name": role["name"],
+                "description": role.get("description"),
+                "scopes": self._get_scope_from_role(role),
+            }
+            for role in [*user_roles_rich, *user_roles_non_jhub_client]
+        ]
+        # note: because the roles check is comprehensive, we need to re-add the admin and user roles
+        if auth_model["admin"]:
+            auth_model["roles"].append({"name": "admin"})
+        if await self.check_allowed(auth_model["name"], auth_model):
+            auth_model["roles"].append({"name": "user"})
+        execution_time = time.time() - start
+        self.log.info(
+            f"Auth model update complete, time taken: {execution_time}s "
+            f"time taken for keycloak api call: {keycloak_api_call_time_taken}s "
+            f"delta between full execution and keycloak call: {execution_time - keycloak_api_call_time_taken}s"
+        )
+        return auth_model
+
+    async def _get_jupyterhub_client_roles(self, jupyterhub_client_id, token):
+        """Get roles for the client named 'jupyterhub'."""
+        # Includes roles like "jupyterhub_admin", "jupyterhub_developer", "dask_gateway_developer"
+
+        client_roles = await self._fetch_api(
+            endpoint=f"clients/{jupyterhub_client_id}/roles", token=token
+        )
+        client_roles_rich = await self._get_roles_with_attributes(
+            client_roles, client_id=jupyterhub_client_id, token=token
+        )
+        return client_roles_rich
+
+    async def _get_jupyterhub_client_id(self, token):
+        # Get the clients list to find the "id" of "jupyterhub" client.
+        clients_data = await self._fetch_api(endpoint="clients/", token=token)
+        jupyterhub_clients = [
+            client for client in clients_data if client["clientId"] == "jupyterhub"
+        ]
+        assert len(jupyterhub_clients) == 1
+        jupyterhub_client_id = jupyterhub_clients[0]["id"]
+        return jupyterhub_client_id
+
+    async def load_managed_roles(self):
+        self.log.info("Loading managed roles")
+        if not self.manage_roles:
+            raise ValueError(
+                "Managed roles can only be loaded when `manage_roles` is True"
+            )
+        token = await self._get_token()
+        jupyterhub_client_id = await self._get_jupyterhub_client_id(token=token)
+        client_roles_rich = await self._get_jupyterhub_client_roles(
+            jupyterhub_client_id=jupyterhub_client_id, token=token
+        )
+        # Includes roles like "default-roles-nebari", "offline_access", "uma_authorization"
+        realm_roles = await self._fetch_api(endpoint="roles", token=token)
+        roles = {
+            role["name"]: {
+                "name": role["name"],
+                "description": role["description"],
+                "scopes": self._get_scope_from_role(role),
+            }
+            for role in [*realm_roles, *client_roles_rich]
+        }
+        # we could use either `name` (e.g. "developer") or `path` ("/developer");
+        # since the default claim key returns `path`, it seems preferable.
+        group_name_key = "path"
+        for realm_role in realm_roles:
+            role_name = realm_role["name"]
+            role = roles[role_name]
+            # fetch role assignments to groups
+            groups = await self._fetch_api(f"roles/{role_name}/groups", token=token)
+            role["groups"] = [group[group_name_key] for group in groups]
+            # fetch role assignments to users
+            users = await self._fetch_api(f"roles/{role_name}/users", token=token)
+            role["users"] = [user["username"] for user in users]
+        for client_role in client_roles_rich:
+            role_name = client_role["name"]
+            role = roles[role_name]
+            # fetch role assignments to groups
+            groups = await self._fetch_api(
+                f"clients/{jupyterhub_client_id}/roles/{role_name}/groups", token=token
+            )
+            role["groups"] = [group[group_name_key] for group in groups]
+            # fetch role assignments to users
+            users = await self._fetch_api(
+                f"clients/{jupyterhub_client_id}/roles/{role_name}/users", token=token
+            )
+            role["users"] = [user["username"] for user in users]
+
+        return list(roles.values())
+
+    def _get_scope_from_role(self, role):
+        """Return scopes from role if the component is jupyterhub"""
+        role_scopes = role.get("attributes", {}).get("scopes", [])
+        component = role.get("attributes", {}).get("component")
+        # Attributes are returned as a single-element array, unless `##` delimiter is used in Keycloak
+        # See this: https://stackoverflow.com/questions/68954733/keycloak-client-role-attribute-array
+        if component == ["jupyterhub"] and role_scopes:
+            return self.validate_scopes(role_scopes[0].split(","))
+        else:
+            return []
+
+    def validate_scopes(self, role_scopes):
+        """Validate role scopes to sanity check user provided scopes from keycloak"""
+        self.log.info(f"Validating role scopes: {role_scopes}")
+        try:
+            # This is not a public function, but there isn't any alternative
+            # method to verify scopes, and we do need to do this sanity check
+            # as a invalid scopes could cause hub pod to fail
+            scopes._check_scopes_exist(role_scopes)
+            return role_scopes
+        except scopes.ScopeNotFound as e:
+            self.log.error(f"Invalid scopes, skipping: {role_scopes} ({e})")
+        return []
+
+    async def _get_roles_with_attributes(self, roles: dict, client_id: str, token: str):
+        """This fetches all roles by id to fetch there attributes."""
+        roles_rich = []
+        for role in roles:
+            # If this takes too much time, which isn't the case right now, we can
+            # also do multi-threaded requests
+            role_rich = await self._fetch_api(
+                endpoint=f"roles-by-id/{role['id']}?client={client_id}", token=token
+            )
+            roles_rich.append(role_rich)
+        return roles_rich
+
+    async def _get_client_roles_for_user(self, user_id, client_id, token):
+        user_roles = await self._fetch_api(
+            endpoint=f"users/{user_id}/role-mappings/clients/{client_id}/composite",
+            token=token,
+        )
+        return user_roles
+
+    def _get_user_roles(self, user_info):
+        if callable(self.claim_roles_key):
+            return set(self.claim_roles_key(user_info))
+        try:
+            return set(reduce(dict.get, self.claim_roles_key.split("."), user_info))
+        except TypeError:
+            self.log.error(
+                f"The claim_roles_key {self.claim_roles_key} does not exist in the user token"
+            )
+            return set()
+
+    async def _get_token(self) -> str:
+        http = self.http_client
+
+        body = urllib.parse.urlencode(
+            {
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "grant_type": "client_credentials",
+            }
+        )
+        response = await http.fetch(
+            self.token_url,
+            method="POST",
+            body=body,
+        )
+        data = json.loads(response.body)
+        return data["access_token"]  # type: ignore[no-any-return]
+
+    async def _fetch_api(self, endpoint: str, token: str):
+        response = await self.http_client.fetch(
+            f"{self.realm_api_url}/{endpoint}",
+            method="GET",
+            headers={"Authorization": f"Bearer {token}"},
+        )
+        return json.loads(response.body)
+
+
+c.JupyterHub.authenticator_class = KeyCloakOAuthenticator
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index cf86d5a03e..fe7716cf88 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -57,7 +57,7 @@ resource "helm_release" "jupyterhub" {
 
   repository = "https://jupyterhub.github.io/helm-chart/"
   chart      = "jupyterhub"
-  version    = "3.2.1"
+  version    = "4.0.0-0.dev.git.6619.hd126b1bd"
 
   values = concat([
     file("${path.module}/values.yaml"),
@@ -130,6 +130,7 @@ resource "helm_release" "jupyterhub" {
           "01-theme.py"    = file("${path.module}/files/jupyterhub/01-theme.py")
           "02-spawner.py"  = file("${path.module}/files/jupyterhub/02-spawner.py")
           "03-profiles.py" = file("${path.module}/files/jupyterhub/03-profiles.py")
+          "04-auth.py"     = file("${path.module}/files/jupyterhub/04-auth.py")
         }
 
         services = {
@@ -143,25 +144,25 @@ resource "helm_release" "jupyterhub" {
         # for simple key value configuration with jupyterhub traitlets
         # this hub.config property should be used
         config = {
-          JupyterHub = {
-            authenticator_class = "generic-oauth"
-          }
           Authenticator = {
             enable_auth_state = true
           }
-          GenericOAuthenticator = {
+          KeyCloakOAuthenticator = {
             client_id            = module.jupyterhub-openid-client.config.client_id
             client_secret        = module.jupyterhub-openid-client.config.client_secret
             oauth_callback_url   = "https://${var.external-url}/hub/oauth_callback"
             authorize_url        = module.jupyterhub-openid-client.config.authentication_url
             token_url            = module.jupyterhub-openid-client.config.token_url
             userdata_url         = module.jupyterhub-openid-client.config.userinfo_url
+            realm_api_url        = module.jupyterhub-openid-client.config.realm_api_url
             login_service        = "Keycloak"
             username_claim       = "preferred_username"
             claim_groups_key     = "groups"
-            allowed_groups       = ["/analyst", "/developer", "/admin"]
-            admin_groups         = ["/admin"]
+            claim_roles_key      = "roles"
+            allowed_groups       = ["/analyst", "/developer", "/admin", "jupyterhub_admin", "jupyterhub_developer"]
+            admin_groups         = ["/admin", "jupyterhub_admin"]
             manage_groups        = true
+            manage_roles         = true
             refresh_pre_spawn    = true
             validate_server_cert = false
 
@@ -283,6 +284,10 @@ module "jupyterhub-openid-client" {
     var.jupyterhub-logout-redirect-url
   ]
   jupyterlab_profiles_mapper = true
+  service-accounts-enabled   = true
+  service-account-roles = [
+    "view-realm", "view-users", "view-clients"
+  ]
 }
 
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
index fd85eeb7a0..7a2c3e648d 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf
@@ -15,7 +15,8 @@ resource "keycloak_openid_client" "main" {
   access_type           = "CONFIDENTIAL"
   standard_flow_enabled = true
 
-  valid_redirect_uris = var.callback-url-paths
+  valid_redirect_uris      = var.callback-url-paths
+  service_accounts_enabled = var.service-accounts-enabled
 }
 
 
@@ -62,6 +63,33 @@ resource "keycloak_openid_user_attribute_protocol_mapper" "jupyterlab_profiles"
   aggregate_attributes = true
 }
 
+data "keycloak_realm" "master" {
+  realm = "nebari"
+}
+
+data "keycloak_openid_client" "realm_management" {
+  realm_id  = var.realm_id
+  client_id = "realm-management"
+}
+
+data "keycloak_role" "main-service" {
+  for_each = toset(var.service-account-roles)
+
+  realm_id  = data.keycloak_realm.master.id
+  client_id = data.keycloak_openid_client.realm_management.id
+  name      = each.key
+}
+
+resource "keycloak_openid_client_service_account_role" "main" {
+  for_each = toset(var.service-account-roles)
+
+  realm_id                = var.realm_id
+  service_account_user_id = keycloak_openid_client.main.service_account_user_id
+  client_id               = data.keycloak_openid_client.realm_management.id
+  role                    = data.keycloak_role.main-service[each.key].name
+}
+
+
 resource "keycloak_role" "main" {
   for_each = toset(flatten(values(var.role_mapping)))
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
index bd1978bd4b..6077c22b0e 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf
@@ -1,12 +1,14 @@
 output "config" {
   description = "configuration credentials for connecting to openid client"
   value = {
-    client_id     = keycloak_openid_client.main.client_id
-    client_secret = keycloak_openid_client.main.client_secret
+    client_id               = keycloak_openid_client.main.client_id
+    client_secret           = keycloak_openid_client.main.client_secret
+    service_account_user_id = keycloak_openid_client.main.service_account_user_id
 
     authentication_url = "https://${var.external-url}/auth/realms/${var.realm_id}/protocol/openid-connect/auth"
     token_url          = "https://${var.external-url}/auth/realms/${var.realm_id}/protocol/openid-connect/token"
     userinfo_url       = "https://${var.external-url}/auth/realms/${var.realm_id}/protocol/openid-connect/userinfo"
+    realm_api_url      = "https://${var.external-url}/auth/admin/realms/${var.realm_id}"
     callback_urls      = var.callback-url-paths
   }
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
index d20ecca48a..b4e709c6a5 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf
@@ -16,6 +16,19 @@ variable "external-url" {
 }
 
 
+variable "service-accounts-enabled" {
+  description = "Whether the client should have a service account created"
+  type        = bool
+  default     = false
+}
+
+variable "service-account-roles" {
+  description = "Roles to be granted to the service account. Requires setting service-accounts-enabled to true."
+  type        = list(string)
+  default     = []
+}
+
+
 variable "role_mapping" {
   description = "Group to role mapping to establish for client"
   type        = map(list(string))
diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py
index eaaf131117..b589f5fb8f 100644
--- a/src/_nebari/stages/nebari_tf_extensions/__init__.py
+++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py
@@ -72,6 +72,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 "stages/05-kubernetes-keycloak"
             ]["keycloak_nebari_bot_password"]["value"],
             "helm_extensions": [_.model_dump() for _ in self.config.helm_extensions],
+            "forwardauth_middleware_name": stage_outputs[
+                "stages/07-kubernetes-services"
+            ]["forward-auth-middleware"]["value"]["name"],
         }
 
 
diff --git a/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/locals.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/locals.tf
index 4c5f0de3e7..b3616d4d29 100644
--- a/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/locals.tf
+++ b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/locals.tf
@@ -1,6 +1,6 @@
 locals {
   middlewares = (var.private) ? ([{
-    name      = "traefik-forward-auth"
+    name      = var.forwardauth_middleware_name
     namespace = var.namespace
   }]) : ([])
 
diff --git a/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/variables.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/variables.tf
index 071c11ffbd..9a255ff5e1 100644
--- a/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/variables.tf
+++ b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/variables.tf
@@ -70,3 +70,8 @@ variable "keycloak_nebari_bot_password" {
   type        = string
   default     = ""
 }
+
+variable "forwardauth_middleware_name" {
+  description = "Name of the traefik forward auth middleware"
+  type        = string
+}
diff --git a/src/_nebari/stages/nebari_tf_extensions/template/tf-extensions.tf b/src/_nebari/stages/nebari_tf_extensions/template/tf-extensions.tf
index dd87639393..915b78879e 100644
--- a/src/_nebari/stages/nebari_tf_extensions/template/tf-extensions.tf
+++ b/src/_nebari/stages/nebari_tf_extensions/template/tf-extensions.tf
@@ -16,6 +16,7 @@ module "extension" {
   nebari-realm-id  = var.realm_id
 
   keycloak_nebari_bot_password = each.value.keycloakadmin ? var.keycloak_nebari_bot_password : ""
+  forwardauth_middleware_name  = var.forwardauth_middleware_name
 
   envs = lookup(each.value, "envs", [])
 }
diff --git a/src/_nebari/stages/nebari_tf_extensions/template/variables.tf b/src/_nebari/stages/nebari_tf_extensions/template/variables.tf
index 144a6049cb..e17d86ffca 100644
--- a/src/_nebari/stages/nebari_tf_extensions/template/variables.tf
+++ b/src/_nebari/stages/nebari_tf_extensions/template/variables.tf
@@ -31,3 +31,8 @@ variable "helm_extensions" {
 variable "keycloak_nebari_bot_password" {
   description = "Keycloak password for nebari-bot"
 }
+
+variable "forwardauth_middleware_name" {
+  description = "Name of the traefik forward auth middleware"
+  type        = string
+}
diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 64e593be66..e35d7ea309 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -3,6 +3,7 @@
 import re
 import secrets
 import string
+import textwrap
 from abc import ABC
 from pathlib import Path
 from typing import Any, ClassVar, Dict
@@ -773,6 +774,89 @@ def _version_specific_upgrade(
         return config
 
 
+class Upgrade_2024_5_1(UpgradeStep):
+    version = "2024.5.1"
+
+    def _version_specific_upgrade(
+        self, config, start_version, config_filename: Path, *args, **kwargs
+    ):
+        rich.print("Ready to upgrade to Nebari version [green]2024.5.1[/green].")
+
+        return config
+
+
+class Upgrade_2024_6_1(UpgradeStep):
+    version = "2024.6.1"
+
+    def _version_specific_upgrade(
+        self, config, start_version, config_filename: Path, *args, **kwargs
+    ):
+        if (provider := config.get("provider", "")) == ProviderEnum.gcp.value:
+            provider_full_name = provider_enum_name_map[provider]
+            if not config.get(provider_full_name, {}).get("node_groups", {}):
+                try:
+                    text = textwrap.dedent(
+                        f"""
+                        The default node groups for GCP have been changed to cost efficient e2 family nodes reducing the running cost of Nebari on GCP by ~50%.
+                        This change will affect your current deployment, and will result in ~15 minutes of downtime during the upgrade step as the node groups are switched out, but shouldn't result in data loss.
+
+                        As always, make sure to backup data before upgrading.  See https://www.nebari.dev/docs/how-tos/manual-backup for more information.
+
+                        Would you like to upgrade to the cost effective node groups [purple]{config_filename}[/purple]?
+                        If not, select "N" and the old default node groups will be added to the nebari config file.
+                    """
+                    )
+                    continue_ = Prompt.ask(
+                        text,
+                        choices=["y", "N"],
+                        default="y",
+                    )
+                    if continue_ == "N":
+                        config[provider_full_name]["node_groups"] = {
+                            "general": {
+                                "instance": "n1-standard-8",
+                                "min_nodes": 1,
+                                "max_nodes": 1,
+                            },
+                            "user": {
+                                "instance": "n1-standard-4",
+                                "min_nodes": 0,
+                                "max_nodes": 5,
+                            },
+                            "worker": {
+                                "instance": "n1-standard-4",
+                                "min_nodes": 0,
+                                "max_nodes": 5,
+                            },
+                        }
+                except KeyError:
+                    pass
+            else:
+                text = textwrap.dedent(
+                    """
+                    The default node groups for GCP have been changed to cost efficient e2 family nodes reducing the running cost of Nebari on GCP by ~50%.
+                    Consider upgrading your node group instance types to the new default configuration.
+
+                    Upgrading your general node will result in ~15 minutes of downtime during the upgrade step as the node groups are switched out, but shouldn't result in data loss.
+
+                    As always, make sure to backup data before upgrading.  See https://www.nebari.dev/docs/how-tos/manual-backup for more information.
+
+                    The new default node groups instances are:
+                """
+                )
+                text += json.dumps(
+                    {
+                        "general": {"instance": "e2-highmem-4"},
+                        "user": {"instance": "e2-standard-4"},
+                        "worker": {"instance": "e2-standard-4"},
+                    },
+                    indent=4,
+                )
+                text += "\n\nHit enter to continue"
+                Prompt.ask(text)
+        return config
+
+
 __rounded_version__ = str(rounded_ver_parse(__version__))
 
 # Manually-added upgrade steps must go above this line
diff --git a/tests/common/navigator.py b/tests/common/navigator.py
index 12a1445bd5..f846d9a545 100644
--- a/tests/common/navigator.py
+++ b/tests/common/navigator.py
@@ -256,7 +256,7 @@ def reset_workspace(self):
             self._set_environment_via_popup(kernel=None)
 
         # go to Kernel menu
-        kernel_menuitem = self.page.get_by_text("Kernel", exact=True)
+        kernel_menuitem = self.page.get_by_role("menuitem", name="Kernel", exact=True)
         kernel_menuitem.click()
         # shut down multiple running kernels
         with contextlib.suppress(Exception):
@@ -320,14 +320,23 @@ def _set_environment_via_popup(self, kernel=None):
             # failure here indicates that the environment doesn't exist either
             # because of incorrect naming syntax or because the env is still
             # being built
-            self.page.get_by_role("combobox").nth(1).select_option(kernel)
-            # click Select to close popup (deal with the two formats of this dialog)
-            try:
-                self.page.get_by_role("button", name="Select Kernel").click()
-            except Exception:
-                self.page.locator("div").filter(has_text="No KernelSelect").get_by_role(
-                    "button", name="Select Kernel"
-                ).click()
+
+            new_launcher_popup = self.page.locator(
+                ".jp-KernelSelector-Dialog .jp-NewLauncher-table table"
+            ).nth(0)
+            if new_launcher_popup.is_visible():
+                # for when the jupyterlab-new-launcher extension is installed
+                new_launcher_popup.locator("td").nth(0).click()
+            else:
+                # for when only the native launcher is available
+                self.page.get_by_role("combobox").nth(1).select_option(kernel)
+                # click Select to close popup (deal with the two formats of this dialog)
+                try:
+                    self.page.get_by_role("button", name="Select Kernel").click()
+                except Exception:
+                    self.page.locator("div").filter(
+                        has_text="No KernelSelect"
+                    ).get_by_role("button", name="Select Kernel").click()
 
     def set_environment(self, kernel):
         """Set environment of a jupyter notebook.
@@ -350,7 +359,7 @@ def set_environment(self, kernel):
         popup = self._check_for_kernel_popup()
         # if there is not a kernel popup, make it appear
         if not popup:
-            self.page.get_by_text("Kernel", exact=True).click()
+            self.page.get_by_role("menuitem", name="Kernel", exact=True).click()
             self.page.get_by_role("menuitem", name="Change Kernel…").get_by_text(
                 "Change Kernel…"
             ).click()
diff --git a/tests/common/run_notebook.py b/tests/common/run_notebook.py
index 10d28d6637..019fd26710 100644
--- a/tests/common/run_notebook.py
+++ b/tests/common/run_notebook.py
@@ -212,7 +212,7 @@ def _get_outputs(self) -> List[str]:
 
     def _restart_run_all(self):
         # restart run all cells
-        self.nav.page.get_by_text("Kernel", exact=True).click()
+        self.nav.page.get_by_role("menuitem", name="Kernel", exact=True).click()
         self.nav.page.get_by_role(
             "menuitem", name="Restart Kernel and Run All Cells…"
         ).get_by_text("Restart Kernel and Run All Cells…").click()
diff --git a/tests/tests_deployment/conftest.py b/tests/tests_deployment/conftest.py
new file mode 100644
index 0000000000..fa71302823
--- /dev/null
+++ b/tests/tests_deployment/conftest.py
@@ -0,0 +1,11 @@
+import pytest
+
+from tests.tests_deployment.keycloak_utils import delete_client_keycloak_test_roles
+
+
+@pytest.fixture()
+def cleanup_keycloak_roles():
+    # setup
+    yield
+    # teardown
+    delete_client_keycloak_test_roles(client_name="jupyterhub")
diff --git a/tests/tests_deployment/keycloak_utils.py b/tests/tests_deployment/keycloak_utils.py
new file mode 100644
index 0000000000..6e6f6c21e6
--- /dev/null
+++ b/tests/tests_deployment/keycloak_utils.py
@@ -0,0 +1,96 @@
+import os
+import pathlib
+
+from _nebari.config import read_configuration
+from _nebari.keycloak import get_keycloak_admin_from_config
+from nebari.plugins import nebari_plugin_manager
+
+
+def get_keycloak_client_details_by_name(client_name, keycloak_admin=None):
+    if not keycloak_admin:
+        keycloak_admin = get_keycloak_admin()
+    clients = keycloak_admin.get_clients()
+    for client in clients:
+        if client["clientId"] == client_name:
+            return client
+
+
+def get_keycloak_user_details_by_name(username, keycloak_admin=None):
+    if not keycloak_admin:
+        keycloak_admin = get_keycloak_admin()
+    users = keycloak_admin.get_users()
+    for user in users:
+        if user["username"] == username:
+            return user
+
+
+def get_keycloak_role_details_by_name(roles, role_name):
+    for role in roles:
+        if role["name"] == role_name:
+            return role
+
+
+def get_keycloak_admin():
+    config_schema = nebari_plugin_manager.config_schema
+    config_filepath = os.environ.get("NEBARI_CONFIG_PATH", "nebari-config.yaml")
+    assert pathlib.Path(config_filepath).exists()
+    config = read_configuration(config_filepath, config_schema)
+    return get_keycloak_admin_from_config(config)
+
+
+def create_keycloak_client_role(
+    client_id: str, role_name: str, scopes: str, component: str
+):
+    keycloak_admin = get_keycloak_admin()
+    keycloak_admin.create_client_role(
+        client_id,
+        payload={
+            "name": role_name,
+            "description": f"{role_name} description",
+            "attributes": {"scopes": [scopes], "component": [component]},
+        },
+    )
+    client_roles = keycloak_admin.get_client_roles(client_id=client_id)
+    return get_keycloak_role_details_by_name(client_roles, role_name)
+
+
+def assign_keycloak_client_role_to_user(username: str, client_name: str, role: dict):
+    """Given a keycloak role and client name, assign that to the user"""
+    keycloak_admin = get_keycloak_admin()
+    user_details = get_keycloak_user_details_by_name(
+        username=username, keycloak_admin=keycloak_admin
+    )
+    client_details = get_keycloak_client_details_by_name(
+        client_name=client_name, keycloak_admin=keycloak_admin
+    )
+    keycloak_admin.assign_client_role(
+        user_id=user_details["id"], client_id=client_details["id"], roles=[role]
+    )
+
+
+def create_keycloak_role(client_name: str, role_name: str, scopes: str, component: str):
+    """Create a role keycloak role for the given client with scopes and
+    component set in attributes
+    """
+    keycloak_admin = get_keycloak_admin()
+    client_details = get_keycloak_client_details_by_name(
+        client_name=client_name, keycloak_admin=keycloak_admin
+    )
+    return create_keycloak_client_role(
+        client_details["id"], role_name=role_name, scopes=scopes, component=component
+    )
+
+
+def delete_client_keycloak_test_roles(client_name):
+    keycloak_admin = get_keycloak_admin()
+    client_details = get_keycloak_client_details_by_name(
+        client_name=client_name, keycloak_admin=keycloak_admin
+    )
+    client_roles = keycloak_admin.get_client_roles(client_id=client_details["id"])
+    for role in client_roles:
+        if not role["name"].startswith("test"):
+            continue
+        keycloak_admin.delete_client_role(
+            client_role_id=client_details["id"],
+            role_name=role["name"],
+        )
diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py
new file mode 100644
index 0000000000..5e1a54562b
--- /dev/null
+++ b/tests/tests_deployment/test_jupyterhub_api.py
@@ -0,0 +1,94 @@
+import pytest
+
+from tests.tests_deployment import constants
+from tests.tests_deployment.keycloak_utils import (
+    assign_keycloak_client_role_to_user,
+    create_keycloak_role,
+)
+from tests.tests_deployment.utils import create_jupyterhub_token, get_jupyterhub_session
+
+
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
+def test_jupyterhub_loads_roles_from_keycloak():
+    session = get_jupyterhub_session()
+    xsrf_token = session.cookies.get("_xsrf")
+    response = session.get(
+        f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{constants.KEYCLOAK_USERNAME}",
+        headers={"X-XSRFToken": xsrf_token},
+        verify=False,
+    )
+    user = response.json()
+    assert set(user["roles"]) == {
+        "user",
+        "manage-account",
+        "jupyterhub_developer",
+        "argo-developer",
+        "dask_gateway_developer",
+        "grafana_viewer",
+        "conda_store_developer",
+        "argo-viewer",
+        "grafana_developer",
+        "manage-account-links",
+        "view-profile",
+    }
+
+
+@pytest.mark.parametrize(
+    "component,scopes,expected_scopes_difference",
+    (
+        [
+            "jupyterhub",
+            "read:users:shares,read:groups:shares,users:shares",
+            {"read:groups:shares", "users:shares", "read:users:shares"},
+        ],
+        ["invalid-component", "read:users:shares,read:groups:shares,users:shares", {}],
+        ["invalid-component", "admin:invalid-scope", {}],
+    ),
+)
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
+@pytest.mark.filterwarnings(
+    "ignore:.*auto_refresh_token is deprecated:DeprecationWarning"
+)
+def test_keycloak_roles_attributes_parsed_as_jhub_scopes(
+    component, scopes, expected_scopes_difference, cleanup_keycloak_roles
+):
+    # check token scopes before role creation and assignment
+    token_response_before = create_jupyterhub_token(
+        note="before-role-creation-and-assignment"
+    )
+    token_scopes_before = set(token_response_before.json()["scopes"])
+    # create keycloak role with jupyterhub scopes in attributes
+    role = create_keycloak_role(
+        client_name="jupyterhub",
+        # Note: we're clearing this role after every test case, and we're clearing
+        # it by name, so it must start with test- to be deleted afterward
+        role_name="test-custom-role",
+        scopes=scopes,
+        component=component,
+    )
+    assert role
+    # assign created role to the user
+    assign_keycloak_client_role_to_user(
+        constants.KEYCLOAK_USERNAME, client_name="jupyterhub", role=role
+    )
+    token_response_after = create_jupyterhub_token(
+        note="after-role-creation-and-assignment"
+    )
+    token_scopes_after = set(token_response_after.json()["scopes"])
+    # verify new scopes added/removed
+    expected_scopes_difference = token_scopes_after - token_scopes_before
+    # Comparing token scopes for the user before and after role assignment
+    assert expected_scopes_difference == expected_scopes_difference
+
+
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
+def test_jupyterhub_loads_groups_from_keycloak():
+    session = get_jupyterhub_session()
+    xsrf_token = session.cookies.get("_xsrf")
+    response = session.get(
+        f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{constants.KEYCLOAK_USERNAME}",
+        headers={"X-XSRFToken": xsrf_token},
+        verify=False,
+    )
+    user = response.json()
+    assert set(user["groups"]) == {"/analyst", "/developer", "/users"}
diff --git a/tests/tests_deployment/utils.py b/tests/tests_deployment/utils.py
index d175a2dd05..b0965dd1ae 100644
--- a/tests/tests_deployment/utils.py
+++ b/tests/tests_deployment/utils.py
@@ -26,20 +26,24 @@ def get_jupyterhub_session():
     return session
 
 
-def get_jupyterhub_token(note="jupyterhub-tests-deployment"):
+def create_jupyterhub_token(note):
     session = get_jupyterhub_session()
     xsrf_token = session.cookies.get("_xsrf")
     headers = {"Referer": f"https://{constants.NEBARI_HOSTNAME}/hub/token"}
     if xsrf_token:
         headers["X-XSRFToken"] = xsrf_token
     data = {"note": note, "expires_in": None}
-    r = session.post(
+    return session.post(
         f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{constants.KEYCLOAK_USERNAME}/tokens",
         headers=headers,
         json=data,
+        verify=False,
     )
 
-    return r.json()["token"]
+
+def get_jupyterhub_token(note="jupyterhub-tests-deployment"):
+    response = create_jupyterhub_token(note=note)
+    return response.json()["token"]
 
 
 def monkeypatch_ssl_context():
diff --git a/tests/tests_e2e/cypress/integration/main.js b/tests/tests_e2e/cypress/integration/main.js
index 1184ba76d6..e25d60fd8d 100644
--- a/tests/tests_e2e/cypress/integration/main.js
+++ b/tests/tests_e2e/cypress/integration/main.js
@@ -52,8 +52,8 @@ describe('First Test', () => {
         cy.get('h1')
         .should('contain', 'Server Options');
 
-      cy.get('input.btn.btn-jupyter')
-        .should('have.attr', 'value', 'Start').click();
+      cy.get('button.btn.btn-jupyter')
+        .should('contain', 'Start').click();
 
       // Minimal check that JupyterLab has opened
       cy.get('div#jp-MainLogo', { timeout: 60000 }).should('exist').wait(4000);