Merge branch 'develop' into typing-hookspecs

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -104,7 +104,6 @@ body:
       description: Is this issue related to any of the Nebari integrations?
       multiple: true
       options:
-        - "Prefect"
         - "Keycloak"
         - "conda-store"
         - "Dask"

.github/ISSUE_TEMPLATE/release-checklist.md

@@ -39,6 +39,8 @@ Release captain responsible - <@gh_username>
   - [Do we need to update the `dask` versions in the `nebari-dask`?](https://github.com/conda-forge/nebari-dask-feedstock/blob/main/recipe/meta.yaml#L13-L16)
   - Will there be an accompanying blog post?
 - [ ] Prepare for the release.
+  - [ ] Update the [`nebari upgrade`](https://github.com/nebari-dev/nebari/blob/develop/src/_nebari/upgrade.py) for this release
+    - [ ] Add upgrade messaging including deprecation warnings, version specific warnings and so on.
   - [ ] Announce build freeze.
   - [ ] Release Candidate (RC) cycle.
     - Is this a hotfix?
@@ -63,6 +65,7 @@ _These steps must be actioned in the order they appear in this checklist._
 
 - [ ] [Tag, build and push docker images](https://github.com/nebari-dev/nebari-docker-images/releases/new)
 - [ ] [Update and cut release for `nebari-dask` meta package on Conda-Forge.](https://github.com/conda-forge/nebari-dask-feedstock)
+- [ ] Update `CURRENT_RELEASE` (and any other tags) in the [`constants.py`](https://github.com/nebari-dev/nebari/blob/develop/src/_nebari/constants.py#L1)
 - [ ] [Cut PyPI release via GHA release workflow.](https://github.com/nebari-dev/nebari/releases/new)
   - Avoid appending `v` to tag.
     - Copy release notes from `RELEASE.md`.

.github/workflows/test_helm_charts.yaml

@@ -7,9 +7,14 @@ on:
   schedule:
     # Run every Monday at 13:00 UTC
     - cron: "0 13 * * 1"
+  pull_request:
+    paths:
+      - ".github/workflows/test_helm_charts.yaml"
+      - "scripts/helm-validate.py"
   push:
     paths:
       - ".github/workflows/test_helm_charts.yaml"
+      - "scripts/helm-validate.py"
   workflow_dispatch:
 
 jobs:

.pre-commit-config.yaml

@@ -30,8 +30,6 @@ repos:
         exclude: "^docs-sphinx/cli.html"
       - id: check-json
       - id: check-yaml
-        # jinja2 templates for helm charts
-        exclude: "src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/(clearml/chart/templates/.*|prefect/chart/templates/.*)"
         args: [--allow-multiple-documents]
       - id: check-toml
       # Lint: Checks that non-binary executables have a proper shebang.
@@ -53,13 +51,13 @@ repos:
 
   # python
   - repo: https://github.com/psf/black
-    rev: 23.10.1
+    rev: 23.11.0
     hooks:
       - id: black
         args: ["--line-length=88", "--exclude=/src/_nebari/template/"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.4
+    rev: v0.1.6
     hooks:
       - id: ruff
         args: ["--fix"]
@@ -75,7 +73,7 @@ repos:
 
   # terraform
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.83.5
+    rev: v1.83.6
     hooks:
       - id: terraform_fmt
         args:

README.md

@@ -14,7 +14,7 @@
 | :---------- | :-----|
 |   Project   | [![License](https://img.shields.io/badge/License-BSD%203--Clause-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Nebari documentation](https://img.shields.io/badge/%F0%9F%93%96%20Read-the%20docs-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://www.nebari.dev/docs/welcome) [![PyPI](https://img.shields.io/pypi/v/nebari)](https://badge.fury.io/py/nebari) [![conda version](https://img.shields.io/conda/vn/conda-forge/nebari)]((https://anaconda.org/conda-forge/nebari))  |
 |  Community  | [![GH discussions](https://img.shields.io/badge/%F0%9F%92%AC%20-Participate%20in%20discussions-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://github.com/nebari-dev/nebari/discussions) [![Open an issue](https://img.shields.io/badge/%F0%9F%93%9D%20Open-an%20issue-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://github.com/nebari-dev/nebari/issues/new/choose) [![Community guidelines](https://img.shields.io/badge/🤝%20Community-guidelines-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://www.nebari.dev/docs/community/) |
-|     CI      | [![Kubernetes Tests](https://github.com/nebari-dev/nebari/actions/workflows/kubernetes_test.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/kubernetes_test.yaml) [![Tests](https://github.com/nebari-dev/nebari/actions/workflows/test.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test.yaml) [![Test Nebari Provider](https://github.com/nebari-dev/nebari/actions/workflows/test-provider.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test-provider.yaml) |
+|     CI      | [![Kubernetes Tests](https://github.com/nebari-dev/nebari/actions/workflows/test_local_integration.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/kubernetes_test.yaml) [![Tests](https://github.com/nebari-dev/nebari/actions/workflows/test.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test.yaml) [![Test Nebari Provider](https://github.com/nebari-dev/nebari/actions/workflows/test-provider.yaml/badge.svg)](https://github.com/nebari-dev/nebari/actions/workflows/test-provider.yaml) |
 
 ## Table of contents
 

RELEASE.md

@@ -12,6 +12,39 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 ## Upcoming Release
 
 
+## Release 2023.11.1 - November 15, 2023
+
+### Feature changes and enhancements
+
+* Upgrade conda-store to latest version 2023  .10.1
+* Minor improvements and bug fixes
+
+### Breaking Changes
+
+> WARNING: Prefect, ClearML and kbatch were removed in this release and upgrading to this version will result in all of them being uninstalled.
+
+### What's Changed
+* BUG: fix incorrect config override #2086 by @fangchenli in https://github.com/nebari-dev/nebari/pull/2087
+* ENH: add AWS IAM permissions_boundary option #2078 by @fangchenli in https://github.com/nebari-dev/nebari/pull/2082
+* CI: cleanup local integration workflow by @fangchenli in https://github.com/nebari-dev/nebari/pull/2079
+* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/nebari-dev/nebari/pull/2099
+* ENH: check missing GCP services by @fangchenli in https://github.com/nebari-dev/nebari/pull/2036
+* ENH: use packaging for version parsing, add unit tests by @fangchenli in https://github.com/nebari-dev/nebari/pull/2048
+* ENH: specify required field when retrieving available gcp regions by @fangchenli in https://github.com/nebari-dev/nebari/pull/2033
+* Upgrade conda-store to 2023.10.1 by @iameskild in https://github.com/nebari-dev/nebari/pull/2092
+* Add upgrade command for 2023.11.1 by @iameskild in https://github.com/nebari-dev/nebari/pull/2103
+* CLN: cleanup typing and typing import in init by @fangchenli in https://github.com/nebari-dev/nebari/pull/2107
+* Remove kbatch, prefect and clearml by @iameskild in https://github.com/nebari-dev/nebari/pull/2101
+* Fix integration tests, helm-validate script by @iameskild in https://github.com/nebari-dev/nebari/pull/2102
+* Re-enable AWS tags support by @iameskild in https://github.com/nebari-dev/nebari/pull/2096
+* Update upgrade instructions for 2023.11.1 by @iameskild in https://github.com/nebari-dev/nebari/pull/2112
+* Update nebari-git env pins by by @iameskild in https://github.com/nebari-dev/nebari/pull/2113
+* Update release notes for 2023.11.1 by @iameskild in https://github.com/nebari-dev/nebari/pull/2114
+
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2023.10.1...2023.11.1
+
+
 ## Release 2023.10.1 - October 20, 2023
 
 This release includes a major refactor which introduces a Pluggy-based extension mechanism which allow developers to build new stages. This is the initial implementation

pyproject.toml

@@ -79,22 +79,24 @@ dependencies = [
 [project.optional-dependencies]
 dev = [
     "black==22.3.0",
-    "mypy==1.6.1",
+    "coverage[toml]",
     "dask-gateway",
     "diagrams",
-    "python-dotenv",
     "escapism",
     "importlib-metadata<5.0",
     "jhub-client",
+    "jinja2",
+    "mypy==1.6.1",
     "paramiko",
     "pre-commit",
-    "pytest",
-    "pytest-timeout",
-    "pytest-playwright",
     "pytest-cov",
-    "coverage[toml]",
-    "jinja2",
+    "pytest-playwright",
+    "pytest-timeout",
+    "pytest",
+    "python-dotenv",
+    "python-hcl2",
     "setuptools==63.4.3",
+    "tqdm",
 ]
 docs = [
     "sphinx",

scripts/helm-validate.py

@@ -214,8 +214,8 @@ def add_workflow_job_summary(chart_index: dict):
 
 if __name__ == "__main__":
     # charts = generate_index_of_helm_charts()
-    STAGES_DIR = "nebari/template/stages"
-    SKIP_CHARTS = ["prefect", "clearml", "helm-extensions"]
+    STAGES_DIR = "src/_nebari/stages"
+    SKIP_CHARTS = ["helm-extensions"]
 
     charts = HelmChartIndexer(
         stages_dir=STAGES_DIR, skip_charts=SKIP_CHARTS

src/_nebari/constants.py

@@ -1,4 +1,4 @@
-CURRENT_RELEASE = "2023.10.1"
+CURRENT_RELEASE = "2023.12.1"
 
 # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes
 # implemented in August 2023.
@@ -15,7 +15,7 @@
 DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE
 DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = "2023.7.2"
 
-DEFAULT_CONDA_STORE_IMAGE_TAG = "v0.4.14"
+DEFAULT_CONDA_STORE_IMAGE_TAG = "2023.10.1"
 
 LATEST_SUPPORTED_PYTHON_VERSION = "3.10"
 

src/_nebari/initialize.py

@@ -47,7 +47,7 @@ def render_config(
     ssl_cert_email: str = None,
 ):
     config = {
-        "provider": cloud_provider.value,
+        "provider": cloud_provider,
         "namespace": namespace,
         "nebari_version": __version__,
     }

src/_nebari/provider/cicd/github.py

@@ -108,12 +108,6 @@ def gha_env_vars(config: schema.Main):
     if os.environ.get("NEBARI_GH_BRANCH"):
         env_vars["NEBARI_GH_BRANCH"] = "${{ secrets.NEBARI_GH_BRANCH }}"
 
-    # This assumes that the user is using the omitting sensitive values configuration for the token.
-    if config.prefect.enabled:
-        env_vars[
-            "NEBARI_SECRET_prefect_token"
-        ] = "${{ secrets.NEBARI_SECRET_PREFECT_TOKEN }}"
-
     if config.provider == schema.ProviderEnum.aws:
         env_vars["AWS_ACCESS_KEY_ID"] = "${{ secrets.AWS_ACCESS_KEY_ID }}"
         env_vars["AWS_SECRET_ACCESS_KEY"] = "${{ secrets.AWS_SECRET_ACCESS_KEY }}"