Merge branch 'develop' into typing-hookspecs

nebari-dev · Oct 19, 2023 · 7989b3f · 7989b3f
2 parents cb7b004 + ee1b4c9
commit 7989b3f
Show file tree

Hide file tree

Showing 37 changed files with 605 additions and 207 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -26,6 +26,9 @@ jobs:
   test-general:
     name: "Pytest"
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
     strategy:
       matrix:
         python-version:
@@ -34,21 +37,28 @@ jobs:
           - "3.10"
           - "3.11"
       fail-fast: false
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ matrix.python-version }}
+      cancel-in-progress: true
     steps:
       - name: "Checkout Infrastructure"
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+
       - name: Setup miniconda
         uses: conda-incubator/setup-miniconda@v2
         with:
+          auto-update-conda: true
           python-version: ${{ matrix.python-version }}
-          channel-priority: strict
-          channels: conda-forge
+          channels: conda-forge,defaults
+          activate-environment: nebari-dev
+
       - name: Install Nebari
         run: |
+          python --version
           pip install -e .[dev]
-          conda install --quiet --yes conda-build
+
       - name: Test Nebari
         run: |
           pytest --version

diff --git a/.github/workflows/test_aws_integration.yaml b/.github/workflows/test_aws_integration.yaml
@@ -56,7 +56,6 @@ jobs:
       - name: Install Nebari
         run: |
           pip install .[dev]
-          conda install --quiet --yes conda-build
           playwright install
 
       - name: Retrieve secret from Vault

diff --git a/.github/workflows/test_conda_build.yaml b/.github/workflows/test_conda_build.yaml
@@ -0,0 +1,54 @@
+name: "Test Conda Build"
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/test_conda_build.yaml"
+      - "pyproject.toml"
+  push:
+    branches:
+      - main
+      - develop
+      - release/\d{4}.\d{1,2}.\d{1,2}
+    paths:
+      - ".github/workflows/test_conda_build.yaml"
+      - "pyproject.toml"
+
+jobs:
+  test-conda-build:
+    runs-on: ubuntu-latest
+    defaults:
+        run:
+          shell: bash -el {0}
+    concurrency:
+        group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+        cancel-in-progress: true
+    steps:
+      - name: "Checkout Infrastructure"
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: 3.8
+          channels: conda-forge
+          activate-environment: nebari-dev
+
+      - name: Install dependencies
+        run: |
+          conda install build grayskull conda-build
+
+      - name: Generate sdist
+        run: |
+          python -m build --sdist
+
+      - name: Generate meta.yaml
+        run: |
+          python -m grayskull pypi dist/*.tar.gz
+
+      - name: Build conda package
+        run: |
+          conda build nebari
diff --git a/.github/workflows/test_do_integration.yaml b/.github/workflows/test_do_integration.yaml
@@ -0,0 +1,83 @@
+name: test-gcp-integration
+
+on:
+  schedule:
+    - cron: "0 0 * * MON"
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Nebari branch to deploy, test, destroy'
+        required: true
+        default: develop
+        type: string
+      image-tag:
+        description: 'Nebari image tag created by the nebari-docker-images repo'
+        required: true
+        default: main
+        type: string
+      tf-log-level:
+        description: 'Change Terraform log levels'
+        required: false
+        default: info
+        type: choice
+        options:
+        - info
+        - warn
+        - debug
+        - trace
+        - error
+
+env:
+  NEBARI_GH_BRANCH: ${{ github.event.inputs.branch || 'develop' }}
+  NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
+  TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}
+
+
+jobs:
+  test-do-integration:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.NEBARI_GH_BRANCH }}
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+            python-version: 3.11
+
+      - name: Install Nebari
+        run: |
+            pip install .[dev]
+            playwright install
+
+      - name: Retrieve secret from Vault
+        uses: hashicorp/[email protected]
+        with:
+          method: jwt
+          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
+          namespace: "admin/quansight"
+          role: "repository-nebari-dev-nebari-role"
+          secrets: |
+            kv/data/repository/nebari-dev/nebari/shared_secrets DIGITALOCEAN_TOKEN | DIGITALOCEAN_TOKEN;
+            kv/data/repository/nebari-dev/nebari/cloudflare/[email protected]/nebari-dev-ci token | CLOUDFLARE_TOKEN;
+
+      - name: Set Environment DO
+        run: |
+          echo "SPACES_ACCESS_KEY_ID=${{ secrets.SPACES_ACCESS_KEY_ID }}" >> $GITHUB_ENV
+          echo "SPACES_SECRET_ACCESS_KEY=${{ secrets.SPACES_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
+          echo "NEBARI_K8S_VERSION"=1.25.12-do.0 >> $GITHUB_ENV
+
+      - name: Integration Tests
+        run: |
+          pytest --version
+          pytest tests/tests_integration/ -vvv -s --cloud do
+        env:
+            NEBARI_SECRET__default_images__jupyterhub: "quay.io/nebari/nebari-jupyterhub:${{ env.NEBARI_IMAGE_TAG }}"
+            NEBARI_SECRET__default_images__jupyterlab: "quay.io/nebari/nebari-jupyterlab:${{ env.NEBARI_IMAGE_TAG }}"
+            NEBARI_SECRET__default_images__dask_worker: "quay.io/nebari/nebari-dask-worker:${{ env.NEBARI_IMAGE_TAG }}"
diff --git a/.github/workflows/test_gcp_integration.yaml b/.github/workflows/test_gcp_integration.yaml
@@ -0,0 +1,91 @@
+name: test-gcp-integration
+
+on:
+  schedule:
+    - cron: "0 0 * * MON"
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Nebari branch to deploy, test, destroy'
+        required: true
+        default: develop
+        type: string
+      image-tag:
+        description: 'Nebari image tag created by the nebari-docker-images repo'
+        required: true
+        default: main
+        type: string
+      tf-log-level:
+        description: 'Change Terraform log levels'
+        required: false
+        default: info
+        type: choice
+        options:
+        - info
+        - warn
+        - debug
+        - trace
+        - error
+
+env:
+  NEBARI_GH_BRANCH: ${{ github.event.inputs.branch || 'develop' }}
+  NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
+  TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}
+
+
+jobs:
+  test-gcp-integration:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.NEBARI_GH_BRANCH }}
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install Nebari
+        run: |
+          pip install .[dev]
+          playwright install
+
+      - name: Retrieve secret from Vault
+        uses: hashicorp/[email protected]
+        with:
+          method: jwt
+          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
+          namespace: "admin/quansight"
+          role: "repository-nebari-dev-nebari-role"
+          secrets: |
+            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci project_id | PROJECT_ID;
+            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci workload_identity_provider | GCP_WORKFLOW_PROVIDER;
+            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci service_account_name | GCP_SERVICE_ACCOUNT;
+            kv/data/repository/nebari-dev/nebari/cloudflare/[email protected]/nebari-dev-ci token | CLOUDFLARE_TOKEN;
+
+      - name: 'Authenticate to GCP'
+        uses: 'google-github-actions/auth@v1'
+        with:
+            token_format: access_token
+            workload_identity_provider: ${{ env.GCP_WORKFLOW_PROVIDER }}
+            service_account: ${{ env.GCP_SERVICE_ACCOUNT }}
+
+      - name: Set required environment variables
+        run: |
+          echo "GOOGLE_CREDENTIALS=${{ env.GOOGLE_APPLICATION_CREDENTIALS }}" >> $GITHUB_ENV
+
+      - name: Integration Tests
+        run: |
+          pytest --version
+          pytest tests/tests_integration/ -vvv -s --cloud gcp
+        env:
+          NEBARI_SECRET__default_images__jupyterhub: "quay.io/nebari/nebari-jupyterhub:${{ env.NEBARI_IMAGE_TAG }}"
+          NEBARI_SECRET__default_images__jupyterlab: "quay.io/nebari/nebari-jupyterlab:${{ env.NEBARI_IMAGE_TAG }}"
+          NEBARI_SECRET__default_images__dask_worker: "quay.io/nebari/nebari-dask-worker:${{ env.NEBARI_IMAGE_TAG }}"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
         exclude: "^src/_nebari/template/"
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
       - id: codespell
         args:
@@ -53,13 +53,13 @@ repos:
 
   # python
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
       - id: black
         args: ["--line-length=88", "--exclude=/src/_nebari/template/"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.287
+    rev: v0.0.292
     hooks:
       - id: ruff
         args: ["--fix"]
@@ -75,7 +75,7 @@ repos:
 
   # terraform
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.83.2
+    rev: v1.83.4
     hooks:
       - id: terraform_fmt
         args:

diff --git a/README.md b/README.md
@@ -77,7 +77,7 @@ Amazon [AWS](https://aws.amazon.com/), [GCP](https://cloud.google.com/ "Google C
 
 - Operating System: Currently, Nebari supports development on macOS and Linux operating systems. Windows is NOT supported.
   However, we would welcome contributions that add and improve support for Windows.
-- You need Python >= 3.7 on your local machine or virtual environment to work on Nebari.
+- You need Python >= 3.8 on your local machine or virtual environment to work on Nebari.
 - Adopting virtual environments ([`conda`](https://docs.conda.io/en/latest/), [`pipenv`](https://github.com/pypa/pipenv) or
   [`venv`](https://docs.python.org/3/library/venv.html)) is also encouraged.
 

diff --git a/RELEASE.md b/RELEASE.md
@@ -441,7 +441,7 @@ Enhancements for this release include:
 
 ### Bug fixes
 
-This release addresses several bugs with a slight emphasis on stablizing the core services while also improving the end user experience.
+This release addresses several bugs with a slight emphasis on stabilizing the core services while also improving the end user experience.
 
 ### What's Changed
 * [BUG] Adding back feature of limiting profiles for users and groups by @costrouc in [PR 1169](https://github.com/Quansight/qhub/pull/1169)

diff --git a/flake.nix b/flake.nix
@@ -36,7 +36,6 @@
         pythonPackages.pytest
         pythonPackages.pytest-timeout
         pythonPackages.black
-        pythonPackages.flake8
         pythonPackages.sphinx
         pythonPackages.dask-gateway
         pythonPackages.paramiko

diff --git a/pyproject.toml b/pyproject.toml
@@ -45,6 +45,7 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Intended Audience :: Developers",
     "Intended Audience :: Education",
     "Intended Audience :: Information Technology",
@@ -81,7 +82,6 @@ dev = [
     "diagrams",
     "python-dotenv",
     "escapism",
-    "flake8==3.8.4",
     "importlib-metadata<5.0",
     "jhub-client",
     "paramiko",
@@ -91,8 +91,6 @@ dev = [
     "pytest-playwright",
     "pytest-cov",
     "coverage[toml]",
-    "grayskull",
-    "build",
     "jinja2",
     "setuptools==63.4.3",
 ]

diff --git a/pytest.ini b/pytest.ini
@@ -7,7 +7,6 @@ addopts =
     # turn warnings into errors
     -Werror
 markers =
-    conda: conda required to run this test (deselect with '-m \"not conda\"')
     gpu: test gpu working properly
     preemptible: test preemptible instances
 testpaths =

diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py
@@ -47,7 +47,7 @@ def render_config(
     ssl_cert_email: str = None,
 ):
     config = {
-        "provider": cloud_provider,
+        "provider": cloud_provider.value,
         "namespace": namespace,
         "nebari_version": __version__,
     }
@@ -74,7 +74,7 @@ def render_config(
     config["theme"] = {"jupyterhub": {"hub_title": f"Nebari - { project_name }"}}
     config["theme"]["jupyterhub"][
         "welcome"
-    ] = """Welcome! Learn about Nebari's features and configurations in <a href="https://www.nebari.dev/docs">the documentation</a>. If you have any questions or feedback, reach the team on <a href="https://www.nebari.dev/docs/community#getting-support">Nebari's support forums</a>."""
+    ] = """Welcome! Learn about Nebari's features and configurations in <a href="https://www.nebari.dev/docs/welcome">the documentation</a>. If you have any questions or feedback, reach the team on <a href="https://www.nebari.dev/docs/community#getting-support">Nebari's support forums</a>."""
 
     config["security"]["authentication"] = {"type": auth_provider}