diff --git a/.github/workflows/generate_cli_doc.yml b/.github/workflows/generate_cli_doc.yml index c0ae6ee7d2..bc9aa6c4d5 100644 --- a/.github/workflows/generate_cli_doc.yml +++ b/.github/workflows/generate_cli_doc.yml @@ -25,10 +25,10 @@ jobs: working-directory: ./docs-sphinx steps: - name: Check out repository 🛎️ - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 655bd96206..771999f51d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -8,10 +8,11 @@ jobs: test-pypi: name: Test PyPi release runs-on: ubuntu-latest - + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: - name: Set up python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -19,7 +20,7 @@ jobs: run: python -m pip install --upgrade pip build - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -35,8 +36,6 @@ jobs: - name: Publish to test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository-url: https://test.pypi.org/legacy/ - name: Sleep @@ -59,7 +58,7 @@ jobs: steps: - name: Set up python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -67,26 +66,12 @@ jobs: run: python -m pip install --upgrade pip build - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Build source and binary run: python -m build --sdist --wheel . - - name: Retrieve secret from Vault - uses: hashicorp/vault-action@v2.5.0 - with: - method: jwt - url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200" - namespace: "admin/quansight" - role: "repository-nebari-dev-nebari-role" - secrets: | - kv/data/repository/nebari-dev/nebari/shared_secrets PYPI_USERNAME | PYPI_USERNAME; - kv/data/repository/nebari-dev/nebari/shared_secrets PYPI_PASSWORD | PYPI_PASSWORD; - - name: Publish package uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: ${{ env.PYPI_USERNAME }} - password: ${{ env.PYPI_PASSWORD }} diff --git a/.github/workflows/run-precommit.yaml b/.github/workflows/run-precommit.yaml index 50904ae178..9592a58373 100644 --- a/.github/workflows/run-precommit.yaml +++ b/.github/workflows/run-precommit.yaml @@ -17,9 +17,9 @@ jobs: shell: bash -l {0} steps: - name: Checkout repository 🔔 - uses: actions/checkout@v3 + uses: actions/checkout@v4.1.1 - name: Run terraform pre-commit ⚡️ - uses: pre-commit/action@v3.0.0 + uses: pre-commit/action@v3.0.1 with: extra_args: --all-files terraform_fmt diff --git a/.github/workflows/test-provider.yaml b/.github/workflows/test-provider.yaml index 3c0a3fa89c..79057ce6fb 100644 --- a/.github/workflows/test-provider.yaml +++ b/.github/workflows/test-provider.yaml @@ -56,7 +56,7 @@ jobs: fail-fast: false steps: - name: "Checkout Infrastructure" - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Checkout the branch from the PR that triggered the job if: ${{ github.event_name == 'issue_comment' }} @@ -65,7 +65,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6a8fa4a446..cc4887f62c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,12 +42,12 @@ jobs: cancel-in-progress: true steps: - name: "Checkout Infrastructure" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test_aws_integration.yaml b/.github/workflows/test_aws_integration.yaml index fa1a2332df..36112ccd50 100644 --- a/.github/workflows/test_aws_integration.yaml +++ b/.github/workflows/test_aws_integration.yaml @@ -43,13 +43,13 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.NEBARI_GH_BRANCH }} fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/.github/workflows/test_conda_build.yaml b/.github/workflows/test_conda_build.yaml index e34363d9a3..17d2683477 100644 --- a/.github/workflows/test_conda_build.yaml +++ b/.github/workflows/test_conda_build.yaml @@ -25,12 +25,12 @@ jobs: cancel-in-progress: true steps: - name: "Checkout Infrastructure" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: 3.8 diff --git a/.github/workflows/test_do_integration.yaml b/.github/workflows/test_do_integration.yaml index dbe10a3028..dcfacf3175 100644 --- a/.github/workflows/test_do_integration.yaml +++ b/.github/workflows/test_do_integration.yaml @@ -42,12 +42,12 @@ jobs: pull-requests: write steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.NEBARI_GH_BRANCH }} fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/.github/workflows/test_gcp_integration.yaml b/.github/workflows/test_gcp_integration.yaml index 57ef84288f..0418e0af40 100644 --- a/.github/workflows/test_gcp_integration.yaml +++ b/.github/workflows/test_gcp_integration.yaml @@ -42,13 +42,13 @@ jobs: pull-requests: write steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.NEBARI_GH_BRANCH }} fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/.github/workflows/test_helm_charts.yaml b/.github/workflows/test_helm_charts.yaml index daf9abb6da..05c47bcbbb 100644 --- a/.github/workflows/test_helm_charts.yaml +++ b/.github/workflows/test_helm_charts.yaml @@ -23,11 +23,11 @@ jobs: runs-on: ubuntu-latest steps: - name: "Checkout Infrastructure" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.8" - name: Install additional Python dependencies diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml index 05dec384b0..4dc038afee 100644 --- a/.github/workflows/test_local_integration.yaml +++ b/.github/workflows/test_local_integration.yaml @@ -57,7 +57,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 env: CONDA: /home/runnerx/miniconda3 with: @@ -70,7 +70,7 @@ jobs: pip install .[dev] playwright install - - uses: azure/setup-kubectl@v3 + - uses: azure/setup-kubectl@v4.0.0 with: version: v1.19.16 @@ -140,9 +140,9 @@ jobs: nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config nebari-config.yaml nebari keycloak listusers --config nebari-config.yaml - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: - node-version: 16 + node-version: 20 - name: Get nebari-config.yaml full path run: echo "NEBARI_CONFIG_PATH=`realpath ./local-deployment/nebari-config.yaml`" >> "$GITHUB_ENV" @@ -170,7 +170,7 @@ jobs: - name: Save Cypress screenshots and videos if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.3.1 with: name: e2e-cypress path: | diff --git a/.github/workflows/typing.yaml b/.github/workflows/typing.yaml index ae3fa18b93..de70d69483 100644 --- a/.github/workflows/typing.yaml +++ b/.github/workflows/typing.yaml @@ -24,12 +24,12 @@ jobs: cancel-in-progress: true steps: - name: "Checkout Repository" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" cache: "pip" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ccda1916e6..2427219a8e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,13 +51,13 @@ repos: # python - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.1.1 hooks: - id: black args: ["--line-length=88", "--exclude=/src/_nebari/template/"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.9 + rev: v0.2.0 hooks: - id: ruff args: ["--fix"] diff --git a/RELEASE.md b/RELEASE.md index 41433e9e13..2b249c6603 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,7 +9,18 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. --> --- -## Upcoming Release + +## Release 2024.3.2 - March 14, 2024 + +### What's Changed +* update max k8s versions and remove depreciated api usage in local deploy by @dcmcand in https://github.com/nebari-dev/nebari/pull/2276 +* update keycloak image repo by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2312 +* Generate random password for Grafana by @aktech in https://github.com/nebari-dev/nebari/pull/2289 +* update conda store to 2024.3.1 by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2316 +* Switch PyPI release workflow to use trusted publishing by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2323 + + +**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.3.1...2024.3.2 ## Release 2024.3.1 - March 11, 2024 diff --git a/scripts/helm-validate.py b/scripts/helm-validate.py index a916d2a2e5..c623ef0620 100644 --- a/scripts/helm-validate.py +++ b/scripts/helm-validate.py @@ -67,7 +67,7 @@ def _load_variable_value(self, argument, parent_contents): var_name = self._clean_var_name(argument, "var") for var in parent_contents.get("variable", {}): if var_name in var: - return var[var_name] + return var[var_name]["default"] else: raise ValueError(f"Could not find variable {var_name}") diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 0d49bc6e1b..1f4d9ef81e 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -1,4 +1,4 @@ -CURRENT_RELEASE = "2024.3.1" +CURRENT_RELEASE = "2024.3.2" # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes # implemented in August 2023. @@ -8,14 +8,14 @@ # 04-kubernetes-ingress DEFAULT_TRAEFIK_IMAGE_TAG = "2.9.1" -HIGHEST_SUPPORTED_K8S_VERSION = ("1", "26", "9") +HIGHEST_SUPPORTED_K8S_VERSION = ("1", "29", "2") DEFAULT_GKE_RELEASE_CHANNEL = "UNSPECIFIED" DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = CURRENT_RELEASE -DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.1.1" +DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.3.1" LATEST_SUPPORTED_PYTHON_VERSION = "3.10" diff --git a/src/_nebari/stages/infrastructure/template/local/main.tf b/src/_nebari/stages/infrastructure/template/local/main.tf index 00c1ca97b0..fb0d0997e1 100644 --- a/src/_nebari/stages/infrastructure/template/local/main.tf +++ b/src/_nebari/stages/infrastructure/template/local/main.tf @@ -1,8 +1,8 @@ terraform { required_providers { kind = { - source = "kyma-incubator/kind" - version = "0.0.11" + source = "tehcyx/kind" + version = "0.4.0" } docker = { source = "kreuzwerker/docker" @@ -48,7 +48,7 @@ resource "kind_cluster" "default" { node { role = "general" - image = "kindest/node:v1.23.13" + image = "kindest/node:v1.29.2" } } } diff --git a/src/_nebari/stages/infrastructure/template/local/metallb.yaml b/src/_nebari/stages/infrastructure/template/local/metallb.yaml index 9d6b6833c8..c832baebde 100644 --- a/src/_nebari/stages/infrastructure/template/local/metallb.yaml +++ b/src/_nebari/stages/infrastructure/template/local/metallb.yaml @@ -1,82 +1,3 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - labels: - app: metallb - name: controller -spec: - allowPrivilegeEscalation: false - allowedCapabilities: [] - allowedHostPaths: [] - defaultAddCapabilities: [] - defaultAllowPrivilegeEscalation: false - fsGroup: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - hostIPC: false - hostNetwork: false - hostPID: false - privileged: false - readOnlyRootFilesystem: true - requiredDropCapabilities: - - ALL - runAsUser: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - seLinux: - rule: RunAsAny - supplementalGroups: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - volumes: - - configMap - - secret - - emptyDir ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - labels: - app: metallb - name: speaker -spec: - allowPrivilegeEscalation: false - allowedCapabilities: - - NET_RAW - allowedHostPaths: [] - defaultAddCapabilities: [] - defaultAllowPrivilegeEscalation: false - fsGroup: - rule: RunAsAny - hostIPC: false - hostNetwork: true - hostPID: false - hostPorts: - - max: 7472 - min: 7472 - - max: 7946 - min: 7946 - privileged: true - readOnlyRootFilesystem: true - requiredDropCapabilities: - - ALL - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - volumes: - - configMap - - secret - - emptyDir ---- apiVersion: v1 kind: ServiceAccount metadata: diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 2c55e0cae9..13cf3f9bfc 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -181,9 +181,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): cert_details["acme-email"] = self.config.certificate.acme_email cert_details["acme-server"] = self.config.certificate.acme_server elif cert_type == "existing": - cert_details[ - "certificate-secret-name" - ] = self.config.certificate.secret_name + cert_details["certificate-secret-name"] = ( + self.config.certificate.secret_name + ) return { **{ diff --git a/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/values.yaml b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/values.yaml index 94359cf451..abe7d4d3e3 100644 --- a/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/values.yaml +++ b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/values.yaml @@ -4,6 +4,9 @@ ingress: # we will need to define our own IngressRoute elsewhere. enabled: false +image: + repository: quay.io/keycloak/keycloak + imagePullSecrets: - name: "extcrcreds" diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index a9124f41ac..9c47fee6ec 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -51,9 +51,15 @@ class Storage(schema.Base): class JupyterHubTheme(schema.Base): hub_title: str = "Nebari" hub_subtitle: str = "Your open source data science platform" - welcome: str = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" - logo: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" - favicon: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/symbol/favicon.ico" + welcome: str = ( + """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" + ) + logo: str = ( + "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" + ) + favicon: str = ( + "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/symbol/favicon.ico" + ) primary_color: str = "#4f4173" primary_color_dark: str = "#4f4173" secondary_color: str = "#957da6" diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index 06aa97287a..50d527b863 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -208,12 +208,14 @@ def base_profile_extra_mounts(): extra_pod_config = { "volumes": [ - { - "name": volume["name"], - "persistentVolumeClaim": {"claimName": volume["name"]}, - } - if volume["kind"] == "persistentvolumeclaim" - else {"name": volume["name"], "configMap": {"name": volume["name"]}} + ( + { + "name": volume["name"], + "persistentVolumeClaim": {"claimName": volume["name"]}, + } + if volume["kind"] == "persistentvolumeclaim" + else {"name": volume["name"], "configMap": {"name": volume["name"]}} + ) for mount_path, volume in extra_mounts.items() ] } @@ -367,9 +369,11 @@ def configure_user(username, groups, uid=1000, gid=100): # mount the shared directories for user only if there are # shared folders (groups) that the user is a member of # else ensure that the `shared` folder symlink does not exist - f"ln -sfn /shared /home/{username}/shared" - if groups - else f"rm -f /home/{username}/shared", + ( + f"ln -sfn /shared /home/{username}/shared" + if groups + else f"rm -f /home/{username}/shared" + ), # conda-store environment configuration f"printf '{condarc}' > /home/{username}/.condarc", # jupyter configuration diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/main.tf index 7ba919ec54..413a9e08d2 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/main.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/main.tf @@ -1,3 +1,8 @@ +resource "random_password" "grafana_admin_password" { + length = 32 + special = false +} + resource "helm_release" "prometheus-grafana" { name = "nebari" namespace = var.namespace @@ -176,6 +181,9 @@ resource "helm_release" "prometheus-grafana" { "${var.node-group.key}" = var.node-group.value } + # Avoid using the default password, as that's a security risk + adminPassword : random_password.grafana_admin_password.result + sidecar = { dashboards = { annotations = { diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 5c095f04a2..dcadc1a029 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -716,6 +716,17 @@ def _version_specific_upgrade( return config +class Upgrade_2024_3_2(UpgradeStep): + version = "2024.3.2" + + def _version_specific_upgrade( + self, config, start_version, config_filename: Path, *args, **kwargs + ): + rich.print("Ready to upgrade to Nebari version [green]2024.3.2[/green].") + + return config + + __rounded_version__ = str(rounded_ver_parse(__version__)) # Manually-added upgrade steps must go above this line diff --git a/tests/tests_deployment/test_grafana_api.py b/tests/tests_deployment/test_grafana_api.py new file mode 100644 index 0000000000..cdb489f349 --- /dev/null +++ b/tests/tests_deployment/test_grafana_api.py @@ -0,0 +1,18 @@ +import base64 + +import pytest +import requests + +from tests.tests_deployment import constants + + +@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning") +def test_grafana_api_not_accessible_with_default_credentials(): + """Making sure that Grafana's API is not accessible on default user/pass""" + user_pass_b64_encoded = base64.b64encode(b"admin:prom-operator").decode() + response = requests.get( + f"https://{constants.NEBARI_HOSTNAME}/monitoring/api/datasources", + headers={"Authorization": f"Basic {user_pass_b64_encoded}"}, + verify=False, + ) + assert response.status_code == 401