Include helm chart and healthchecks

.github/workflows/helm-chart.yml

@@ -0,0 +1,26 @@
+name: Helm chart
+
+on:
+  push:
+    tags:
+    - 'v*'
+
+jobs:
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+    - uses: actions/checkout@v4
+    - name: Chart | Push
+      uses: appany/[email protected]
+      with:
+        name: Metagrid
+        repository: esgf2-us
+        tag: ${{ github.ref_name }}
+        path: helm
+        registry: ghcr.io
+        registry_username: ${{ github.actor }}
+        registry_password: ${{ secrets.GITHUB_TOKEN }}
+        update_dependencies: 'true'

.gitignore

@@ -188,3 +188,6 @@ docker-compose.*-overlay.yml
 # Documentation artifacts
 docs/site
 frontend/public/runtime_env.js
+
+#Helm chart deps
+helm/charts/*

.pre-commit-config.yaml

@@ -9,6 +9,7 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
+        exclude: helm/templates
 
   # Back-end
   # ------------------------------------------------------------------------------

backend/.dockerignore

@@ -1,4 +1,3 @@
 .*
 !.coveragerc
-!.env
 !.pylintrc

backend/.gitignore

@@ -90,3 +90,6 @@ ENV/
 .env
 .envs/.production/.django
 .envs/.production/.postgres
+
+# Helm
+helm/charts/*

backend/config/urls.py

@@ -26,6 +26,7 @@
     set_temp_storage,
 )
 from metagrid.cart.views import CartViewSet, SearchViewSet
+from metagrid.observability.views import liveness, readiness
 from metagrid.projects.views import ProjectsViewSet
 from metagrid.users.views import UserCreateViewSet, UserViewSet
 
@@ -79,6 +80,8 @@ class KeycloakLogin(SocialLoginView):
     path("globus/auth", get_access_token, name="globus_auth"),
     path("globus/transfer", create_globus_transfer, name="globus_transfer"),
     path("frontend-config.js", get_frontend_config, name="frontend_config"),
+    path("liveness", liveness, name="liveness"),
+    path("readiness", readiness, name="readiness"),
     re_path(
         r"^account-confirm-email/",
         VerifyEmailView.as_view(),

backend/metagrid/api_globus/tests/test_views.py

@@ -1,4 +1,3 @@
-from typing import Any, Generator
 from unittest.mock import patch
 
 import pytest
@@ -11,15 +10,6 @@
 from metagrid.api_globus.views import globus_info_from_doc, search_files
 
 
-@pytest.fixture(scope="function")
-def api_client() -> Generator[APIClient, Any, Any]:
-    """
-    Fixture to provide an API client
-    :return: APIClient
-    """
-    yield APIClient()
-
-
 @responses.activate
 @pytest.mark.django_db
 def test_get_access_token(api_client: APIClient):

backend/metagrid/api_proxy/views.py

@@ -45,8 +45,7 @@ def do_globus_auth(request):
 @csrf_exempt
 def do_globus_logout(request):
     logout(request)
-    homepage_url = getattr(settings, "LOGOUT_REDIRECT_URL")
-    return redirect(homepage_url)
+    return redirect(settings.LOGOUT_REDIRECT_URL)
 
 
 @api_view()
@@ -75,12 +74,7 @@ def do_globus_search_endpoints(request):
 @require_http_methods(["GET", "POST"])
 @csrf_exempt
 def do_search(request):
-    esgf_host = getattr(
-        settings,
-        "SEARCH_URL",
-        "",
-    )
-    return do_request(request, esgf_host)
+    return do_request(request, settings.SEARCH_URL)
 
 
 @require_http_methods(["POST"])
@@ -115,12 +109,7 @@ def do_citation(request):
 @require_http_methods(["GET", "POST"])
 @csrf_exempt
 def do_status(request):
-    status_url = getattr(
-        settings,
-        "STATUS_URL",
-        "",
-    )  # pragma: no cover
-    resp = requests.get(status_url)  # pragma: no cover
+    resp = requests.get(settings.STATUS_URL)  # pragma: no cover
     if resp.status_code == 200:  # pragma: no cover
         return HttpResponse(resp.text)
     else:  # pragma: no cover
@@ -130,14 +119,7 @@ def do_status(request):
 @require_http_methods(["GET", "POST"])
 @csrf_exempt
 def do_wget(request):
-    return do_request(
-        request,
-        getattr(
-            settings,
-            "WGET_URL",
-            "",
-        ),
-    )
+    return do_request(request, settings.WGET_URL)
 
 
 def do_request(request, urlbase):
@@ -178,7 +160,6 @@ def get_temp_storage(request):
         data_key = request_body["dataKey"]
 
         if "temp_storage" not in request.session:
-            print({"msg": "Temporary storage empty.", data_key: "None"})
             return HttpResponse(
                 json.dumps(
                     {"msg": "Temporary storage empty.", data_key: "None"}

backend/metagrid/conftest.py

@@ -0,0 +1,13 @@
+from typing import Any, Generator
+
+import pytest
+from rest_framework.test import APIClient
+
+
+@pytest.fixture(scope="function")
+def api_client() -> Generator[APIClient, Any, Any]:
+    """
+    Fixture to provide an API client
+    :return: APIClient
+    """
+    yield APIClient()

backend/metagrid/observability/tests/test_k8s.py

@@ -0,0 +1,29 @@
+from unittest.mock import MagicMock
+
+import pytest
+from django.db import OperationalError, connection
+from django.urls import reverse
+from rest_framework import status
+from rest_framework.test import APIClient
+
+
+@pytest.mark.django_db
+def test_liveness_returns_200(api_client: APIClient):
+    response = api_client.post(reverse("liveness"), {})
+    assert response.status_code == status.HTTP_200_OK
+
+
+@pytest.mark.django_db
+def test_readiness_returns_200(api_client: APIClient):
+    response = api_client.post(reverse("readiness"), {})
+    assert response.status_code == status.HTTP_200_OK
+
+
+@pytest.mark.django_db
+def test_readiness_returns_500_on_error(api_client: APIClient, monkeypatch):
+    mock_cursor = MagicMock()
+    mock_cursor.fetchone.side_effect = OperationalError("Mock Error")
+    monkeypatch.setattr(connection, "cursor", lambda: mock_cursor)
+
+    response = api_client.post(reverse("readiness"), {})
+    assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR

backend/metagrid/observability/views.py

@@ -0,0 +1,19 @@
+from django.db import connection
+from django.http import HttpResponse, HttpResponseServerError
+from rest_framework.views import csrf_exempt
+
+
+@csrf_exempt
+def liveness(request) -> HttpResponse:
+    return HttpResponse()
+
+
+@csrf_exempt
+def readiness(request) -> HttpResponse:
+    try:
+        cursor = connection.cursor()
+        cursor.execute("SELECT 1")
+        row = cursor.fetchone()
+        return HttpResponse({"db": row})
+    except Exception as e:
+        return HttpResponseServerError(e)

docs/docs/users/quickstart.md → docs/docs/users/compose/quickstart.md

@@ -35,4 +35,4 @@ docker compose -f docker-compose.yml -f docker-compose.SITENAME-overlay.yml run
 ```
 
 ### Access the site
-- <http://localhost:8080>
+- <http://localhost:8080>

docs/docs/users/k8s/helm_values.md

@@ -0,0 +1,28 @@
+# Metagrid settings
+The `config:` section of `values.yaml` will be passed through Helm's `tpl` function and the result will be stored in a Kubernetes Secret and mounted as environment variables in the Django backend pod. See [Configurable Environment Variables](../configurable_environment_variables.md) for available environment variables.
+
+# Configuring Postgres
+This chart includes the [Bitnami Postgres-ha chart](https://github.com/bitnami/charts/tree/main/bitnami/postgresql-ha) as a dependency by default. You can customize this instance using the `postgres:` key in values.yaml.
+
+# Using an external Postgres database
+First, disable the included Postgres instance by setting `postgres.enabled: false` in values.yaml.
+Then, use the [standard libpq environment variables](https://www.postgresql.org/docs/current/libpq-envars.html) in the `config:` section to point to the external Postgres instance. For example:
+```yaml
+postgres:
+  enabled: false
+
+config:
+  PGHOST: postgres.example.local
+  PGUSER: metagrid_user
+  PGPASSWORD: some_password
+```
+
+# Using an external Node Status API endpoint
+This chart includes a minimal Prometheus and Blackbox Exporter installation to serve the Node Status API. To use an existing or external Node Status API endpoint, disable the included instance and point `METAGRID_STATUS_URL` to your existing API endpoint:
+```yaml
+nodeStatusBackend:
+  enabled: false
+
+config:
+  METAGRID_STATUS_URL: https://thanos-querier.openshift-monitoring.svc.cluster.local:9092/api/v1/query?query=probe_success%7Bjob%3D%22http_2xx%22%2C+target%3D~%22.%2Athredds.%2A%22%7D
+```

docs/docs/users/k8s/quickstart.md

@@ -0,0 +1,34 @@
+# Quick Start
+### Prerequisites
+
+- [helm](https://helm.sh/docs/intro/install/)
+- A working Index Node to which Metagrid is able to connect
+- A registered Globus Client UUID and Secret
+
+### Create a helm values file
+```yaml
+# my-values.yaml
+ingress:
+  host: esgf-node.example.com
+
+config:
+  METAGRID_SOCIAL_AUTH_GLOBUS_KEY: 94c44808-9efd-4236-bffd-1185b1071736
+  METAGRID_SOCIAL_AUTH_GLOBUS_SECRET: 34364292-2752-4d5e-8295
+  METAGRID_GLOBUS_CLIENT_ID: 21982de0-2b7a-4ba8-bef5-5606ae098201
+  METAGRID_SEARCH_URL: https://esgf-node.ornl.gov/esg-search/search
+  METAGRID_WGET_URL: https://esgf-node.ornl.gov/esg-search/wget
+
+postgresql:
+  postgresql:
+    password: pgpass
+    repmgrPassword: repmgrpass
+
+  pgpool:
+    adminPassword: pgpooladminpass
+```
+
+### Install the helm chart
+```bash
+helm install my-release oci://ghcr.io/esgf2-us/metagrid -f my-values.yaml
+
+```

docs/mkdocs.yml

@@ -13,8 +13,12 @@ markdown_extensions:
 nav:
 - Home: "index.md"
 - User Guide:
-  - Quickstart: "users/quickstart.md"
-  - Moving to Production: "users/moving_to_production.md"
+  - Docker Compose:
+    - Quickstart: "users/compose/quickstart.md"
+    - Moving to Production: "users/compose/moving_to_production.md"
+  - Kubernetes:
+    - Quickstart: "users/k8s/quickstart.md"
+    - Configuring Helm values: "users/k8s/helm_values.md"
   - Configurable environment variables: "users/configurable_environment_variables.md"
 - Contributor Guide:
   - Project Standards: "contributors/project_standards.md"

helm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: postgresql-ha
+  repository: https://charts.bitnami.com/bitnami
+  version: 15.1.6
+digest: sha256:51b380ff616ae810df547d8befcea2d798e5a9cf9df48e3131b164647ff7bdbb
+generated: "2025-01-31T16:55:11.067764702-05:00"

helm/Chart.yaml

@@ -0,0 +1,31 @@
+apiVersion: v2
+name: metagrid
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.3.0"
+
+dependencies:
+- name: postgresql-ha
+  alias: postgresql
+  condition: postgresql.enabled
+  version: ">=9.2.0"
+  repository: "https://charts.bitnami.com/bitnami"