freeze bundle

.github/workflows/_bundle-release.yaml

@@ -0,0 +1,95 @@
+name: Integration matrix
+
+on: 
+  workflow_call:
+    inputs:
+      charm-channel:
+        type: string
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  validate-inputs:
+    name: Validate action inputs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Validate charm-channel
+        if: ${{ inputs.charm-channel != 'edge' && inputs.charm-channel != 'beta' && inputs.charm-channel != 'candidate' && inputs.charm-channel != 'stable'}}
+        run: |
+          echo "Error: The 'charm-channel' input must be one of edge, beta, candidate, stable."
+          exit 1
+  render-freeze-bundle:
+    name: Render and freeze bundle
+    needs: [ validate-inputs ]
+    # We render and freeze at the start to avoid possible races, in case a new charm was release
+    # while these tests are still running.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Render and freeze bundle
+        env:
+          CHARMHUB_TOKEN: "${{ secrets.CHARMHUB_TOKEN }}"
+        run: |
+          tox -e render-bundle -- --channel=${{ inputs.charm-channel }}
+          python3 scripts/freeze_bundle.py bundle.yaml > bundle.yaml
+      - name: Upload bundle as artifact to be used by the next job
+        uses: actions/upload-artifact@v3
+        with:
+          name: frozen-bundle
+          path: bundle.yaml
+  integration-matrix:
+    name: Matrix tests for charms
+    needs: [ render-freeze-bundle ]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        juju-track: ["3.4" ]
+        microk8s-channel: [ "1.27-strict/stable", "1.28-strict/stable" ]
+        include:
+          - juju-track: "3.4"
+            juju-channel: "3.4/stable"
+            juju-agent-version: "3.4.0"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Get prefsrc
+        run: |
+          echo "IPADDR=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc')" >> $GITHUB_ENV
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          juju-channel: ${{ matrix.juju-channel }}
+          provider: microk8s
+          channel: ${{ matrix.microk8s-channel }}
+          microk8s-addons: "hostpath-storage dns metallb:${{ env.IPADDR }}-${{ env.IPADDR }}"
+          bootstrap-options: "--agent-version ${{ matrix.juju-agent-version }}"
+      - name: Update python-libjuju dependency to match juju version
+        # Assuming the dep is given on a separate tox.ini line
+        run: sed -E -i 's/^\s*juju\s*~=.+/    juju~=${{ matrix.juju-track }}.0/g' tox.ini
+      - uses: actions/download-artifact@v3
+        with:
+          name: frozen-bundle
+      - name: Run tests (juju ${{ matrix.juju-channel }}, microk8s ${{ matrix.microk8s-channel }})
+        run: tox -e integration
+      - name: Dump logs
+        if: failure()
+        uses: canonical/charming-actions/dump-logs@main
+  release-pinned-bundle:
+    name: Release pinned bundle
+    needs: [ integration-matrix ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: frozen-bundle
+      - name: Upload bundle to the pinned track
+        uses: canonical/charming-actions/[email protected]
+        with:
+          channel: "pinned/${{ inputs.charm-channel }}"
+          credentials: "${{ secrets.CHARMHUB_TOKEN }}"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/bundle-release.yaml

@@ -0,0 +1,19 @@
+name: Release Bundle
+
+on:
+  workflow_call:
+
+jobs:
+    per-channel-integration-matrix:
+        name: Per-channel integration matrix
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                charm-channel: [ "edge", "beta", "candidate", "stable" ]
+        steps:
+        -   name: Run integration matrix for ${{ matrix.charm-channel }} channel
+            uses:  canonical/observability/.github/workflows/_bundle_release.yaml@main
+            with:
+                charm-channel: ${{ matrix.charm-channel }}
+

README.md

@@ -44,10 +44,10 @@ Periodically, CI checks whether the charm libraries are up-to-date; if not (i.e.
 There's also a manual action to promote the charm (i.e., from `latest/edge` to `latest/beta`), making the process more user-friendly.
 
 ### Bundle Workflows
-| On PRs                              |
-| ------------------------------------|
-| **`bundle-pull-request.yaml`**      |
-| `├── _charm-codeql-analysis.yaml`   |
+| On PRs                              | Periodically                |
+| ------------------------------------| ----------------------------|
+| **`bundle-pull-request.yaml`**      | **`bundle-release.yaml`**   |
+| `├── _charm-codeql-analysis.yaml`   | `├── _bundle-release.yaml`  |
 | `├── _charm-linting.yaml`           |
 | `└── _charm-tests-integration.yaml` |
 
@@ -56,7 +56,7 @@ Whenever a PR is opened to a bundle repository, some quality checks are run:
 * run the Canonical inclusive naming workflow.
 * run linting, analyses and tests to ensure the code quality.
 
-<!-- TODO: add merging PR workflow -->
+Periodically, integration matrix tests will run against a COS-related bundle and then, once the integration tests pass for any of the tracks: `edge`, `beta`, `candidate`, `stable`, a bundle gets released to each respective pinned track on Charmhub.
 
 ### Rock Workflows
 
@@ -105,8 +105,13 @@ This repo also contains a `scripts` directory that could hold helper scripts for
 ### `render-bundle`
 This helper script is used by COS bundles as a `pip` package in a `tox.ini` file to render a `bundle.yaml.j2` template into a `bundle.yaml` file that can be deployed using `juju deploy ./bundle.yaml`.
 
+### `freeze-bundle`
+This script takes a `bundle.yaml` file and for each `application` along with its defined channel, it obtains the revision number for that application charm from Charmhub and updates `bundle.yaml` file with a pinned `revision` on each application. Currently, this script is used inside the `bundle-release.yaml` workflow.
+
 ### Contributing
 To add similar helper scripts (e.g: `my_helper.py`) to be used as a `pip` package:
 
 1. Add the script inside `scripts` directory.
-2. In `scripts/pyproject.toml`, under `[project.scripts]`, add an entrypoint to your newly added script.
+2. In `scripts/pyproject.toml`, under `[project.scripts]`, add an entrypoint to your newly added script.
+3. Increment `version` in `scripts/pyproject.toml`.
+4. Add the script's description in `README.md`.

scripts/freeze_bundle.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+"""This script updates a bundle.yaml file with revisions from charmhub."""
+
+import base64
+import json
+import os
+import sys
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+import yaml
+
+
+def obtain_charm_releases(charm_name: str) -> dict:
+    """Obtain charm releases from charmhub as a dict.
+
+    Args:
+        charm_name: e.g. "grafana-k8s".
+    """
+    if token := os.environ.get("CHARMHUB_TOKEN"):
+        macaroon = json.loads(base64.b64decode(token))["v"]
+    elif file := os.environ.get("CREDS_FILE"):
+        macaroon = json.loads(base64.b64decode(Path(file).read_text()))["v"]
+    else:
+        raise RuntimeError("Must set one of CHARMHUB_TOKEN, CREDS_FILE envvars.")
+    headers = {"Authorization": f"Macaroon {macaroon}"}
+
+    url = f"https://api.charmhub.io/v1/charm/{charm_name}/releases"
+    with urlopen(Request(url, headers=headers), timeout=10) as response:
+        body = response.read()
+
+    # Output looks like this:
+    # {
+    #   "channel-map": [
+    #     {
+    #       "base": {
+    #         "architecture": "amd64",
+    #         "channel": "20.04",
+    #         "name": "ubuntu"
+    #       },
+    #       "channel": "1.0/beta",
+    #       "expiration-date": null,
+    #       "progressive": {
+    #         "paused": null,
+    #         "percentage": null
+    #       },
+    #       "resources": [
+    #         {
+    #           "name": "grafana-image",
+    #           "revision": 62,
+    #           "type": "oci-image"
+    #         },
+    #         {
+    #           "name": "litestream-image",
+    #           "revision": 43,
+    #           "type": "oci-image"
+    #         }
+    #       ],
+    #       "revision": 93,
+    #       "when": "2023-11-22T09:12:26Z"
+    #     },
+    return json.loads(body)
+
+
+def obtain_revisions_from_charmhub(
+    charm_name: str, channel: str, base_arch: str, base_channel: str
+) -> dict:
+    """Obtain revisions for a given channel and arch.
+
+    Args:
+        charm_name: e.g. "grafana-k8s".
+        channel: e.g. "latest/edge".
+        base_arch: base architecture, e.g. "amd64".
+        base_channel: e.g. "22.04". TODO: remove arg and auto pick the latest
+
+    Returns: Dict of resources. Looks like this:
+        {
+          "grafana-k8s": {
+            "revision": 106,
+            "resources": {
+              "grafana-image": 68,
+              "litestream-image": 43
+            }
+          }
+        }
+    """
+    releases = obtain_charm_releases(charm_name)
+    for channel_dict in releases["channel-map"]:
+        print(
+            charm_name,
+            channel_dict["channel"],
+            channel_dict["base"]["architecture"],
+            channel_dict["base"]["channel"],
+        )
+        if not (
+            channel_dict["channel"] == channel
+            and channel_dict["base"]["architecture"] == base_arch
+            and channel_dict["base"]["channel"] == base_channel
+        ):
+            continue
+
+        return {
+            charm_name: {
+                "revision": channel_dict["revision"],
+                "resources": {res["name"]: res["revision"] for res in channel_dict["resources"]},
+            }
+        }
+
+    raise ValueError(
+        f"Didn't find any entry in {charm_name} releases with {base_arch}/{base_channel}"
+    )
+
+
+def freeze_bundle(bundle: dict, cleanup: bool = True):
+    """Take a bundle (dict) and update (freeze) revision entries."""
+    bundle = bundle.copy()
+    for app_name in bundle["applications"]:
+        app = bundle["applications"][app_name]
+        charm_name = app["charm"]
+        app_channel = app["channel"] if "/" in app["channel"] else f"latest/{app['channel']}"
+        # TODO externalize "base_arch" as an input to the script.
+        frozen_app = obtain_revisions_from_charmhub(charm_name, app_channel, "amd64", "22.04")
+        app["revision"] = frozen_app[charm_name]["revision"]
+        app["resources"].update(frozen_app[charm_name]["resources"])
+
+        if cleanup:
+            app.pop("constraints", None)
+            app.pop("storage", None)
+
+    return bundle
+
+
+def main():
+    if len(sys.argv) != 2:
+        raise RuntimeError("Expecting one arg: path to bundle yaml")
+
+    bundle_path = sys.argv[1]
+    frozen = freeze_bundle(yaml.safe_load(Path(bundle_path).read_text()))
+    print(yaml.safe_dump(frozen))
+
+if __name__ == "__main__":
+    main()

scripts/pyproject.toml

@@ -11,11 +11,12 @@ authors = [
     {name = "Observability team"}
 ]
 description = "Helper scripts for COS charms and bundles"
-version = "0.0.1"
+version = "0.0.2"
 requires-python = ">=3.8"
 dependencies = [
     "jinja2"
 ]
 
 [project.scripts]
-render-bundle = "render_bundle:main"
+render-bundle = "render_bundle:main"
+freeze-bundle = "freeze_bundle:main"