diff --git a/.github/workflows/_bundle-release.yaml b/.github/workflows/_bundle-release.yaml new file mode 100644 index 0000000..88c15cf --- /dev/null +++ b/.github/workflows/_bundle-release.yaml @@ -0,0 +1,95 @@ +name: Integration matrix + +on: + workflow_call: + inputs: + charm-channel: + type: string + required: true + +defaults: + run: + shell: bash + +jobs: + validate-inputs: + name: Validate action inputs + runs-on: ubuntu-latest + steps: + - name: Validate charm-channel + if: ${{ inputs.charm-channel != 'edge' && inputs.charm-channel != 'beta' && inputs.charm-channel != 'candidate' && inputs.charm-channel != 'stable'}} + run: | + echo "Error: The 'charm-channel' input must be one of edge, beta, candidate, stable." + exit 1 + render-freeze-bundle: + name: Render and freeze bundle + needs: [ validate-inputs ] + # We render and freeze at the start to avoid possible races, in case a new charm was release + # while these tests are still running. + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Render and freeze bundle + env: + CHARMHUB_TOKEN: "${{ secrets.CHARMHUB_TOKEN }}" + run: | + tox -e render-bundle -- --channel=${{ inputs.charm-channel }} + python3 scripts/freeze_bundle.py bundle.yaml > bundle.yaml + - name: Upload bundle as artifact to be used by the next job + uses: actions/upload-artifact@v3 + with: + name: frozen-bundle + path: bundle.yaml + integration-matrix: + name: Matrix tests for charms + needs: [ render-freeze-bundle ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + juju-track: ["3.4" ] + microk8s-channel: [ "1.27-strict/stable", "1.28-strict/stable" ] + include: + - juju-track: "3.4" + juju-channel: "3.4/stable" + juju-agent-version: "3.4.0" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Get prefsrc + run: | + echo "IPADDR=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc')" >> $GITHUB_ENV + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + juju-channel: ${{ matrix.juju-channel }} + provider: microk8s + channel: ${{ matrix.microk8s-channel }} + microk8s-addons: "hostpath-storage dns metallb:${{ env.IPADDR }}-${{ env.IPADDR }}" + bootstrap-options: "--agent-version ${{ matrix.juju-agent-version }}" + - name: Update python-libjuju dependency to match juju version + # Assuming the dep is given on a separate tox.ini line + run: sed -E -i 's/^\s*juju\s*~=.+/ juju~=${{ matrix.juju-track }}.0/g' tox.ini + - uses: actions/download-artifact@v3 + with: + name: frozen-bundle + - name: Run tests (juju ${{ matrix.juju-channel }}, microk8s ${{ matrix.microk8s-channel }}) + run: tox -e integration + - name: Dump logs + if: failure() + uses: canonical/charming-actions/dump-logs@main + release-pinned-bundle: + name: Release pinned bundle + needs: [ integration-matrix ] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v3 + with: + name: frozen-bundle + - name: Upload bundle to the pinned track + uses: canonical/charming-actions/upload-bundle@1.0.0 + with: + channel: "pinned/${{ inputs.charm-channel }}" + credentials: "${{ secrets.CHARMHUB_TOKEN }}" + github-token: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/.github/workflows/bundle-release.yaml b/.github/workflows/bundle-release.yaml new file mode 100644 index 0000000..7257e0e --- /dev/null +++ b/.github/workflows/bundle-release.yaml @@ -0,0 +1,19 @@ +name: Release Bundle + +on: + workflow_call: + +jobs: + per-channel-integration-matrix: + name: Per-channel integration matrix + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + charm-channel: [ "edge", "beta", "candidate", "stable" ] + steps: + - name: Run integration matrix for ${{ matrix.charm-channel }} channel + uses: canonical/observability/.github/workflows/_bundle-release.yaml@main + with: + charm-channel: ${{ matrix.charm-channel }} + diff --git a/README.md b/README.md index 9a27096..f121f62 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,10 @@ Periodically, CI checks whether the charm libraries are up-to-date; if not (i.e. There's also a manual action to promote the charm (i.e., from `latest/edge` to `latest/beta`), making the process more user-friendly. ### Bundle Workflows -| On PRs | -| ------------------------------------| -| **`bundle-pull-request.yaml`** | -| `├── _charm-codeql-analysis.yaml` | +| On PRs | Periodically | +| ------------------------------------| ----------------------------| +| **`bundle-pull-request.yaml`** | **`bundle-release.yaml`** | +| `├── _charm-codeql-analysis.yaml` | `├── _bundle-release.yaml` | | `├── _charm-linting.yaml` | | `└── _charm-tests-integration.yaml` | @@ -56,7 +56,7 @@ Whenever a PR is opened to a bundle repository, some quality checks are run: * run the Canonical inclusive naming workflow. * run linting, analyses and tests to ensure the code quality. - +Periodically, integration matrix tests will run against a COS-related bundle and then, once the integration tests pass for any of the tracks: `edge`, `beta`, `candidate`, `stable`, a bundle gets released to each respective pinned track on Charmhub. ### Rock Workflows @@ -105,8 +105,13 @@ This repo also contains a `scripts` directory that could hold helper scripts for ### `render-bundle` This helper script is used by COS bundles as a `pip` package in a `tox.ini` file to render a `bundle.yaml.j2` template into a `bundle.yaml` file that can be deployed using `juju deploy ./bundle.yaml`. +### `freeze-bundle` +This script takes a `bundle.yaml` file and for each `application` along with its defined channel, it obtains the revision number for that application charm from Charmhub and updates `bundle.yaml` file with a pinned `revision` on each application. Currently, this script is used inside the `bundle-release.yaml` workflow. + ### Contributing To add similar helper scripts (e.g: `my_helper.py`) to be used as a `pip` package: 1. Add the script inside `scripts` directory. -2. In `scripts/pyproject.toml`, under `[project.scripts]`, add an entrypoint to your newly added script. \ No newline at end of file +2. In `scripts/pyproject.toml`, under `[project.scripts]`, add an entrypoint to your newly added script. +3. Increment `version` in `scripts/pyproject.toml`. +4. Add the script's description in `README.md`. \ No newline at end of file diff --git a/scripts/freeze_bundle.py b/scripts/freeze_bundle.py new file mode 100644 index 0000000..d9ccb02 --- /dev/null +++ b/scripts/freeze_bundle.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + +"""This script updates a bundle.yaml file with revisions from charmhub.""" + +import base64 +import json +import os +import sys +from pathlib import Path +from urllib.request import Request, urlopen + +import yaml + + +def obtain_charm_releases(charm_name: str) -> dict: + """Obtain charm releases from charmhub as a dict. + + Args: + charm_name: e.g. "grafana-k8s". + """ + if token := os.environ.get("CHARMHUB_TOKEN"): + macaroon = json.loads(base64.b64decode(token))["v"] + elif file := os.environ.get("CREDS_FILE"): + macaroon = json.loads(base64.b64decode(Path(file).read_text()))["v"] + else: + raise RuntimeError("Must set one of CHARMHUB_TOKEN, CREDS_FILE envvars.") + headers = {"Authorization": f"Macaroon {macaroon}"} + + url = f"https://api.charmhub.io/v1/charm/{charm_name}/releases" + with urlopen(Request(url, headers=headers), timeout=10) as response: + body = response.read() + + # Output looks like this: + # { + # "channel-map": [ + # { + # "base": { + # "architecture": "amd64", + # "channel": "20.04", + # "name": "ubuntu" + # }, + # "channel": "1.0/beta", + # "expiration-date": null, + # "progressive": { + # "paused": null, + # "percentage": null + # }, + # "resources": [ + # { + # "name": "grafana-image", + # "revision": 62, + # "type": "oci-image" + # }, + # { + # "name": "litestream-image", + # "revision": 43, + # "type": "oci-image" + # } + # ], + # "revision": 93, + # "when": "2023-11-22T09:12:26Z" + # }, + return json.loads(body) + + +def obtain_revisions_from_charmhub( + charm_name: str, channel: str, base_arch: str, base_channel: str +) -> dict: + """Obtain revisions for a given channel and arch. + + Args: + charm_name: e.g. "grafana-k8s". + channel: e.g. "latest/edge". + base_arch: base architecture, e.g. "amd64". + base_channel: e.g. "22.04". TODO: remove arg and auto pick the latest + + Returns: Dict of resources. Looks like this: + { + "grafana-k8s": { + "revision": 106, + "resources": { + "grafana-image": 68, + "litestream-image": 43 + } + } + } + """ + releases = obtain_charm_releases(charm_name) + for channel_dict in releases["channel-map"]: + print( + charm_name, + channel_dict["channel"], + channel_dict["base"]["architecture"], + channel_dict["base"]["channel"], + ) + if not ( + channel_dict["channel"] == channel + and channel_dict["base"]["architecture"] == base_arch + and channel_dict["base"]["channel"] == base_channel + ): + continue + + return { + charm_name: { + "revision": channel_dict["revision"], + "resources": {res["name"]: res["revision"] for res in channel_dict["resources"]}, + } + } + + raise ValueError( + f"Didn't find any entry in {charm_name} releases with {base_arch}/{base_channel}" + ) + + +def freeze_bundle(bundle: dict, cleanup: bool = True): + """Take a bundle (dict) and update (freeze) revision entries.""" + bundle = bundle.copy() + for app_name in bundle["applications"]: + app = bundle["applications"][app_name] + charm_name = app["charm"] + app_channel = app["channel"] if "/" in app["channel"] else f"latest/{app['channel']}" + # TODO externalize "base_arch" as an input to the script. + frozen_app = obtain_revisions_from_charmhub(charm_name, app_channel, "amd64", "22.04") + app["revision"] = frozen_app[charm_name]["revision"] + app["resources"].update(frozen_app[charm_name]["resources"]) + + if cleanup: + app.pop("constraints", None) + app.pop("storage", None) + + return bundle + + +def main(): + if len(sys.argv) != 2: + raise RuntimeError("Expecting one arg: path to bundle yaml") + + bundle_path = sys.argv[1] + frozen = freeze_bundle(yaml.safe_load(Path(bundle_path).read_text())) + print(yaml.safe_dump(frozen)) + +if __name__ == "__main__": + main() diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index be96adf..1d1344a 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -11,11 +11,12 @@ authors = [ {name = "Observability team"} ] description = "Helper scripts for COS charms and bundles" -version = "0.0.1" +version = "0.0.2" requires-python = ">=3.8" dependencies = [ "jinja2" ] [project.scripts] -render-bundle = "render_bundle:main" \ No newline at end of file +render-bundle = "render_bundle:main" +freeze-bundle = "freeze_bundle:main" \ No newline at end of file