From 0e85e5c828d523ba8d29404801310a2e8b277610 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Thu, 7 Sep 2023 17:26:50 +0200 Subject: [PATCH] ci: add scheduled tests for continuous testing --- .github/workflows/Continuous-Testing.yaml | 63 ++++++++++++ src/tests/get_released_revisions.py | 111 ++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 .github/workflows/Continuous-Testing.yaml create mode 100755 src/tests/get_released_revisions.py diff --git a/.github/workflows/Continuous-Testing.yaml b/.github/workflows/Continuous-Testing.yaml new file mode 100644 index 000000000..3f7d6b290 --- /dev/null +++ b/.github/workflows/Continuous-Testing.yaml @@ -0,0 +1,63 @@ +name: Continuous image testing + +on: + schedule: + - cron: "*/5 * * * *" + +jobs: + list-released-images: + runs-on: ubuntu-latest + name: List the revisions of released images + outputs: + released-revisions-matrix: ${{ steps.prepare-test-matrix.outputs.released-revisions-matrix }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - run: pip install -r src/tests/requirements.txt + + - name: Prepare test matrix + id: prepare-test-matrix + run: ./src/tests/get_released_revisions.py --oci-images-path $PWD/oci + + dispatch-tests: + runs-on: ubuntu-latest + name: Dispatch tests for released images + needs: [list-released-images] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.list-released-images.outputs.released-revisions-matrix) }} + steps: + - name: Run tests for ${{ matrix.source-image }} + # Using this actions cause others can have this problem: + # https://github.com/convictional/trigger-workflow-and-wait/issues/61 + uses: mathze/workflow-dispatch-action@v1.2.0 + id: run-tests + env: + IS_A_ROCK: ${{ matrix.dockerfile-build == '' && true || false }} + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ github.ref_name }} + fail-on-error: true + workflow-name: Tests.yaml + # For continuous auditing, let's assume all images are NOT ROCKs and + # thus only run the most generic tests + payload: '{ "oci-image-name": "${{ matrix.source-image }}", "oci-image-path": "oci/${{ matrix.name }}", "is-a-rock": false, "test-from": "registry"}' + trigger-timeout: "5m" + wait-timeout: "45m" + run-id: dummy + use-marker-step: true + + - name: Write step summary + run: | + url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-tests.outputs.run-id }}' + echo " - Triggered tests for '${{ matrix.source-image }}' at [${url}](${url})" >> "$GITHUB_STEP_SUMMARY" + + - name: Enforce test conclusion + if: ${{ steps.run-tests.outputs.run-conclusion != 'success' }} + # The previous step doesn't always raise an error + run: | + url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-tests.outputs.run-id }}' + echo "Testing of image '${{ matrix.source-image }}' failed at [${url}](${url})." + exit 1 diff --git a/src/tests/get_released_revisions.py b/src/tests/get_released_revisions.py new file mode 100755 index 000000000..68e5107da --- /dev/null +++ b/src/tests/get_released_revisions.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +"""Scans the OCI images directory, and for each one, looks up the currently +released revision numbers. From that number, it queries GHCR in order to +form and return a list of image names in their canonical format, i.e.: + ghcr.io/canonical/oci-factory/:_ + ... + +TODO: this script could eventually be adjusted and converted to a Temporal +Activity that runs from within a scheduled workflow. +""" + +import argparse +import docker +import json +import logging +import os +import sys + +SKOPEO_IMAGE = os.getenv("SKOPEO_IMAGE", "quay.io/skopeo/stable:v1.13") +REGISTRY = "ghcr.io/canonical/oci-factory" + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + +def get_image_name_in_registry(img_name: str, revision: str) -> str: + """For a given revision number, search the registry for that image's tag + + :param img_name: name of the container image + :param revision: revision number of the tag we're looking for + """ + + d_client = docker.from_env() + + tagless_image_name = f"{REGISTRY}/{img_name}" + cmd = f"list-tags docker://{tagless_image_name}" + logging.info(f"Running Skopeo with '{cmd}'") + try: + all_tags = json.loads( + d_client.containers.run( + SKOPEO_IMAGE, + command=cmd, + remove=True, + ).strip() + )["Tags"] + except docker.errors.ContainerError as err: + if "timeout" not in str(err): + raise + logging.error( + f"Timed out while listing tags for {tagless_image_name}: {str(err)}" + ) + + for tag in all_tags: + if tag.endswith(revision): + return f"{tagless_image_name}:{tag}" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=str( + "Goes through all the OCI images and " + "gets the revision tags for the released images" + ) + ) + parser.add_argument( + "--oci-images-path", + required=True, + help="absolute path to the OCI folder where all images are", + ) + + args = parser.parse_args() + + logging.info(f"Looping through OCI images in {args.oci_images_path}") + + released_revisions = {} + ghcr_images = [] + for img in os.listdir(args.oci_images_path): + _releases_file = f"{args.oci_images_path}/{img}/_releases.json" + if not os.path.isfile(_releases_file): + continue + + with open(_releases_file) as rf: + releases = json.load(rf) + + released_revisions[img] = [] + for risks in releases.values(): + for targets in risks.values(): + try: + if int(targets["target"]) in released_revisions[img]: + continue + except ValueError: + # this target is following another tag and thus is not + # a revision number + continue + + released_revisions[img].append(int(targets["target"])) + ghcr_images.append( + { + "name": img, + "source-image": get_image_name_in_registry( + img, targets["target"] + ), + } + ) + + logging.info(f"Released revisions: {json.dumps(released_revisions, indent=2)}") + logging.info(f"Released revisions in GHCR: {ghcr_images}") + + matrix = {"include": ghcr_images} + with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out: + print(f"released-revisions-matrix={matrix}", file=gh_out)