ci: add scheduled tests for continuous testing

.github/workflows/Continuous-Testing.yaml

@@ -0,0 +1,63 @@
+name: Continuous image testing
+
+on:
+  schedule:
+    - cron: "*/5 * * * *"
+
+jobs:
+  list-released-images:
+    runs-on: ubuntu-latest
+    name: List the revisions of released images
+    outputs:
+      released-revisions-matrix: ${{ steps.prepare-test-matrix.outputs.released-revisions-matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - run: pip install -r src/tests/requirements.txt
+
+      - name: Prepare test matrix
+        id: prepare-test-matrix
+        run: ./src/tests/get_released_revisions.py --oci-images-path $PWD/oci
+
+  dispatch-tests:
+    runs-on: ubuntu-latest
+    name: Dispatch tests for released images
+    needs: [list-released-images]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.list-released-images.outputs.released-revisions-matrix) }}
+    steps:
+      - name: Run tests for ${{ matrix.source-image }}
+        # Using this actions cause others can have this problem:
+        # https://github.com/convictional/trigger-workflow-and-wait/issues/61
+        uses: mathze/[email protected]
+        id: run-tests
+        env:
+          IS_A_ROCK: ${{ matrix.dockerfile-build == '' && true || false }}
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ github.ref_name }}
+          fail-on-error: true
+          workflow-name: Tests.yaml
+          # For continuous auditing, let's assume all images are NOT ROCKs and
+          # thus only run the most generic tests
+          payload: '{ "oci-image-name": "${{ matrix.source-image }}", "oci-image-path": "oci/${{ matrix.name }}", "is-a-rock": false, "test-from": "registry"}'
+          trigger-timeout: "5m"
+          wait-timeout: "45m"
+          run-id: dummy
+          use-marker-step: true
+
+      - name: Write step summary
+        run: |
+          url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-tests.outputs.run-id }}'
+          echo " - Triggered tests for '${{ matrix.source-image }}' at [${url}](${url})" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Enforce test conclusion
+        if: ${{ steps.run-tests.outputs.run-conclusion != 'success' }}
+        # The previous step doesn't always raise an error
+        run: |
+          url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-tests.outputs.run-id }}'
+          echo "Testing of image '${{ matrix.source-image }}' failed at [${url}](${url})."
+          exit 1

.github/workflows/Tests.yaml

@@ -180,10 +180,16 @@ jobs:
     name: Vulnerability scan
     needs: [fetch-oci-image]
     outputs:
-      vulnerability-report: ${{ steps.vulnerability-scan.outputs.report }}
+      vulnerability-report: ${{ steps.vulnerability-report.outputs.name }}
     steps:
       - uses: actions/checkout@v3
 
+      - id: vulnerability-report
+        run: |
+          full_name="${{ inputs.oci-image-name }}${{ inputs.vulnerability-report-suffix }}"
+          final_name="$(echo ${full_name} | sed 's/ghcr.io\/canonical\/oci-factory\///g' | tr ':' '_')"
+          echo "name=$final_name" >> "$GITHUB_OUTPUT" 
+
       - uses: actions/cache/restore@v3
         with:
           path: ${{ env.TEST_IMAGE_NAME}}
@@ -223,21 +229,19 @@ jobs:
           # CVE-2021-43565, CVE-2022-27191
           skip-files: /bin/pebble
           # missing ${{ runner.arch }}
-          output: '${{ inputs.oci-image-name }}${{ inputs.vulnerability-report-suffix }}'
+          output: '${{ steps.vulnerability-report.outputs.name }}'
           image-ref: '${{ env.TEST_IMAGE_NAME}}:${{ env.TEST_IMAGE_TAG }}'
 
-      - id: vulnerability-scan
-        if: ${{ always() }}
+      - if: ${{ always() }}
         run: |
-          report="${{ inputs.oci-image-name }}${{ inputs.vulnerability-report-suffix }}"
-          cat $report
+          cat ${{ steps.vulnerability-report.outputs.name }}
           echo "report=$report" >> "$GITHUB_OUTPUT"
 
       - uses: actions/cache/save@v3
         if: ${{ always() }}
         with:
-          path: ${{ steps.vulnerability-scan.outputs.report }}
-          key: ${{ github.run_id }}-${{ steps.vulnerability-scan.outputs.report }}
+          path: ${{ steps.vulnerability-report.outputs.name }}
+          key: ${{ github.run_id }}-${{ steps.vulnerability-report.outputs.name }}
 
 
   test-malware:

src/tests/get_released_revisions.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+"""Scans the OCI images directory, and for each one, looks up the currently
+released revision numbers. From that number, it queries GHCR in order to
+form and return a list of image names in their canonical format, i.e.:
+    ghcr.io/canonical/oci-factory/<img-name>:<canonical-track>_<revision>
+    ...
+    
+TODO: this script could eventually be adjusted and converted to a Temporal
+Activity that runs from within a scheduled workflow.
+"""
+
+import argparse
+import docker
+import json
+import logging
+import os
+import sys
+
+SKOPEO_IMAGE = os.getenv("SKOPEO_IMAGE", "quay.io/skopeo/stable:v1.13")
+REGISTRY = "ghcr.io/canonical/oci-factory"
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+
+def get_image_name_in_registry(img_name: str, revision: str) -> str:
+    """For a given revision number, search the registry for that image's tag
+
+    :param img_name: name of the container image
+    :param revision: revision number of the tag we're looking for
+    """
+
+    d_client = docker.from_env()
+
+    tagless_image_name = f"{REGISTRY}/{img_name}"
+    cmd = f"list-tags docker://{tagless_image_name}"
+    logging.info(f"Running Skopeo with '{cmd}'")
+    try:
+        all_tags = json.loads(
+            d_client.containers.run(
+                SKOPEO_IMAGE,
+                command=cmd,
+                remove=True,
+            ).strip()
+        )["Tags"]
+    except docker.errors.ContainerError as err:
+        if "timeout" not in str(err):
+            raise
+        logging.error(
+            f"Timed out while listing tags for {tagless_image_name}: {str(err)}"
+        )
+
+    for tag in all_tags:
+        if tag.endswith(revision):
+            return f"{tagless_image_name}:{tag}"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=str(
+            "Goes through all the OCI images and "
+            "gets the revision tags for the released images"
+        )
+    )
+    parser.add_argument(
+        "--oci-images-path",
+        required=True,
+        help="absolute path to the OCI folder where all images are",
+    )
+
+    args = parser.parse_args()
+
+    logging.info(f"Looping through OCI images in {args.oci_images_path}")
+
+    released_revisions = {}
+    ghcr_images = []
+    for img in os.listdir(args.oci_images_path):
+        _releases_file = f"{args.oci_images_path}/{img}/_releases.json"
+        if not os.path.isfile(_releases_file):
+            continue
+
+        with open(_releases_file) as rf:
+            releases = json.load(rf)
+
+        released_revisions[img] = []
+        for risks in releases.values():
+            for targets in risks.values():
+                try:
+                    if int(targets["target"]) in released_revisions[img]:
+                        continue
+                except ValueError:
+                    # this target is following another tag and thus is not
+                    # a revision number
+                    continue
+
+                released_revisions[img].append(int(targets["target"]))
+                ghcr_images.append(
+                    {
+                        "name": img,
+                        "source-image": get_image_name_in_registry(
+                            img, targets["target"]
+                        ),
+                    }
+                )
+
+    logging.info(f"Released revisions: {json.dumps(released_revisions, indent=2)}")
+    logging.info(f"Released revisions in GHCR: {ghcr_images}")
+
+    matrix = {"include": ghcr_images}
+    with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
+        print(f"released-revisions-matrix={matrix}", file=gh_out)