From 7aaf2512f5307ec8f02a30a4f1f8db6131ff89ee Mon Sep 17 00:00:00 2001 From: thesayyn Date: Tue, 5 Mar 2024 19:16:19 -0800 Subject: [PATCH] ci: implement image diff pipeline --- .github/workflows/buildifier.yaml | 11 +- .github/workflows/image-check.yaml | 97 ++++++++++++ private/oci/sign_and_push.bzl | 21 ++- private/tools/BUILD.bazel | 15 ++ private/tools/diff.bash | 238 +++++++++++++++++++++++++++++ 5 files changed, 374 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/image-check.yaml create mode 100644 private/tools/BUILD.bazel create mode 100755 private/tools/diff.bash diff --git a/.github/workflows/buildifier.yaml b/.github/workflows/buildifier.yaml index 0be154cb6..04e1f0ed6 100644 --- a/.github/workflows/buildifier.yaml +++ b/.github/workflows/buildifier.yaml @@ -2,31 +2,28 @@ name: Buildifier on: pull_request: - branches: [ 'main' ] + branches: ["main"] permissions: contents: read jobs: - autoformat: name: Auto-format and Check runs-on: ubuntu-latest steps: - - name: Set up Go 1.15.x + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.15.x - id: go + go-version: "1.21" - name: Check out code uses: actions/checkout@v4 - name: Install Dependencies run: | - cd $(mktemp -d) - GO111MODULE=on go get github.com/bazelbuild/buildtools/buildifier@3.2.0 + go install github.com/bazelbuild/buildtools/buildifier@3.2.0 - name: Run buildifier shell: bash diff --git a/.github/workflows/image-check.yaml b/.github/workflows/image-check.yaml new file mode 100644 index 000000000..e249f619c --- /dev/null +++ b/.github/workflows/image-check.yaml @@ -0,0 +1,97 @@ +name: Image Check + +on: + workflow_dispatch: + pull_request: + branches: ["main"] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + diff: + runs-on: distroless-ci-large-ubuntu-20.04 # custom runner most compatible with debian 11 + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.21" + - uses: actions/cache@v4 + with: + path: | + ~/.cache/bazel-repo + key: bazel-cache-deps-ci2-${{ github.sha }} + restore-keys: | + bazel-cache-deps-ci2-${{ github.sha }} + bazel-cache-deps-ci2- + + - name: Build all images + run: bazel build //:sign_and_push + + - name: Install Deps + run: | + go install github.com/google/go-containerregistry/cmd/crane@main + go install github.com/reproducible-containers/diffoci/cmd/diffoci@master + go install filippo.io/mkcert@master + sudo curl -fsSL "https://github.com/project-zot/zot/releases/download/v2.0.2-rc2/zot-linux-amd64-minimal" > /usr/local/bin/zot + sudo chmod +x /usr/local/bin/zot + + - name: Diff All Images + id: diff + run: | + ./private/tools/diff.bash \ + --query-bazel --registry-spawn-https \ + --head-ref ${{ github.head_ref }} \ + --base-ref ${{ github.event.pull_request.base.ref }} \ + --set-github-output-on-diff \ + --skip-image-index \ + --jobs $(($(nproc --all) * 2)) \ + --logs ./verbose.log \ + --report ./report.log + + - uses: actions/upload-artifact@v4 + id: report + with: + name: "Report" + path: | + ./verbose.log + ./report.log + + - uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: 🌳 🔄 Image Check + + - name: Report diff + if: ${{ steps.diff.outputs.changed_targets }} + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + 🌳 🔄 Image Check + + This pull request has modified the following images: + + ```starlark + ${{steps.diff.outputs.changed_targets}} + ``` + + You can check the details in the report [here](${{steps.report.outputs.artifact-url}}) + edit-mode: replace + + - name: Report no diff + if: ${{ !steps.diff.outputs.changed_targets }} + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + 🌳 🔄 Image Check + This pull request doesn't make any changes to the images. 👍 + You can check the details in the report [here](${{steps.report.outputs.artifact-url}}) + edit-mode: replace diff --git a/private/oci/sign_and_push.bzl b/private/oci/sign_and_push.bzl index 2c739238e..a247fb9ff 100644 --- a/private/oci/sign_and_push.bzl +++ b/private/oci/sign_and_push.bzl @@ -1,5 +1,8 @@ -load("@rules_oci//oci:defs.bzl", "oci_push") +"rules for signing, attesting and pushing images" + +load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_oci//cosign:defs.bzl", "cosign_attest", "cosign_sign") +load("@rules_oci//oci:defs.bzl", "oci_push") load("//private/pkg:oci_image_spdx.bzl", "oci_image_spdx") PUSH_AND_SIGN_CMD = """\ @@ -69,6 +72,7 @@ def sign_and_push_all(name, images): images: a dict where keys are fully qualified image url and values are image labels """ image_dict = dict() + query_dict = dict() for (idx, (url, image)) in enumerate(images.items()): oci_push( name = "{}_{}_push".format(name, idx), @@ -101,6 +105,21 @@ def sign_and_push_all(name, images): ) image_dict[":{}_{}".format(name, idx)] = url + query_dict[image] = url.split(":") + [":{}_{}_push".format(name, idx)] + + write_file( + name = name + ".query", + content = [ + "{repo} {tag} {push_label} {image_label}".format( + repo = ref[0], + tag = ref[1], + push_label = ref[2], + image_label = image, + ) + for (image, ref) in query_dict.items() + ], + out = name + "_query", + ) sign_and_push( name = name, diff --git a/private/tools/BUILD.bazel b/private/tools/BUILD.bazel new file mode 100644 index 000000000..594d3a9be --- /dev/null +++ b/private/tools/BUILD.bazel @@ -0,0 +1,15 @@ +sh_binary( + name = "diff", + srcs = ["diff.bash"], + args = [ + "--head-ref", + "test", + "--base-ref", + "test", + "--report", + "./report.log", + "--query-bazel", + "--registry-spawn-https", + "--cd-into-workspace", + ], +) diff --git a/private/tools/diff.bash b/private/tools/diff.bash new file mode 100755 index 000000000..851b1770d --- /dev/null +++ b/private/tools/diff.bash @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -o pipefail -o errexit -o nounset + +# ./private/tools/diff.bash --head-ref test --base-ref test --query-bazel --registry-spawn --report ./report.log + +STDERR=$(mktemp) + +# Upon exiting, stop the registry and print STDERR on non-zero exit code. +on_exit() { + last_exit_code=$? + set +o errexit + if [[ $last_exit_code != 0 ]]; then + echo "" + echo "💥 Something went wrong." + if [[ $(wc -c <"${STDERR}") -gt 0 ]]; then + echo "" + echo "Here's the STDERR:" + echo "" + cat $STDERR + fi + fi + pkill -P $$ +} +trap "on_exit" EXIT + +PID= +HEAD_REF= +BASE_REF= +QUERY_FILE= +REPORT_FILE= +REGISTRY= +JOBS= +STDERR=$(mktemp) +CHANGED_IMAGES_FILE=$(mktemp) +SET_GITHUB_OUTPUT="0" +ONLY= +SKIP_INDEX="0" + +while (($# > 0)); do + case $1 in + --base-ref) + BASE_REF="$2" + shift 2 + ;; + --head-ref) + HEAD_REF="$2" + shift 2 + ;; + --registry) + REGISTRY="$2" + shift 2 + ;; + --registry-spawn) + REGISTRY="spawn" + shift + ;; + --registry-spawn-https) + REGISTRY="spawn_https" + shift + ;; + --query) + QUERY_FILE="$2" + shift 2 + ;; + --query-bazel) + QUERY_FILE="bazel" + shift + ;; + --report) + REPORT_FILE="$2" + shift 2 + ;; + --set-github-output-on-diff) + SET_GITHUB_OUTPUT="1" + echo "changed_targets=" >> "$GITHUB_OUTPUT" + shift + ;; + --jobs) + JOBS="$2" + shift 2 + ;; + --logs) + STDERR="$2" + shift 2 + ;; + --only) + ONLY="$2" + shift 2 + ;; + --cd-into-workspace) + cd $BUILD_WORKSPACE_DIRECTORY + shift + ;; + --skip-image-index) + SKIP_INDEX="1" + shift + ;; + *) + echo "unknown arg $1" + exit 1 + ;; + esac +done + +if [[ -z "${REGISTRY}" ]]; then + echo "--registry is required." + exit 1 +fi + +if [[ -z "${BASE_REF}" ]]; then + echo "--base-ref is required." + exit 1 +fi + +if [[ -z "${HEAD_REF}" ]]; then + echo "--head-ref is required." + exit 1 +fi + +if [[ -z "${QUERY_FILE}" ]]; then + echo "--query or --query-bazel must be provided" + exit 1 +fi + +# Redirect stderr to the $STDERR temp file for the rest of the script. +exec 2>>"${STDERR}" + +DISK_STORAGE="/tmp/diff-storage" + +if [[ "${QUERY_FILE}" == "bazel" ]]; then + bazel build :sign_and_push.query + QUERY_FILE=$(bazel cquery --output=files :sign_and_push.query) +fi + +if [[ "${REGISTRY}" == "spawn_https" ]]; then + # Make a self signed cert + rm -f /tmp/localhost.pem /tmp/localhost-key.pem + rm -rf $DISK_STORAGE + mkcert -install + (cd /tmp && mkcert localhost) + echo '{ + "http":{ + "address":"127.0.0.1", "port":"4564", + "tls": { + "cert":"/tmp/localhost.pem", + "key":"/tmp/localhost-key.pem" + } + }, + "log": { "level": "info" }, + "storage":{"rootDirectory":"/tmp/diff-storage"} + }' >/tmp/cfg.json + REGISTRY="localhost:4564" + zot serve /tmp/cfg.json 1>&2 & + sleep 1 +fi + +if [[ "${REGISTRY}" == "spawn" ]]; then + rm -rf $DISK_STORAGE + mkdir $DISK_STORAGE + REGISTRY="localhost:4564" + crane registry serve --address "$REGISTRY" --disk "$DISK_STORAGE" & +fi + +stamp_stage() { + local str="$1" + str=${str/"{COMMIT_SHA}"/"${HEAD_REF}"} + str=${str/"{REGISTRY}"/"${REGISTRY}"} + echo ${str/"{PROJECT_ID}"/"stage"} +} + +stamp_origin() { + local str=$1 + str=${str/"{COMMIT_SHA}"/"${BASE_REF}"} + str=${str/"{REGISTRY}"/"gcr.io"} + echo ${str/"{PROJECT_ID}"/"distroless"} +} + +function test_image() { + IFS=" " read -r repo tag push_label image_label <<<"$1" + + if [[ "${ONLY}" != "" && "${ONLY}" != "$image_label" ]]; then + return + fi + + repo_origin=$(stamp_origin "$repo") + repo_stage=$(stamp_stage "$repo") + tag_stamped=$(stamp_origin "$tag") + + if [[ "${SKIP_INDEX}" == "1" ]]; then + if ! crane manifest "$repo_origin:$tag_stamped" | jq -e '.mediaType == "application/vnd.oci.image.manifest.v1+json"' > /dev/null; then + echo "⏭️ Skipping image index $repo_origin:$tag_stamped " + return + fi + fi + + echo "" + echo "🚧 Diffing $repo_origin:$tag_stamped against $repo_stage:$tag_stamped" + echo "" + + bazel run $push_label -- --repository $repo_stage --tag $tag_stamped + if ! diffoci diff --pull=always --all-platforms --semantic "$repo_origin:$tag_stamped" "$repo_stage:$tag_stamped"; then + echo "" + echo " 🔬 To reproduce: bazel run //private/tools:diff -- --only $image_label" + echo "" + echo "👎 $repo_origin:$tag_stamped and $repo_stage:$tag_stamped are different." + if [[ "${SET_GITHUB_OUTPUT}" == "1" ]]; then + echo "$image_label" >> "$CHANGED_IMAGES_FILE" + fi + else + echo "" + echo "👍 $repo_origin:$tag_stamped and $repo_stage:$tag_stamped are identical." + fi +} + +if [[ -n "${REPORT_FILE}" ]]; then + echo "Report can be found in: $REPORT_FILE" + echo -n "" >$REPORT_FILE + sleep 1 + # Redirect rest of the file into both report file and stdout + exec 1> >(tee -a "${REPORT_FILE}") +fi + +# Parallelize using gnu parallel +if [[ "${JOBS}" -gt 0 ]]; then + export HEAD_REF BASE_REF REGISTRY REPORT_FILE SET_GITHUB_OUTPUT ONLY CHANGED_IMAGES_FILE SKIP_INDEX + export -f stamp_origin stamp_stage test_image + cat "${QUERY_FILE}" | parallel --eta --progress --jobs "${JOBS}" "set -o pipefail -o errexit -o nounset && test_image" +else + while IFS= read -r line || [ -n "$line" ]; do + test_image "${line}" + done <"${QUERY_FILE}" +fi + +if [[ "${SET_GITHUB_OUTPUT}" == "1" ]]; then + echo "changed_targets<> "$GITHUB_OUTPUT" + cat "$CHANGED_IMAGES_FILE" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" +fi \ No newline at end of file