From 0ba8c3b56993ac9aef38fe3bda36d226c3be5e42 Mon Sep 17 00:00:00 2001 From: Vladimir Slavik Date: Tue, 23 Aug 2022 10:28:54 +0200 Subject: [PATCH 1/5] Remove -devel branches from CI (#infra) --- .github/workflows/container-autoupdate-fedora.yml | 8 -------- .github/workflows/container-rebuild-action.yml | 1 - .github/workflows/kickstart-tests.yml | 2 +- .github/workflows/push-tests.yml | 1 - .github/workflows/tests.yml | 11 ++--------- dockerfile/anaconda-ci/Dockerfile | 1 - dockerfile/anaconda-rpm/Dockerfile | 1 - 7 files changed, 3 insertions(+), 22 deletions(-) diff --git a/.github/workflows/container-autoupdate-fedora.yml b/.github/workflows/container-autoupdate-fedora.yml index 280e3255d30..59a1c8b98df 100644 --- a/.github/workflows/container-autoupdate-fedora.yml +++ b/.github/workflows/container-autoupdate-fedora.yml @@ -17,14 +17,6 @@ jobs: container-tag: master branch: master - f36-devel: - if: false - uses: ./.github/workflows/container-rebuild-action.yml - secrets: inherit - with: - container-tag: f36-devel - branch: f36-devel - f36-release: if: false uses: ./.github/workflows/container-rebuild-action.yml diff --git a/.github/workflows/container-rebuild-action.yml b/.github/workflows/container-rebuild-action.yml index 368de4cb71d..c2cff4c84e5 100644 --- a/.github/workflows/container-rebuild-action.yml +++ b/.github/workflows/container-rebuild-action.yml @@ -2,7 +2,6 @@ name: Rebuild container images # Rebuilds both ci and rpm container images for a given "target". Currently known targets: # - master # - eln -# - fNN-devel # - fNN-release # # Image is: diff --git a/.github/workflows/kickstart-tests.yml b/.github/workflows/kickstart-tests.yml index 2ae5ebd576a..f4de52f271e 100644 --- a/.github/workflows/kickstart-tests.yml +++ b/.github/workflows/kickstart-tests.yml @@ -59,7 +59,7 @@ jobs: if [ "$TARGET_BRANCH" == "master" ]; then echo "::set-output name=skip_tests::skip-on-fedora" echo "::set-output name=platform::fedora_rawhide" - elif echo "$TARGET_BRANCH" | grep -qE "f[[:digit:]]+-(devel|release)$"; then + elif echo "$TARGET_BRANCH" | grep -qE "f[[:digit:]]+-release$"; then echo "::set-output name=skip_tests::skip-on-fedora" echo "::set-output name=platform::fedora_rawhide" elif echo "$TARGET_BRANCH" | grep -qE "rhel-8(\.[[:digit:]]+)?$"; then diff --git a/.github/workflows/push-tests.yml b/.github/workflows/push-tests.yml index 247fae4fc91..e9008d39530 100644 --- a/.github/workflows/push-tests.yml +++ b/.github/workflows/push-tests.yml @@ -3,7 +3,6 @@ on: push: branches: - master - - f[0-9]+-devel - f[0-9]+-release permissions: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1cd2b2e1d7b..be08ec65124 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,10 +16,9 @@ jobs: # the current branch always have the same name, so that it can be added to required # checks. # * For various branches, comment out what does not apply: - # on -devel, run everything # on -release, only -release # on master, only master and ELN - #release: [master, eln, '', f34-release] + #release: [master, eln, ''] release: [''] include: - release: '' @@ -29,9 +28,6 @@ jobs: # target_branch: 'master' # ci_tag: 'eln' # build-args: '--build-arg=image=quay.io/fedoraci/fedora:eln-x86_64' - #- release: '' - # target_branch: 'f35-devel' - # ci_tag: 'f35-devel' #- release: f35-release # target_branch: 'f35-release' # ci_tag: 'f35-release' @@ -93,7 +89,7 @@ jobs: fail-fast: false matrix: # For matrix details, see comments for the unit tests above. - #release: [master, eln, '', f34-release] + #release: [master, eln, ''] release: [''] include: - release: '' @@ -103,9 +99,6 @@ jobs: # target_branch: 'master' # ci_tag: 'eln' # build-args: '--build-arg=image=quay.io/fedoraci/fedora:eln-x86_64' - #- release: '' - # target_branch: 'f35-devel' - # ci_tag: 'f35-devel' #- release: f35-release # target_branch: 'f35-release' # ci_tag: 'f35-release' diff --git a/dockerfile/anaconda-ci/Dockerfile b/dockerfile/anaconda-ci/Dockerfile index 35f9315d29c..fb2e8983c4b 100644 --- a/dockerfile/anaconda-ci/Dockerfile +++ b/dockerfile/anaconda-ci/Dockerfile @@ -16,7 +16,6 @@ FROM ${image} # dependencies. # possible values: # master -# f35-devel # f35-release ARG git_branch diff --git a/dockerfile/anaconda-rpm/Dockerfile b/dockerfile/anaconda-rpm/Dockerfile index ce5136c628c..6e283d4c967 100644 --- a/dockerfile/anaconda-rpm/Dockerfile +++ b/dockerfile/anaconda-rpm/Dockerfile @@ -16,7 +16,6 @@ FROM ${image} # dependencies. # possible values: # master -# f35-devel # f35-release ARG git_branch ARG copr_repo=@rhinstaller/Anaconda From 3bf748a18000493285d646f45deef8488fcc3a99 Mon Sep 17 00:00:00 2001 From: Vladimir Slavik Date: Tue, 23 Aug 2022 11:05:41 +0200 Subject: [PATCH 2/5] Switch branched Fedora to 37 in CI (#infra) This alone is not enough, the f37-branch needs work as well. --- .github/workflows/container-autoupdate-fedora.yml | 8 ++++---- .github/workflows/try-release-daily.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container-autoupdate-fedora.yml b/.github/workflows/container-autoupdate-fedora.yml index 59a1c8b98df..f3df291b5c6 100644 --- a/.github/workflows/container-autoupdate-fedora.yml +++ b/.github/workflows/container-autoupdate-fedora.yml @@ -17,10 +17,10 @@ jobs: container-tag: master branch: master - f36-release: - if: false + f37-release: + if: true uses: ./.github/workflows/container-rebuild-action.yml secrets: inherit with: - container-tag: f36-release - branch: f36-release + container-tag: f37-release + branch: f37-release diff --git a/.github/workflows/try-release-daily.yml b/.github/workflows/try-release-daily.yml index 93ca9ea080e..c6cbdcac241 100644 --- a/.github/workflows/try-release-daily.yml +++ b/.github/workflows/try-release-daily.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - branch: ['master', 'rhel-8', 'rhel-9'] # depending on season, also: fNN-release + branch: ['master', 'rhel-8', 'rhel-9', 'f37-release'] # depending on season, also: fNN-release steps: - name: Check out repo From 158aef21b393e5d72590ad7cb328c6864e88acfa Mon Sep 17 00:00:00 2001 From: Vladimir Slavik Date: Tue, 23 Aug 2022 16:03:20 +0200 Subject: [PATCH 3/5] Infra templating script improvements - Remove files if generated empty - Print input and output file names while processing - Don't print Makefile commands for the reload-infra target - Unify environment creation, one place to change settings - Strip block whitespace --- Makefile.am | 2 +- scripts/jinja-render | 43 +++++++++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/Makefile.am b/Makefile.am index d189f9d65d2..6fd1a93b787 100644 --- a/Makefile.am +++ b/Makefile.am @@ -415,6 +415,6 @@ coverage-report: @cat tests/coverage-report.log reload-infra: - for template in $$(find . -type f -name '*.j2'); do \ + @for template in $$(find . -type f -name '*.j2'); do \ $(TEMPLATE_RENDERER) $${template} $${template%.j2}; \ done diff --git a/scripts/jinja-render b/scripts/jinja-render index 025807a92f0..67309f6dae9 100755 --- a/scripts/jinja-render +++ b/scripts/jinja-render @@ -51,17 +51,24 @@ def parse_args(): config = parser.parse_args() return vars(config) -def render_file(config, variables): - """Renders a file as a Jinja2 template and populates it with variables.""" +def make_env(loader): + """Create a jinja environment with a given loader.""" env = Environment( - loader=FileSystemLoader(os.path.dirname(config["input"])), + loader=loader, keep_trailing_newline=True, - undefined = StrictUndefined, + undefined=StrictUndefined, block_start_string=BLOCK_START_STRING, block_end_string=BLOCK_END_STRING, variable_start_string=VARIABLE_START_STRING, - variable_end_string=VARIABLE_END_STRING + variable_end_string=VARIABLE_END_STRING, + trim_blocks=True, + lstrip_blocks=True, ) + return env + +def render_file(config, variables): + """Renders a file as a Jinja2 template and populates it with variables.""" + env = make_env(FileSystemLoader(os.path.dirname(config["input"]))) content = env.get_template(os.path.basename(config["input"])) content = content.render(variables) @@ -69,16 +76,7 @@ def render_file(config, variables): def render_string(string, variables): """Renders a given string as a Jinja2 template and populates it with variables.""" - # Create jinja environment. - env = Environment( - loader=BaseLoader, - keep_trailing_newline=True, - undefined = StrictUndefined, - block_start_string=BLOCK_START_STRING, - block_end_string=BLOCK_END_STRING, - variable_start_string=VARIABLE_START_STRING, - variable_end_string=VARIABLE_END_STRING - ) + env = make_env(BaseLoader) template = env.from_string(string) content = template.render(variables) return content @@ -101,9 +99,18 @@ def main(): variables.update({"template_file_name": os.path.basename(config["input"])}) - rendered_disclaimer = render_string(DISCLAIMER, variables) - content = rendered_disclaimer + render_file(config, variables) - write_file(content, config["output"]) + print("Loading template {}".format(config["input"])) + rendered_file = render_file(config, variables) + if rendered_file.strip(): + rendered_disclaimer = render_string(DISCLAIMER, variables) + content = rendered_disclaimer + rendered_file + print("Saving rendered result to {}".format(config["output"])) + write_file(content, config["output"]) + else: + # delete files that end up completely empty + print("Empty result, removing {}".format(config["output"])) + if os.path.exists(config["output"]): + os.remove(config["output"]) if __name__ == "__main__": main() From 1b871d37bcb1721c7aa4e8491ef13d909c1a7bfd Mon Sep 17 00:00:00 2001 From: Vladimir Slavik Date: Tue, 23 Aug 2022 11:34:19 +0200 Subject: [PATCH 4/5] Templatize GitHub actions (#infra) Add a readme, too. --- .branch-variables.yml | 14 +- .github/workflows/README.md | 24 ++ .github/workflows/build-boot-iso.yml | 7 + .github/workflows/build-boot-iso.yml.j2 | 153 +++++++++ .../workflows/container-autoupdate-eln.yml | 7 + .../workflows/container-autoupdate-eln.yml.j2 | 22 ++ .../workflows/container-autoupdate-fedora.yml | 8 +- .../container-autoupdate-fedora.yml.j2 | 29 ++ .../workflows/container-rebuild-action.yml | 7 + .../workflows/container-rebuild-action.yml.j2 | 100 ++++++ .github/workflows/daily-rhel-copr.yml | 7 + .github/workflows/daily-rhel-copr.yml.j2 | 37 +++ .github/workflows/kickstart-tests.yml | 7 + .github/workflows/kickstart-tests.yml.j2 | 312 ++++++++++++++++++ .github/workflows/push-tests.yml | 7 + .github/workflows/push-tests.yml.j2 | 38 +++ .github/workflows/stale-pull-requests.yml | 7 + .github/workflows/stale-pull-requests.yml.j2 | 26 ++ .github/workflows/tag-release.yml | 7 + .github/workflows/tag-release.yml.j2 | 105 ++++++ .github/workflows/tests-contributors.yml | 7 + .github/workflows/tests-contributors.yml.j2 | 166 ++++++++++ .github/workflows/tests.yml | 27 +- .github/workflows/tests.yml.j2 | 177 ++++++++++ .github/workflows/try-release-daily.yml | 9 +- .github/workflows/try-release-daily.yml.j2 | 76 +++++ 26 files changed, 1367 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/build-boot-iso.yml.j2 create mode 100644 .github/workflows/container-autoupdate-eln.yml.j2 create mode 100644 .github/workflows/container-autoupdate-fedora.yml.j2 create mode 100644 .github/workflows/container-rebuild-action.yml.j2 create mode 100644 .github/workflows/daily-rhel-copr.yml.j2 create mode 100644 .github/workflows/kickstart-tests.yml.j2 create mode 100644 .github/workflows/push-tests.yml.j2 create mode 100644 .github/workflows/stale-pull-requests.yml.j2 create mode 100644 .github/workflows/tag-release.yml.j2 create mode 100644 .github/workflows/tests-contributors.yml.j2 create mode 100644 .github/workflows/tests.yml.j2 create mode 100644 .github/workflows/try-release-daily.yml.j2 diff --git a/.branch-variables.yml b/.branch-variables.yml index b467819dafe..fc2abafa4f2 100644 --- a/.branch-variables.yml +++ b/.branch-variables.yml @@ -1 +1,13 @@ -branch_type: "test" +# Settings for infrastructure autogeneration +# +# See also: +# scripts/jinja-render +# make reload-infra +# +# Follow "type hints" below or be sad. The choice is yours. + +distro_name: "fedora" # "fedora" or "rhel" +distro_release: "rawhide" # "rawhide" or a number without quotation marks + +# The following only applies for rawhide. +branched_fedora_version: 37 # number without quotation marks, or nothing if CI should not run for branched Fedora diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 00000000000..253a00fbfc6 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,24 @@ +Templated workflows +------------------- + +Almost all workflows are now built from jinja2 templates. Make sure you are editing the `.j2` +template, not the rendered `.yml` file. To rebuild the workflow files from templates, run +`make -f Makefile.am reload-infra` at any time. The rebuild depends only on the +`.branch-variables.yml` file in the repo root. + +Most of the workflows are triggered by cron or comment events, so they belong only on the default +branch which is `master`. These workflows are removed by templates on other branches. If the first +line is `{% if distro_release == "rawhide" %}` then the workflow is of such kind. + +When editing a template, the following roughly describes what to expect: + +- Any values available for the templates come from `.branch-variables.yml` in the repo root. + +- Inline variables `{$ ... $}` are replaced by values. + +- Blocks `{% ... %}` let you use conditions to select which block will be present in the output. + If you don't put anything else on a line except for the block itself, the line will completely + disappear from the output. Prefer that and avoid using blocks inline. + +- Whitespace handling is a complicated affair. YAML wants it precise, while Jinja is messy. + If you stick to the two methods above, everything stays mostly deterministic. diff --git a/.github/workflows/build-boot-iso.yml b/.github/workflows/build-boot-iso.yml index 3b99618e318..cf00c08b3cf 100644 --- a/.github/workflows/build-boot-iso.yml +++ b/.github/workflows/build-boot-iso.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: build-boot-iso.yml.j2 + # Build a boot.iso from a PR triggered by a "/boot-iso" comment or manually. name: Build boot.iso on: diff --git a/.github/workflows/build-boot-iso.yml.j2 b/.github/workflows/build-boot-iso.yml.j2 new file mode 100644 index 00000000000..5f9ba38bfb5 --- /dev/null +++ b/.github/workflows/build-boot-iso.yml.j2 @@ -0,0 +1,153 @@ +{% if distro_release == "rawhide" %} +# Build a boot.iso from a PR triggered by a "/boot-iso" comment or manually. +name: Build boot.iso +on: + issue_comment: + types: [created] + # be able to start this action manually from a actions tab when needed + workflow_dispatch: + +permissions: + contents: read + statuses: write + +jobs: + pr-info: + if: github.event_name == 'workflow_dispatch' || startsWith(github.event.comment.body, '/boot-iso') + runs-on: ubuntu-latest + steps: + - name: Query comment author repository permissions + if: github.event_name != 'workflow_dispatch' + uses: octokit/request-action@v2.x + id: user_permission + with: + route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # restrict this workflow to users with admin or write permission for the repository + # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user + # store output if user is allowed in allowed_user job output so it has to be checked in downstream job + - name: Check if user does have correct permissions + if: github.event_name != 'workflow_dispatch' && contains('admin write', fromJson(steps.user_permission.outputs.data).permission) + id: check_user_perm + run: | + echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'" + echo "::set-output name=allowed_user::true" + + - name: Get information for pull request + if: github.event_name != 'workflow_dispatch' + uses: octokit/request-action@v2.x + id: pr_api + with: + route: GET /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set outputs + id: set_outputs + run: | + set -eux + + if [ ${{ github.event_name }} == 'workflow_dispatch' ]; then + echo "::set-output name=allowed_user::true" + echo "::set-output name=sha::$GITHUB_SHA" + else + echo "::set-output name=allowed_user::${{ steps.check_user_perm.outcome == 'success' && steps.check_user_perm.outputs.allowed_user }}" + echo "::set-output name=sha::${{ steps.pr_api.outcome == 'success' && fromJson(steps.pr_api.outputs.data).head.sha }}" + fi + + outputs: + allowed_user: ${{ steps.set_outputs.outputs.allowed_user }} + sha: ${{ steps.set_outputs.outputs.sha }} + + run: + needs: pr-info + # only do this for Fedora for now; once we have RHEL 8/9 boot.iso builds working, also support these + if: needs.pr-info.outputs.allowed_user == 'true' + runs-on: [self-hosted, kstest] + timeout-minutes: 300 + env: + STATUS_NAME: boot-iso + CONTAINER_TAG: 'lorax' + ISO_BUILD_CONTAINER_NAME: 'quay.io/rhinstaller/anaconda-iso-creator' + steps: + # we post statuses manually as this does not run from a pull_request event + # https://developer.github.com/v3/repos/statuses/#create-a-status + - name: Create in-progress status + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}' + state: pending + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Clone repository + uses: actions/checkout@v3 + with: + ref: ${{ needs.pr-info.outputs.sha }} + fetch-depth: 0 + + - name: Build anaconda-iso-creator container image + run: | + # set static tag to avoid complications when looking what tag is used + sudo make -f ./Makefile.am anaconda-iso-creator-build CI_TAG=$CONTAINER_TAG + + - name: Build anaconda-rpm container (for RPM build) + run: | + # set static tag to avoid complications when looking what tag is used + make -f ./Makefile.am anaconda-rpm-build CI_TAG=$CONTAINER_TAG + + - name: Build Anaconda RPM files + run: | + # output of the build will be stored in ./result/build/01-rpm-build/*.rpm + make -f ./Makefile.am container-rpms-scratch CI_TAG=$CONTAINER_TAG + mkdir -p ./anaconda_rpms/ + cp -av ./result/build/01-rpm-build/*.rpm ./anaconda_rpms/ + + - name: Prepare environment for lorax run + run: | + mkdir -p images + # We have to pre-create loop devices because they are not namespaced in kernel so + # podman can't access newly created ones. That caused failures of tests when runners + # were rebooted. + sudo mknod -m 0660 /dev/loop0 b 7 0 2> /dev/null || true + sudo mknod -m 0660 /dev/loop1 b 7 1 2> /dev/null || true + + - name: Build the boot.iso + run: | + # /var/tmp tmpfs speeds up lorax and avoids https://bugzilla.redhat.com/show_bug.cgi?id=1906364 + sudo podman run -i --rm --privileged \ + --tmpfs /var/tmp:rw,mode=1777 \ + -v `pwd`/anaconda_rpms:/anaconda-rpms:ro \ + -v `pwd`/images:/images:z \ + $ISO_BUILD_CONTAINER_NAME:$CONTAINER_TAG + + - name: Collect logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: 'logs' + path: | + images/*.log + + - name: Upload image artifacts + uses: actions/upload-artifact@v3 + with: + name: images + path: | + images/boot.iso + + - name: Set result status + if: always() + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}' + state: ${{ job.status }} + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +{% endif %} diff --git a/.github/workflows/container-autoupdate-eln.yml b/.github/workflows/container-autoupdate-eln.yml index 52c0bab2549..1e816737c34 100644 --- a/.github/workflows/container-autoupdate-eln.yml +++ b/.github/workflows/container-autoupdate-eln.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: container-autoupdate-eln.yml.j2 + # a smaller sibling of the Fedora refresh; split to provide cleaner statuses name: Refresh ELN container images on: diff --git a/.github/workflows/container-autoupdate-eln.yml.j2 b/.github/workflows/container-autoupdate-eln.yml.j2 new file mode 100644 index 00000000000..efbe223fa34 --- /dev/null +++ b/.github/workflows/container-autoupdate-eln.yml.j2 @@ -0,0 +1,22 @@ +{% if distro_release == "rawhide" %} +# a smaller sibling of the Fedora refresh; split to provide cleaner statuses +name: Refresh ELN container images +on: + schedule: + - cron: 0 0 * * * + # be able to start this action manually from a actions tab when needed + workflow_dispatch: + +permissions: + contents: read + +jobs: + + eln: + uses: ./.github/workflows/container-rebuild-action.yml + secrets: inherit + with: + container-tag: eln + branch: master + base-container: 'quay.io/fedoraci/fedora:eln-x86_64' +{% endif %} diff --git a/.github/workflows/container-autoupdate-fedora.yml b/.github/workflows/container-autoupdate-fedora.yml index f3df291b5c6..42aff4833ec 100644 --- a/.github/workflows/container-autoupdate-fedora.yml +++ b/.github/workflows/container-autoupdate-fedora.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: container-autoupdate-fedora.yml.j2 + name: Refresh Fedora container images on: schedule: @@ -18,7 +25,6 @@ jobs: branch: master f37-release: - if: true uses: ./.github/workflows/container-rebuild-action.yml secrets: inherit with: diff --git a/.github/workflows/container-autoupdate-fedora.yml.j2 b/.github/workflows/container-autoupdate-fedora.yml.j2 new file mode 100644 index 00000000000..69978dd84eb --- /dev/null +++ b/.github/workflows/container-autoupdate-fedora.yml.j2 @@ -0,0 +1,29 @@ +{% if distro_release == "rawhide" %} +name: Refresh Fedora container images +on: + schedule: + - cron: 0 0 * * * + # be able to start this action manually from a actions tab when needed + workflow_dispatch: + +permissions: + contents: read + +jobs: + + master: + uses: ./.github/workflows/container-rebuild-action.yml + secrets: inherit + with: + container-tag: master + branch: master + + {% if branched_fedora_version is defined and branched_fedora_version %} + f{$ branched_fedora_version $}-release: + uses: ./.github/workflows/container-rebuild-action.yml + secrets: inherit + with: + container-tag: f{$ branched_fedora_version $}-release + branch: f{$ branched_fedora_version $}-release + {% endif %} +{% endif %} diff --git a/.github/workflows/container-rebuild-action.yml b/.github/workflows/container-rebuild-action.yml index c2cff4c84e5..ff2097f237a 100644 --- a/.github/workflows/container-rebuild-action.yml +++ b/.github/workflows/container-rebuild-action.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: container-rebuild-action.yml.j2 + name: Rebuild container images # Rebuilds both ci and rpm container images for a given "target". Currently known targets: # - master diff --git a/.github/workflows/container-rebuild-action.yml.j2 b/.github/workflows/container-rebuild-action.yml.j2 new file mode 100644 index 00000000000..52422df278d --- /dev/null +++ b/.github/workflows/container-rebuild-action.yml.j2 @@ -0,0 +1,100 @@ +{% if distro_release == "rawhide" %} +name: Rebuild container images +# Rebuilds both ci and rpm container images for a given "target". Currently known targets: +# - master +# - eln +# - fNN-release +# +# Image is: +# - built from the repo at ref , +# - based on the right container according to branch settings, or optionally on if set, +# - tagged as quay.io/rhinstaller/anaconda-{ci|rpm}:. +# +# See also inputs below. + +# Reusable workflow, does not run on its own. Typically, use in an action this way: +# jobs: +# foo: +# uses: ./.github/workflows/container-rebuild-action.yml +# secrets: inherit +# with: +# container-tag: foo +# branch: foo + +on: + workflow_call: + inputs: + container-tag: + required: true + type: string + branch: + required: true + type: string + base-container: + required: false + type: string + # secrets consumed: QUAY_USERNAME, QUAY_PASSWORD + # these are provided by using the quay.io environment below + +permissions: + contents: read + +jobs: + refresh-container: + name: Refresh anaconda container + runs-on: ubuntu-20.04 + environment: quay.io + strategy: + fail-fast: false + matrix: + container-type: ['ci', 'rpm'] + env: + CI_TAG: '${{ inputs.container-tag }}' + timeout-minutes: 60 + steps: + - name: Checkout anaconda repository + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + + - name: Build anaconda-${{ matrix.container-type }} container + run: | + BASE_CONTAINER=${{ inputs.base-container }} + make -f Makefile.am anaconda-${{ matrix.container-type }}-build ${BASE_CONTAINER:+BASE_CONTAINER=}${BASE_CONTAINER:-} + + - name: Run tests in anaconda-ci container + if: matrix.container-type == 'ci' + run: | + # put the log in the output, where it's easy to read and link to + make -f Makefile.am container-ci || { cat test-logs/test-suite.log; exit 1; } + + - name: Run tests in anaconda-rpm container + if: matrix.container-type == 'rpm' + run: | + # put the log in the output, where it's easy to read and link to + make -f Makefile.am container-rpm-test || { cat test-logs/test-suite.log; exit 1; } + + - name: Upload test and coverage logs from local testing + if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.container-tag }}-${{ matrix.container-type }}-logs + path: | + test-logs/test-suite.log + test-logs/unit_tests.log + test-logs/pylint/runpylint*.log + test-logs/coverage-*.log + + - name: Login to container registry + run: podman login -u ${{ secrets.QUAY_USERNAME }} -p ${{ secrets.QUAY_PASSWORD }} quay.io + + # we can hardcode the path to the image here because this will be executed only for master image + - name: Add latest tag for master container + if: ${{ inputs.container-tag == 'master' }} + run: | + podman tag quay.io/rhinstaller/anaconda-${{ matrix.container-type }}:master quay.io/rhinstaller/anaconda-${{ matrix.container-type }}:latest + CI_TAG=latest make -f Makefile.am anaconda-${{ matrix.container-type }}-push + + - name: Push container to registry + run: make -f Makefile.am anaconda-${{ matrix.container-type }}-push +{% endif %} diff --git a/.github/workflows/daily-rhel-copr.yml b/.github/workflows/daily-rhel-copr.yml index 58348f95a95..cf421e3e122 100644 --- a/.github/workflows/daily-rhel-copr.yml +++ b/.github/workflows/daily-rhel-copr.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: daily-rhel-copr.yml.j2 + # Do a build of the rhel-8 branch to RH internal COPR name: Build current anaconda rhel-8 branch in RHEL COPR on: diff --git a/.github/workflows/daily-rhel-copr.yml.j2 b/.github/workflows/daily-rhel-copr.yml.j2 new file mode 100644 index 00000000000..a390d31644c --- /dev/null +++ b/.github/workflows/daily-rhel-copr.yml.j2 @@ -0,0 +1,37 @@ +{% if distro_release == "rawhide" %} +# Do a build of the rhel-8 branch to RH internal COPR +name: Build current anaconda rhel-8 branch in RHEL COPR +on: + schedule: + - cron: 0 2 * * * + # be able to start this action manually from a actions tab when needed + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: copr-builder + runs-on: [self-hosted, kstest] + env: + CI_CONTAINER: rhinstaller/anaconda-rhel-copr:rhel-8 + steps: + - name: Clone anaconda repository + uses: actions/checkout@v3 + with: + ref: master + fetch-depth: 0 + path: anaconda + + - name: Build the build container + run: | + sudo podman build --no-cache -t ${{ env.CI_CONTAINER }} anaconda/dockerfile/anaconda-rhel-copr + + - name: Run copr-builder + run: | + sudo podman run -i --rm --privileged -v /home/github/copr-rhel-token:/copr-builder/copr-rhel-token:ro,z ${{ env.CI_CONTAINER }} <" comment from an organization member. name: kickstart-tests on: diff --git a/.github/workflows/kickstart-tests.yml.j2 b/.github/workflows/kickstart-tests.yml.j2 new file mode 100644 index 00000000000..aba6676d7c0 --- /dev/null +++ b/.github/workflows/kickstart-tests.yml.j2 @@ -0,0 +1,312 @@ +{% if distro_release == "rawhide" %} +# Run kickstart tests in a PR triggered by a "/kickstart-test " comment from an organization member. +name: kickstart-tests +on: + issue_comment: + types: [created] + +permissions: + contents: read + statuses: write + +jobs: + pr-info: + if: startsWith(github.event.comment.body, '/kickstart-test') + runs-on: ubuntu-latest + steps: + - name: Query comment author repository permissions + uses: octokit/request-action@v2.x + id: user_permission + with: + route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # restrict running of tests to users with admin or write permission for the repository + # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user + # store output if user is allowed in allowed_user job output so it has to be checked in downstream job + - name: Check if user does have correct permissions + if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission) + id: check_user_perm + run: | + echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'" + echo "::set-output name=allowed_user::true" + + - name: Get information for pull request + uses: octokit/request-action@v2.x + id: pr_api + with: + route: GET /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Parse launch arguments + id: parse_launch_args + # Do not use comment body directly in the shell command to avoid possible code injection. + env: + BODY: ${{ github.event.comment.body }} + run: | + # extract first line and cut out the "/kickstart-tests" first word + LAUNCH_ARGS=$(echo "$BODY" | sed -n '1 s/^[^ ]* *//p') + echo "launch arguments are: $LAUNCH_ARGS" + echo "::set-output name=launch_args::${LAUNCH_ARGS}" + + - name: Set KS test arguments + id: ks_test_args + run: | + set -eux + TARGET_BRANCH="${{ fromJson(steps.pr_api.outputs.data).base.ref }}" + + if [ "$TARGET_BRANCH" == "master" ]; then + echo "::set-output name=skip_tests::skip-on-fedora" + echo "::set-output name=platform::fedora_rawhide" + elif echo "$TARGET_BRANCH" | grep -qE "f[[:digit:]]+-release$"; then + echo "::set-output name=skip_tests::skip-on-fedora" + echo "::set-output name=platform::fedora_rawhide" + elif echo "$TARGET_BRANCH" | grep -qE "rhel-8(\.[[:digit:]]+)?$"; then + echo "::set-output name=skip_tests::skip-on-rhel,skip-on-rhel-8" + echo "::set-output name=platform::rhel8" + elif echo "$TARGET_BRANCH" | grep -qE "rhel-9(\.[[:digit:]]+)?$"; then + echo "::set-output name=skip_tests::skip-on-rhel,skip-on-rhel-9" + echo "::set-output name=platform::rhel9" + else + echo "Branch $TARGET_BRANCH is not supported by kickstart tests yet!" + exit 1 + fi + + outputs: + allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }} + base_ref: ${{ fromJson(steps.pr_api.outputs.data).base.ref }} + sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }} + launch_args: ${{ steps.parse_launch_args.outputs.launch_args }} + skip_tests: ${{ steps.ks_test_args.outputs.skip_tests }} + platform: ${{ steps.ks_test_args.outputs.platform }} + + run: + needs: pr-info + # only do this for Fedora for now; once we have RHEL 8/9 boot.iso builds working, also support these + if: needs.pr-info.outputs.allowed_user == 'true' && needs.pr-info.outputs.launch_args != '' + runs-on: [self-hosted, kstest] + timeout-minutes: 300 + env: + STATUS_NAME: kickstart-test + TARGET_BRANCH: ${{ needs.pr-info.outputs.base_ref }} + CONTAINER_TAG: 'lorax' + ISO_BUILD_CONTAINER_NAME: 'quay.io/rhinstaller/anaconda-iso-creator' + RPM_BUILD_CONTAINER_NAME: 'quay.io/rhinstaller/anaconda-rpm' + SKIP_KS_TESTS: ${{ needs.pr-info.outputs.skip_tests }} + TEST_JOBS: 16 + steps: + # we post statuses manually as this does not run from a pull_request event + # https://developer.github.com/v3/repos/statuses/#create-a-status + - name: Create in-progress status + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}' + state: pending + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # self-hosted runners don't do this automatically; also useful to keep stuff around for debugging + # need to run sudo as the launch script and the container create root/other user owned files + - name: Clean up previous run + run: | + sudo podman ps -q --all --filter='ancestor=kstest-runner' | xargs -tr sudo podman rm -f + sudo podman volume rm --all || true + sudo rm -rf * .git + + - name: Clone repository + uses: actions/checkout@v3 + with: + ref: ${{ needs.pr-info.outputs.sha }} + fetch-depth: 0 + path: anaconda + + - name: Rebase to current ${{ env.TARGET_BRANCH }} + working-directory: ./anaconda + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git log --oneline -1 origin/${{ env.TARGET_BRANCH }} + git rebase origin/${{ env.TARGET_BRANCH }} + + - name: Check out kickstart-tests + uses: actions/checkout@v3 + with: + repository: rhinstaller/kickstart-tests + path: kickstart-tests + + - name: Generate test cases + working-directory: ./kickstart-tests + run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases + + - name: Clone Permian repository + uses: actions/checkout@v3 + with: + repository: rhinstaller/permian + path: permian + ref: main + + - name: Clone tplib repository + uses: actions/checkout@v3 + with: + repository: rhinstaller/tplib + path: tplib + + - name: Ensure http proxy is running + run: sudo kickstart-tests/containers/squid.sh start + + # This is really fast, but does not catch file removals or dracut changes + # maybe this becomes opt-in via a magic comment for efficiency reasons? + # if you use this, add `--updates ../updates.img` to the launch command line below + #- name: Build updates.img + # run: | + # scripts/makeupdates + # gzip -cd updates.img | cpio -tv + + - name: Update container images used here + run: | + sudo podman pull quay.io/rhinstaller/kstest-runner:latest + + - name: Build anaconda-iso-creator container image + working-directory: ./anaconda + run: | + # set static tag to avoid complications when looking what tag is used + sudo make -f ./Makefile.am anaconda-iso-creator-build CI_TAG=$CONTAINER_TAG + + - name: Build anaconda-rpm container (for RPM build) + working-directory: ./anaconda + run: | + # set static tag to avoid complications when looking what tag is used + make -f ./Makefile.am anaconda-rpm-build CI_TAG=$CONTAINER_TAG + + - name: Build Anaconda RPM files + working-directory: ./anaconda + run: | + # output of the build will be stored in ./result/build/01-rpm-build/*.rpm + make -f ./Makefile.am container-rpms-scratch CI_TAG=$CONTAINER_TAG + mkdir -p ${{ github.workspace }}/kickstart-tests/data/additional_repo/ + cp -av ./result/build/01-rpm-build/*.rpm ${{ github.workspace }}/kickstart-tests/data/additional_repo/ + + - name: Prepare environment for lorax run + run: | + mkdir images + # We have to pre-create loop devices because they are not namespaced in kernel so + # podman can't access newly created ones. That caused failures of tests when runners + # were rebooted. + sudo mknod -m 0660 /dev/loop0 b 7 0 2> /dev/null || true + sudo mknod -m 0660 /dev/loop1 b 7 1 2> /dev/null || true + + - name: Build boot.iso + run: | + # /var/tmp tmpfs speeds up lorax and avoids https://bugzilla.redhat.com/show_bug.cgi?id=1906364 + sudo podman run -i --rm --privileged \ + --tmpfs /var/tmp:rw,mode=1777 \ + -v ${{ github.workspace }}/kickstart-tests/data/additional_repo:/anaconda-rpms:ro \ + -v ${{ github.workspace }}/images:/images:z \ + $ISO_BUILD_CONTAINER_NAME:$CONTAINER_TAG + + - name: Clean up after lorax + if: always() + run: | + sudo losetup -d /dev/loop0 2> /dev/null || true + sudo losetup -d /dev/loop1 2> /dev/null || true + + # remove container images together with the container + sudo podman rmi -f $ISO_BUILD_CONTAINER_NAME:$CONTAINER_TAG || true + sudo podman rmi -f $RPM_BUILD_CONTAINER_NAME:$CONTAINER_TAG || true + + - name: Generate query arguments for "${{ needs.pr-info.outputs.launch_args }}" + id: generate_query + working-directory: ./kickstart-tests + run: | + set -eux + PERMIAN_QUERY=$(scripts/generate-permian-query.py \ + --skip-testtypes ${{ needs.pr-info.outputs.skip_tests }} \ + ${{ needs.pr-info.outputs.launch_args }} ) + if [ $? == 0 ]; then + echo "::set-output name=query::$PERMIAN_QUERY" + else + echo "Parsing of the request arguments failed" + exit 1 + fi + + - name: Create Permian settings file + working-directory: ./permian + run: | + cat < settings.ini + [kickstart_test] + kstest_local_repo=${{ github.workspace }}/kickstart-tests + [library] + directPath=${{ github.workspace }}/kickstart-tests/testlib + EOF + + - name: Run kickstart tests in container + working-directory: ./permian + run: | + sudo --preserve-env=TEST_JOBS \ + PYTHONPATH=${PYTHONPATH:-}:${{ github.workspace }}/tplib \ + ./run_subset --debug-log permian.log \ + --settings settings.ini \ + --override workflows.dry_run=False \ + --testcase-query '${{ steps.generate_query.outputs.query }}' \ + run_event '{ + "type":"everything", + "everything_testplan":{ + "configurations":[{"architecture":"x86_64"}], + "point_person":"rvykydal@redhat.com" + }, + "bootIso":{"x86_64":"file://${{ github.workspace }}/images/boot.iso"}, + "kstestParams":{"platform":"${{ needs.pr-info.outputs.platform }}"} + }' + + # Needed so that other jobs are able to clean the working dir + # FIXME: we should investigate running permian/launch sudo-less + - name: Make artefacts created by sudo cleanable + run: + sudo chown -R github:github . + + - name: Collect anaconda logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: 'logs' + # skip the /anaconda subdirectories, too large + path: | + kickstart-tests/data/logs/kstest.log + kickstart-tests/data/logs/kstest-*/*.log + kickstart-tests/data/additional_repo/*.rpm + + - name: Collect Permian logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: 'logs-permian' + path: | + permian/permian.log + + + # Permian hides the exit code of launcher + - name: Pass the launch script exit code + working-directory: ./permian + run: | + rc=$( awk '/Runner return code: /{ print $4 }' permian.log) + if [ -n "$rc" ]; then + exit $rc + else + exit 111 + fi + + - name: Set result status + if: always() + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}' + state: ${{ job.status }} + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +{% endif %} diff --git a/.github/workflows/push-tests.yml b/.github/workflows/push-tests.yml index e9008d39530..d5ec45aaa26 100644 --- a/.github/workflows/push-tests.yml +++ b/.github/workflows/push-tests.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: push-tests.yml.j2 + name: Run tests on push on: push: diff --git a/.github/workflows/push-tests.yml.j2 b/.github/workflows/push-tests.yml.j2 new file mode 100644 index 00000000000..a5d2884e765 --- /dev/null +++ b/.github/workflows/push-tests.yml.j2 @@ -0,0 +1,38 @@ +{% if distro_release == "rawhide" %} +name: Run tests on push +on: + push: + branches: + - master + - f[0-9]+-release + +permissions: + contents: read + +jobs: + unit-tests: + runs-on: ubuntu-20.04 + timeout-minutes: 30 + steps: + - name: Clone repository + uses: actions/checkout@v3 + + - name: Build anaconda-ci container + run: make -f Makefile.am anaconda-ci-build + + - name: Run tests in anaconda-ci container + run: | + # put the log in the output, where it's easy to read and link to + make -f Makefile.am container-ci || { cat test-logs/test-suite.log; exit 1; } + + - name: Upload test and coverage logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: 'logs' + path: test-logs/* + + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v3 +{% endif %} diff --git a/.github/workflows/stale-pull-requests.yml b/.github/workflows/stale-pull-requests.yml index 0169fb488c9..a815522384a 100644 --- a/.github/workflows/stale-pull-requests.yml +++ b/.github/workflows/stale-pull-requests.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: stale-pull-requests.yml.j2 + name: Close stale pull requests on: schedule: diff --git a/.github/workflows/stale-pull-requests.yml.j2 b/.github/workflows/stale-pull-requests.yml.j2 new file mode 100644 index 00000000000..ebc3e3a345f --- /dev/null +++ b/.github/workflows/stale-pull-requests.yml.j2 @@ -0,0 +1,26 @@ +{% if distro_release == "rawhide" %} +name: Close stale pull requests +on: + schedule: + - cron: 30 1 * * * + +permissions: + contents: read + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - name: Handle stale pull requests + uses: actions/stale@v5 + with: + days-before-stale: 60 + days-before-close: 30 + stale-pr-label: stale + stale-pr-message: | + This PR is stale because it has been open 60 days with no activity. + Remove stale label or comment or this will be closed in 30 days. + close-pr-message: | + This PR was closed because it has been stalled for 30 days with no activity. +{% endif %} diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index e8d9898d98e..2ce4e4c004f 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: tag-release.yml.j2 + name: Release from tags # Create a GitHub release when a tag is pushed. diff --git a/.github/workflows/tag-release.yml.j2 b/.github/workflows/tag-release.yml.j2 new file mode 100644 index 00000000000..b80c23643d7 --- /dev/null +++ b/.github/workflows/tag-release.yml.j2 @@ -0,0 +1,105 @@ +{% if distro_release == "rawhide" %} +name: Release from tags +# Create a GitHub release when a tag is pushed. + +# Input can ingest any string, but it will not work with arbitrary hashes. The code expects to get +# a tag that sits on a release commit, so that commit is automatically skipped. Similarly, it +# expects that the tag name conforms to a particular scheme. + +on: + push: + tags: + # this is a glob, not a regexp + - 'anaconda-*' + + workflow_dispatch: + inputs: + tag-ref: + # it might be possible to release from arbitrary ref, but only tags work + description: Name of the tag to release from, such as "anaconda-33.12-1". Branch names or arbitrary commit hashes are not supported. Make sure to trim whitespace when copying. + type: string + required: true + +permissions: + contents: write + +jobs: + release-from-tag: + runs-on: ubuntu-latest + steps: + - name: Resolve tag reference + id: get_ref + # Get reference of the tag that is to be released. + # The tag can be in two places, depending on how the workflow was started. + run: | + if [ -n "${{ github.event.inputs.tag-ref }}" ] ; then + echo "Using tag from manual input: ${{ github.event.inputs.tag-ref }}" + echo "::set-output name=ref::${{ github.event.inputs.tag-ref }}" + else + echo "Using tag from automatic input: ${{ github.ref }}" + echo "::set-output name=ref::${{ github.ref }}" + fi + + - name: Extract version from the tag + id: get_version + # Get the actual version number from the tag, eg. anaconda-33.12-1 -> 33.12 + # If the version does not fit the regexp, we'll just fail. TODO maybe abort instead? + run: | + echo "Checking if ref ${{ steps.get_ref.outputs.ref }} is a valid release tag." + VER=$(echo ${{ steps.get_ref.outputs.ref }} | perl -ne '/^(?:refs\/tags\/)?anaconda-([0-9]+(?:\.[0-9]+){1,3})-1$/ && print "$1\n";') + if [ -z "$VER" ] ; then + echo "Tag ref ${{ steps.get_ref.outputs.ref }} is not a valid release tag." + exit 1 + else + echo "Tag ref ${{ steps.get_ref.outputs.ref }} detected as release version $VER." + echo "::set-output name=version::$VER" + fi + + - name: Check out repo + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + + - name: Build anaconda container (to make the release) + run: | + make -f Makefile.am anaconda-release-build + + - name: Run the build in the container + run: | + mkdir /tmp/results + make -f Makefile.am container-release + + # copy out stuff + cp anaconda-*.tar.bz2 /tmp/results + + - name: Create the release + id: create_release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_NAME: ${{ steps.get_version.outputs.version }} + run: | + # find version tags + CURRENT_VERSION="anaconda-$RELEASE_NAME-1" + PREVIOUS_VERSION="$(git describe --tags --abbrev=0 --exclude="$CURRENT_VERSION")" + + # print commit details; skip the last = release commit + git log \ + --no-merges \ + --pretty=format:"- %s (%aL)" \ + "$PREVIOUS_VERSION..$CURRENT_VERSION~1" \ + | grep -v "#infra" \ + | grep -v "build(deps-dev)" \ + | tee release.txt + + # drop line ending + truncate -s -1 release.txt + + # create release with the release notes + gh release create \ + ${{ steps.get_ref.outputs.ref }} \ + --draft \ + --title "Anaconda $RELEASE_NAME" \ + --notes-file release.txt \ + /tmp/results/* +{% endif %} diff --git a/.github/workflows/tests-contributors.yml b/.github/workflows/tests-contributors.yml index 17164cbf1f9..f01c683a5af 100644 --- a/.github/workflows/tests-contributors.yml +++ b/.github/workflows/tests-contributors.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: tests-contributors.yml.j2 + # Run rhel-8 unit tests in a PR triggered by a "/test [...]" comment from an organization member. # This avoids running untrusted and unreviewed code on self-hosted runners and with R/W permission and ability to read secrets. # See https://securitylab.github.com/research/github-actions-preventing-pwn-requests diff --git a/.github/workflows/tests-contributors.yml.j2 b/.github/workflows/tests-contributors.yml.j2 new file mode 100644 index 00000000000..af472b84dd4 --- /dev/null +++ b/.github/workflows/tests-contributors.yml.j2 @@ -0,0 +1,166 @@ +{% if distro_release == "rawhide" %} +# Run rhel-8 unit tests in a PR triggered by a "/test [...]" comment from an organization member. +# This avoids running untrusted and unreviewed code on self-hosted runners and with R/W permission and ability to read secrets. +# See https://securitylab.github.com/research/github-actions-preventing-pwn-requests +name: Run validation tests for external contributors +on: + issue_comment: + types: [created] + +permissions: + contents: read + statuses: write + +jobs: + pr-info: + if: startsWith(github.event.comment.body, '/test') + runs-on: ubuntu-latest + steps: + - name: Query author repository permissions + uses: octokit/request-action@v2.x + id: user_permission + with: + route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # restrict running of tests to users with admin or write permission for the repository + # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user + # store output if user is allowed in allowed_user job output so it has to be checked in downstream job + - name: Check if user does have correct permissions + if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission) + id: check_user_perm + run: | + echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'" + echo "::set-output name=allowed_user::true" + + - name: Get information for pull request + uses: octokit/request-action@v2.x + id: pr_api + with: + route: GET /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + outputs: + allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }} + base_ref: ${{ fromJson(steps.pr_api.outputs.data).base.ref }} + sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }} + + unit-tests-rhel-8: + needs: pr-info + if: needs.pr-info.outputs.base_ref == 'rhel-8' && needs.pr-info.outputs.allowed_user == 'true' + runs-on: [self-hosted, kstest] + timeout-minutes: 30 + env: + TARGET_BRANCH_NAME: origin/rhel-8 + STATUS_NAME: unit-tests + steps: + # we post statuses manually as this does not run from a pull_request event + # https://developer.github.com/v3/repos/statuses/#create-a-status + - name: Create in-progress status + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: ${{ env.STATUS_NAME }} + state: pending + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Clone repository + uses: actions/checkout@v3 + with: + ref: ${{ needs.pr-info.outputs.sha }} + fetch-depth: 0 + + - name: Rebase to current rhel-8 + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git log --oneline -1 ${{ env.TARGET_BRANCH_NAME }} + git rebase ${{ env.TARGET_BRANCH_NAME }} + + - name: Build CI test container + run: | + make -f Makefile.am anaconda-ci-build + + - name: Run tests in container + run: | + # put the log in the output, where it's easy to read and link to + make -f Makefile.am container-ci || { cat test-logs/test-suite.log; exit 1; } + + - name: Upload test and coverage logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: logs + path: test-logs/* + + - name: Set result status + if: always() + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: ${{ env.STATUS_NAME }} + state: ${{ job.status }} + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + rpm-tests-rhel-8: + needs: pr-info + if: needs.pr-info.outputs.base_ref == 'rhel-8' && needs.pr-info.outputs.allowed_user == 'true' + runs-on: [self-hosted, kstest] + timeout-minutes: 30 + env: + TARGET_BRANCH_NAME: origin/rhel-8 + STATUS_NAME: rpm-tests + steps: + - name: Create in-progress status + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: ${{ env.STATUS_NAME }} + state: pending + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Clone repository + uses: actions/checkout@v3 + with: + ref: ${{ needs.pr-info.outputs.sha }} + fetch-depth: 0 + + - name: Rebase to current ${{ env.TARGET_BRANCH_NAME }} + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git log --oneline -1 ${{ env.TARGET_BRANCH_NAME }} + git rebase ${{ env.TARGET_BRANCH_NAME }} + + - name: Build RPM test container + run: make -f Makefile.am anaconda-rpm-build + + - name: Run RPM tests in container + run: make -f Makefile.am container-rpm-test + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: 'logs-rpm-test' + path: test-logs/* + + - name: Set result status + if: always() + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: ${{ env.STATUS_NAME }} + state: ${{ job.status }} + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +{% endif %} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index be08ec65124..cf27a32de5c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: tests.yml.j2 + name: Run validation tests on: pull_request @@ -11,26 +18,17 @@ jobs: strategy: fail-fast: false matrix: - # * Don't forget to copypaste also the same matrix below in rpm tests! - # * Replace current branch in "release" with empty string. This makes name of the check for - # the current branch always have the same name, so that it can be added to required - # checks. - # * For various branches, comment out what does not apply: - # on -release, only -release - # on master, only master and ELN - #release: [master, eln, ''] release: [''] include: - release: '' target_branch: 'master' ci_tag: 'master' + ## add to release: [...] also eln if re-enabled by uncommenting the below #- release: eln # target_branch: 'master' # ci_tag: 'eln' # build-args: '--build-arg=image=quay.io/fedoraci/fedora:eln-x86_64' - #- release: f35-release - # target_branch: 'f35-release' - # ci_tag: 'f35-release' + env: CI_TAG: '${{ matrix.ci_tag }}' # Always avoid using cache because cache is not correctly invalidated. @@ -88,20 +86,17 @@ jobs: strategy: fail-fast: false matrix: - # For matrix details, see comments for the unit tests above. - #release: [master, eln, ''] release: [''] include: - release: '' target_branch: 'master' ci_tag: 'master' + ## add to release: [...] also eln if re-enabled by uncommenting the below #- release: eln # target_branch: 'master' # ci_tag: 'eln' # build-args: '--build-arg=image=quay.io/fedoraci/fedora:eln-x86_64' - #- release: f35-release - # target_branch: 'f35-release' - # ci_tag: 'f35-release' + env: CI_TAG: '${{ matrix.ci_tag }}' # Always avoid using cache because cache is not correctly invalidated. diff --git a/.github/workflows/tests.yml.j2 b/.github/workflows/tests.yml.j2 new file mode 100644 index 00000000000..35cafe10027 --- /dev/null +++ b/.github/workflows/tests.yml.j2 @@ -0,0 +1,177 @@ +name: Run validation tests +on: pull_request + +permissions: + contents: read + +jobs: + unit-tests: + {% if distro_name == "fedora" %} + runs-on: ubuntu-20.04 + {% elif distro_name == "rhel" %} + runs-on: [self-hosted, kstest] + {% endif %} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + {# + # * The same matrix is below in rpm tests, always change both places! + # * Replace current branch in "release" with empty string. This makes name of the check for + # the current branch always have the same name, so that it can be added to required + # checks on GitHub. + # * This is still a matrix because we might re-enable ELN one day and then we will need + # a matrix here. + #} + {% if distro_name == "fedora" and distro_release == "rawhide" %} + release: [''] + include: + - release: '' + target_branch: 'master' + ci_tag: 'master' + ## add to release: [...] also eln if re-enabled by uncommenting the below + #- release: eln + # target_branch: 'master' + # ci_tag: 'eln' + # build-args: '--build-arg=image=quay.io/fedoraci/fedora:eln-x86_64' + {% elif distro_name == "fedora" and distro_release is number %} + release: [''] + include: + - release: '' + target_branch: 'f{$ distro_release $}-release' + ci_tag: 'f{$ distro_release $}-release' + {% elif distro_name == "rhel" %} + release: [''] + include: + - release: '' + target_branch: 'rhel-{$ distro_release $}' + ci_tag: 'rhel-{$ distro_release $}' + {% endif %} + + env: + CI_TAG: '${{ matrix.ci_tag }}' + # Always avoid using cache because cache is not correctly invalidated. + CONTAINER_BUILD_ARGS: '--no-cache ${{ matrix.build-args }}' + TARGET_BRANCH_NAME: 'origin/${{ matrix.target_branch }}' + + steps: + - name: Clone repository + uses: actions/checkout@v3 + with: + # TODO: Are we able to remove ref, fetch-depth and Rebase task? Seems that the checkout + # without ref is doing the rebase for us. + # otherwise we are testing target branch instead of the PR branch (see pull_request_target trigger) + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Rebase to current ${{ env.TARGET_BRANCH_NAME }} + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git log --oneline -1 ${{ env.TARGET_BRANCH_NAME }} + git rebase ${{ env.TARGET_BRANCH_NAME }} + + {% if distro_name == "fedora" %} + - name: Check if rebuild of the container image is required + id: check-dockerfile-changed + run: | + changes=$(git diff $TARGET_BRANCH_NAME..HEAD -- dockerfile/anaconda-ci/ anaconda.spec.in scripts/testing/install_dependencies.sh) + # print for debugging + echo "$changes" + [ -z "$changes" ] || echo "::set-output name=changed::true" + + # build container if files for dockerfile changed in the PR + - name: Build anaconda-ci container + if: steps.check-dockerfile-changed.outputs.changed + run: make -f Makefile.am anaconda-ci-build + {% elif distro_name == "rhel" %} + - name: Build anaconda-ci container + run: make -f Makefile.am anaconda-ci-build + {% endif %} + + - name: Run tests in anaconda-ci container + run: | + # put the log in the output, where it's easy to read and link to + make -f Makefile.am container-ci || { cat test-logs/test-suite.log; exit 1; } + + - name: Upload test and coverage logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: 'logs (${{ matrix.ci_tag }})' + path: test-logs/* + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + + rpm-tests: + {% if distro_name == "fedora" %} + runs-on: ubuntu-20.04 + {% elif distro_name == "rhel" %} + runs-on: [self-hosted, kstest] + {% endif %} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + {# For matrix details, see comments for the unit tests above. #} + {% if distro_name == "fedora" and distro_release == "rawhide" %} + release: [''] + include: + - release: '' + target_branch: 'master' + ci_tag: 'master' + ## add to release: [...] also eln if re-enabled by uncommenting the below + #- release: eln + # target_branch: 'master' + # ci_tag: 'eln' + # build-args: '--build-arg=image=quay.io/fedoraci/fedora:eln-x86_64' + {% elif distro_name == "fedora" and distro_release is number %} + release: [''] + include: + - release: '' + target_branch: 'f{$ distro_release $}-release' + ci_tag: 'f{$ distro_release $}-release' + {% elif distro_name == "rhel" %} + release: [''] + include: + - release: '' + target_branch: 'rhel-{$ distro_release $}' + ci_tag: 'rhel-{$ distro_release $}' + {% endif %} + + env: + CI_TAG: '${{ matrix.ci_tag }}' + # Always avoid using cache because cache is not correctly invalidated. + CONTAINER_BUILD_ARGS: '--no-cache ${{ matrix.build-args }}' + TARGET_BRANCH_NAME: 'origin/${{ matrix.target_branch }}' + + steps: + - name: Clone repository + uses: actions/checkout@v3 + with: + # TODO: Are we able to remove ref, fetch-depth and Rebase task? Seems that the checkout + # without ref is doing the rebase for us. + # otherwise we are testing target branch instead of the PR branch (see pull_request_target trigger) + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Rebase to current ${{ env.TARGET_BRANCH_NAME }} + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git log --oneline -1 ${{ env.TARGET_BRANCH_NAME }} + git rebase ${{ env.TARGET_BRANCH_NAME }} + + - name: Build RPM test container + run: make -f Makefile.am anaconda-rpm-build + + - name: Run RPM tests in container + run: make -f Makefile.am container-rpm-test + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: 'logs-rpm-test (${{ matrix.ci_tag }})' + path: test-logs/* diff --git a/.github/workflows/try-release-daily.yml b/.github/workflows/try-release-daily.yml index c6cbdcac241..7302e42f351 100644 --- a/.github/workflows/try-release-daily.yml +++ b/.github/workflows/try-release-daily.yml @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: try-release-daily.yml.j2 + name: Try building a release daily # Build a release tarball for all supported branches to verify the code is releasable. @@ -16,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - branch: ['master', 'rhel-8', 'rhel-9', 'f37-release'] # depending on season, also: fNN-release + branch: ['master', 'rhel-8', 'rhel-9', 'f37-release'] steps: - name: Check out repo diff --git a/.github/workflows/try-release-daily.yml.j2 b/.github/workflows/try-release-daily.yml.j2 new file mode 100644 index 00000000000..d284bc2fdac --- /dev/null +++ b/.github/workflows/try-release-daily.yml.j2 @@ -0,0 +1,76 @@ +{% if distro_release == "rawhide" %} +name: Try building a release daily +# Build a release tarball for all supported branches to verify the code is releasable. + +on: + schedule: + - cron: 0 3 * * * + # be able to start this action manually from a actions tab when needed + workflow_dispatch: + +permissions: + contents: read + +jobs: + daily-release-test: + runs-on: [self-hosted, kstest] + strategy: + fail-fast: false + matrix: + {% if branched_fedora_version is defined and branched_fedora_version %} + branch: ['master', 'rhel-8', 'rhel-9', 'f{$ branched_fedora_version $}-release'] + {% else %} + branch: ['master', 'rhel-8', 'rhel-9'] + {% endif %} + + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + ref: ${{ matrix.branch }} + + - name: Build anaconda container (to make the release) + run: | + make -f Makefile.am anaconda-release-build + + - name: Run the build in the container + run: | + make -f Makefile.am container-release | tee release.log + + - name: Check logs for malformed translations + continue-on-error: true + run: | + errors=$(grep -E "Unable to compile [^:]+: .*" release.log || true) # grep exits with 1 if no lines found + if [ -n "$errors" ]; then + echo "Failed to compile some translations:" + echo "$errors" + exit 1 + else + echo "No translation compilation errors found." + fi + + - name: Check if translation files are present in tarball + continue-on-error: true + run: | + tarfiles=$(tar -tf anaconda-*.tar.bz2) + missing=0 + required_languages="fr ja zh_CN es pt de ru ar" + # Languages that are presented first in the GUI. + # This list comes from langtable's list_common_languages, which in turn reportedly comes + # from gnome-control-center's cc_common_language_get_initial_languages. + if [[ "${{ matrix.branch }}" == *rhel-* ]]; then + required_languages="$required_languages ko it zh_TW" + # Additionally, we test languages that are supported and translated on RHEL. + fi + for translation in $required_languages ; do + if ! [[ $tarfiles == */po/$translation.po* ]]; then + echo "missing mandatory translation: $translation" + missing=1 + fi + done + if [[ "$missing" == 0 ]]; then + echo "All mandatory translations found: $required_languages" + else + exit 1 + fi +{% endif %} From 607b64357e58e7f49163b60cd47b8fe6a962d372 Mon Sep 17 00:00:00 2001 From: Vladimir Slavik Date: Tue, 23 Aug 2022 17:38:18 +0200 Subject: [PATCH 5/5] Templatize branch-config (#infra) --- branch-config.mk | 8 +++++++ branch-config.mk.j2 | 51 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 branch-config.mk.j2 diff --git a/branch-config.mk b/branch-config.mk index f148475a46d..48c374ae4fa 100644 --- a/branch-config.mk +++ b/branch-config.mk @@ -1,3 +1,10 @@ +# ====================================== +# WARNING! +# THIS FILE IS GENERATED FROM A TEMPLATE +# DO NOT EDIT THIS FILE MANUALLY! +# ====================================== +# The template is located in: branch-config.mk.j2 + # Makefile include for branch specific configuration settings # # Copyright (C) 2020 Red Hat, Inc. @@ -32,3 +39,4 @@ BASE_CONTAINER ?= registry.fedoraproject.org/fedora:rawhide # COPR repo for use in container builds. # Can be @rhinstaller/Anaconda for master, or @rhinstaller/Anaconda-devel for branched Fedora. COPR_REPO ?= \@rhinstaller/Anaconda + diff --git a/branch-config.mk.j2 b/branch-config.mk.j2 new file mode 100644 index 00000000000..d0e3f5b766e --- /dev/null +++ b/branch-config.mk.j2 @@ -0,0 +1,51 @@ +# Makefile include for branch specific configuration settings +# +# Copyright (C) 2020 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +# +# Store a branch specific configuration here to avoid dealing with +# conflicts on multiple places. + +{% if distro_name == "fedora" and distro_release == "rawhide" %} + +# Name of the expected current git branch. +# This could be master, fXX-devel, fXX-release, rhelX-branch, rhel-X ... +GIT_BRANCH ?= master + +# Directory for this anaconda branch in anaconda-l10n repository. This could be master, fXX, rhel-8 etc. +L10N_DIR ?= master + +# Base container for our containers. +BASE_CONTAINER ?= registry.fedoraproject.org/fedora:rawhide + +# COPR repo for use in container builds. +# Can be @rhinstaller/Anaconda for master, or @rhinstaller/Anaconda-devel for branched Fedora. +COPR_REPO ?= \@rhinstaller/Anaconda + +{% elif distro_name == "fedora" %} + +GIT_BRANCH ?= f{$ distro_release $}-release +L10N_DIR ?= f{$ distro_release $} +BASE_CONTAINER ?= registry.fedoraproject.org/fedora:{$ distro_release $} +COPR_REPO ?= \@rhinstaller/Anaconda-devel + +{% elif distro_name == "rhel" %} + +GIT_BRANCH ?= rhel-{$ distro_release $} +L10N_DIR ?= rhel-{$ distro_release $} +BASE_CONTAINER ?= registry-proxy.engineering.redhat.com/rh-osbs/ubi{$ distro_release $}:latest +COPR_REPO ?= rhinstaller-group/Anaconda + +{% endif %}