build-workflow.yml

name: build-workflow

on:
  push:
    tags:
      - "*"

  merge_group:
  workflow_dispatch:
    inputs:
      include:
        description: Only run tests matching tests with the given tags
        type: string
        required: false
        default: ""
      processes:
        description: Number of processes to run tests
        type: string
        required: false
        default: "10"
      run_rust_tests:
        description: Run Rust tests (generally this is not required as the PR check will run full Rust tests)
        type: boolean
        required: false
        default: false

  # Use a manual approval process before PR's are given access to
  # the secrets which are required to run the integration tests.
  # The PR code should be manually approved to see if it can be trusted.
  # When in doubt, do not approve the test run.
  # Reference: https://dev.to/petrsvihlik/using-environment-protection-rules-to-secure-secrets-when-building-external-forks-with-pullrequesttarget-hci
  pull_request_target:
    branches: [ main ]

env:
  CARGO_TERM_COLOR: always

jobs:
  build:
    name: Build ${{ matrix.target }}
    runs-on: ${{ matrix.host_os }}
    strategy:
      fail-fast: false
      matrix:
        # Note: Targets which requires a non-default rust_channel
        # then they should not be included in the .target list, but
        # rather in the include section.
        # This is just how Github processes matrix jobs
        target:
          - aarch64-unknown-linux-musl
          - armv7-unknown-linux-musleabihf
          - arm-unknown-linux-musleabihf
          - arm-unknown-linux-musleabi
          - armv5te-unknown-linux-musleabi
          - x86_64-unknown-linux-musl
          - i686-unknown-linux-musl
          - riscv64gc-unknown-linux-gnu
          - aarch64-apple-darwin
          - x86_64-apple-darwin
        mode:
          - --release

        rust_channel:
          - "1.78"

        include:
          - target: aarch64-unknown-linux-musl
            host_os: ubuntu-22.04
            cargo_options: --no-run
          
          - target: armv7-unknown-linux-musleabihf
            host_os: ubuntu-22.04
            cargo_options: --no-run

          - target: arm-unknown-linux-musleabihf
            host_os: ubuntu-22.04
            cargo_options: --no-run

          - target: arm-unknown-linux-musleabi
            host_os: ubuntu-22.04
            cargo_options: --no-run

          - target: armv5te-unknown-linux-musleabi
            host_os: ubuntu-22.04
            cargo_options: --no-run

          - target: x86_64-unknown-linux-musl
            host_os: ubuntu-22.04

          - target: i686-unknown-linux-musl
            host_os: ubuntu-22.04
            cargo_options: --no-run

          - target: riscv64gc-unknown-linux-gnu
            host_os: ubuntu-22.04
            cargo_options: --no-run

          - target: aarch64-apple-darwin
            host_os: macos-14
            cargo_options: --no-run

          - target: x86_64-apple-darwin
            host_os: macos-14

    steps:
      - if: ${{ contains(matrix.host_os, 'ubuntu') }}
        run: |
          sudo apt-get update -y
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}
          fetch-depth: 0

      - run: mk/install-build-tools.sh +${{ matrix.rust_channel }} --target=${{ matrix.target }}
        shell: sh
      - name: Build
        run: |
          . ./ci/build_scripts/version.sh
          mk/cargo.sh +${{ matrix.rust_channel }} build --target=${{ matrix.target }} ${{ matrix.mode }}

      - if: ${{ inputs.run_rust_tests && !contains(matrix.host_os, 'windows') }}
        name: Test
        run: |
          mk/cargo.sh +${{ matrix.rust_channel }} test -vv --target=${{ matrix.target }} ${{ matrix.cargo_options }} ${{ matrix.mode }} --no-fail-fast --locked --all-features --all-targets

      # Install nfpm used to for linux packaging
      - uses: actions/setup-go@v5
        with:
          go-version: 'stable'
          cache: false
      - run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest

      - name: Package
        run: |
          ./ci/build_scripts/build.sh ${{ matrix.target }} --skip-build --skip-deprecated-packages

      - name: Upload packages as zip
        # https://github.com/marketplace/actions/upload-a-build-artifact
        uses: actions/upload-artifact@v4
        with:
            name: packages-${{ matrix.target }}
            path: target/${{ matrix.target }}/packages/*.*

  # Note: Validate that all build jobs completed successfully
  # If a Runner failues on the setup, it can leave a job marked as "skipped"
  # and this does not cause the entire job to fail, instead it silently continues
  # leading to downstream problems (e.g. only publishing half of the artifacts!)
  #
  # see https://stackoverflow.com/a/67532120/4907315
  # Fail if any `needs` job was not a success.
  # Along with `if: always()`, this allows this job to act as a single required status check for the entire build job
  #
  check-build:
    runs-on: ubuntu-latest
    needs: [build]
    if: always()
    steps:
    - name: Fail on workflow error
      run: exit 1
      if: >-
        ${{
          contains(needs.*.result, 'failure')
          || contains(needs.*.result, 'cancelled')
          || contains(needs.*.result, 'skipped')
        }}

  approve:
    # Note: Use approval as a job so that the downstream jobs are only prompted once (if more than 1 matrix job is defined)
    name: Approve
    environment:
      # For security reasons, all pull requests need to be approved first before granting access to secrets
      # So the environment should be set to have a reviewer/s inspect it before approving it
      name: ${{ github.event_name == 'pull_request_target' && 'Test Pull Request' || 'Test Auto'  }}
    runs-on: ubuntu-22.04
    steps:
      - name: Wait for approval
        run: echo "Approved"

  test:
    name: Test ${{ matrix.job.name }}
    # Don't tests on tagging as it has already run in the merge queue
    if: ${{ !startsWith(github.ref, 'refs/tags/') }}
    needs: [approve, check-build]
    environment:
      name: Test Auto
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        job:
          - { name: x86_64, target: x86_64-unknown-linux-musl, output: target/packages }
    steps:
      # Checkout either the PR or the branch
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}
          fetch-depth: 0

      - name: Download release artifacts
        uses: actions/download-artifact@v4
        with:
          name: packages-${{ matrix.job.target }}
          path: tests/images/debian-systemd/files/packages/

      - name: create .env file
        working-directory: tests/RobotFramework
        run: |
          touch .env
          echo 'C8Y_BASEURL="${{ secrets.C8Y_BASEURL }}"' >> .env
          echo 'C8Y_USER="${{ secrets.C8Y_USER }}"' >> .env
          echo 'C8Y_TENANT="${{ secrets.C8Y_TENANT }}"' >> .env
          echo 'C8Y_PASSWORD="${{ secrets.C8Y_PASSWORD }}"' >> .env
          echo 'CA_KEY="${{ secrets.CA_KEY || '' }}"' >> .env
          echo 'CA_PUB="${{ secrets.CA_PUB || '' }}"' >> .env

      - uses: actions/setup-python@v5
        with:
          python-version: '3.9'
          cache: 'pip'
          cache-dependency-path: |
            **/requirements/requirements*.txt

      - name: Install dependencies
        run: |
          ./bin/setup.sh
        working-directory: tests/RobotFramework

      - name: Build images
        working-directory: tests/RobotFramework
        run: |
          source .venv/bin/activate
          invoke build

      - name: Run tests
        working-directory: tests/RobotFramework
        run: |
          source .venv/bin/activate
          invoke test \
            --processes "${{ inputs.processes || '' }}" \
            --include "${{ inputs.include || '' }}" \
            --exclude "test:on_demand OR theme:benchmarks" \
            --outputdir output

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: report-${{ matrix.job.target }}
          path: tests/RobotFramework/output

      - name: Send report to commit
        uses: joonvena/robotframework-reporter-action@v2.5
        if: always() && github.event_name == 'pull_request_target'
        with:
          gh_access_token: ${{ secrets.GITHUB_TOKEN }}
          report_path: 'tests/RobotFramework/output'
          show_passed_tests: 'false'

  publish:
    name: Publish ${{ matrix.job.target }}
    if: |
      always() &&
      github.event_name != 'pull_request_target' &&
      (needs.check-build.result == 'success') &&
      (needs.test.result == 'success' || needs.test.result == 'skipped')
    runs-on: ubuntu-20.04
    needs: [check-build, test]
    strategy:
      fail-fast: false
      matrix:
        job:
          - { target: x86_64-unknown-linux-musl,      repo: tedge-main,          component: main }
          - { target: aarch64-unknown-linux-musl,     repo: tedge-main,          component: main }
          # Keep arm-unknown-linux-musleabihf in separate repo due to armhf conflict between raspbian and debian
          - { target: arm-unknown-linux-musleabihf,   repo: tedge-main-armv6,    component: main }
          - { target: armv7-unknown-linux-musleabihf, repo: tedge-main,          component: main }
          - { target: arm-unknown-linux-musleabi,     repo: tedge-main,          component: main }
          - { target: i686-unknown-linux-musl,        repo: tedge-main,          component: main }
          # Debian also calls this "armel" (conflict with arm-unknown-linux-musleabi)
          # - { target: armv5te-unknown-linux-musleabi, repo: tedge-main,          component: main }
          - { target: riscv64gc-unknown-linux-gnu,    repo: tedge-main,          component: main }
          - { target: aarch64-apple-darwin,      repo: tedge-main,          component: main }
          - { target: x86_64-apple-darwin,       repo: tedge-main,          component: main }

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      # Setup python required by cloudsmith cli
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Download release artifacts
        uses: actions/download-artifact@v4
        # https://github.com/marketplace/actions/download-a-build-artifact
        with:
          name: packages-${{ matrix.job.target }}
          path: target/${{ matrix.job.target }}/packages/

      - uses: taiki-e/install-action@just
      - name: Publish packages
        env:
          PUBLISH_OWNER: ${{ secrets.PUBLISH_OWNER }}
          PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }}
        run: |
          just publish-linux-target "${{ matrix.job.target }}" \
            --repo "${{ matrix.job.repo }}" \
            --component "${{ matrix.job.component }}"

  # Wait until all other publishing jobs are finished
  # before publishing the virtual packages (which are architecture agnostic)
  publish-containers:
    name: Publish Containers
    if: |
      always() &&
      github.event_name != 'pull_request_target' &&
      (needs.check-build.result == 'success') &&
      (needs.test.result == 'success' || needs.test.result == 'skipped')
    runs-on: ubuntu-20.04
    needs: [check-build, test]
    env:
      BUILDX_NO_DEFAULT_ATTESTATIONS: 1
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: taiki-e/install-action@just
      - id: tedge
        name: Get Version
        run: |
          version=$(just version container)
          echo "Detected version: $version"
          echo "version=$version" >> "$GITHUB_OUTPUT"

      # Download artifacts for all targets
      # The docker build step will select the correct target for the
      # given container target platform
      - name: Download release artifacts
        uses: actions/download-artifact@v4
        with:
          path: containers/tedge/packages/

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            name=ghcr.io/thin-edge/tedge,enable=${{ startsWith(github.ref, 'refs/tags/') }}
            name=ghcr.io/thin-edge/tedge-main,enable=true
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=raw,value=${{ steps.tedge.outputs.version }},enable=${{ !startsWith(github.ref, 'refs/tags/') }}
            type=raw,value=latest
      
      - name: Build and push
        uses: docker/build-push-action@v6
        with:
          context: containers/tedge
          push: ${{ github.event_name != 'pull_request_target' }}
          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: |
            BUILDTIME=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
            VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
            REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}

  # Wait until all other publishing jobs are finished
  # before publishing the virtual packages (which are architecture agnostic)
  publish-virtual-packages:
    name: Publish Virtual Packages
    if: |
      always() &&
      github.event_name != 'pull_request_target' &&
      needs.publish.result == 'success'
    runs-on: ubuntu-20.04
    needs: [publish]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      # Setup python required by cloudsmith cli
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'
      # Install nfpm used to for linux packaging
      - uses: actions/setup-go@v5
        with:
          go-version: 'stable'
          cache: false
      - run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
      - uses: taiki-e/install-action@just
      - name: Build virtual packages
        run: just release-linux-virtual
      - name: Publish packages
        env:
          PUBLISH_OWNER: ${{ secrets.PUBLISH_OWNER }}
          PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }}
        run: |
          just publish-linux-virtual --repo tedge-main
          just publish-linux-virtual --repo tedge-main-armv6

  release:
    runs-on: ubuntu-latest
    needs: [publish-virtual-packages, publish-containers]
    if: |
      always() &&
      startsWith(github.ref, 'refs/tags/') &&
      needs.publish-virtual-packages.result == 'success' &&
      needs.publish-containers.result == 'success'
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          persist-credentials: false
      - uses: taiki-e/install-action@just
      - uses: taiki-e/install-action@git-cliff
      - name: Generate changelog
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if ! just generate-changelog --from-tags; then
            echo "Warning: Failed to generate changelog, but it should not block a release" > _CHANGELOG.md
          fi
      - name: Release
        uses: softprops/action-gh-release@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          body_path: _CHANGELOG.md
          generate_release_notes: false
          draft: true

      - name: Create tedge-docs snapshot
        run: |
          gh workflow run snapshot.yml -R thin-edge/tedge-docs -f version=${{github.ref_name}}
        env:
          # Triggering another workflow requires more additional credentials
          GITHUB_TOKEN: ${{ secrets.ACTIONS_PAT }}

      - name: Promote cloudsmith packages
        env:
          VERSION: ${{ github.ref_name }}
          PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }}
        run: |
          ./ci/admin/cloudsmith_admin.sh promote "$VERSION"