ci-linux.yml

name: CI Linux

on:
  workflow_call:
  workflow_dispatch:
  pull_request:
  merge_group:
  push:
    branches:
      - main

concurrency:
  # A PR number if a pull request and otherwise the commit hash. This cancels
  # queued and in-progress runs for the same PR (presubmit) or commit
  # (postsubmit).
  group: ci-build-test-cpp-linux-${{ github.event.number || github.sha }}
  cancel-in-progress: true

jobs:
  build_and_ctest:
    name: Build and Test (linux, ASSERTIONS)
    runs-on: nod-ai-shared-cpubuilder-manylinux-x86_64
    strategy:
      fail-fast: true
    env:
      CACHE_DIR: ${{ github.workspace }}/.container-cache
      # either the PR number or `branch-N` where N always increments
      CACHE_KEY: linux-build-test-cpp-asserts-manylinux-v2-${{ format('{0}-{1}', github.ref_name, github.run_number) }}
    steps:
      - name: Set unified TZ
        uses: szenius/set-timezone@v2.0
        with:
          # this is an arbitrary choice
          timezoneLinux: "Asia/Singapore"
          timezoneMacos: "Asia/Singapore"
          timezoneWindows: "Singapore Standard Time"

      - name: Configure local git mirrors
        run: |
          /gitmirror/scripts/trigger_update_mirrors.sh
          /gitmirror/scripts/git_config.sh

      - name: "Checking out repository"
        env:
          BRANCH_NAME: ${{ github.ref }}
          REPO_ADDRESS: ${{ github.server_url }}/${{ github.repository }}
        run: |
          git init
          git remote add origin $REPO_ADDRESS
          git -c protocol.version=2 fetch --depth 1 origin $BRANCH_NAME
          git reset --hard FETCH_HEAD
          git -c submodule."third_party/torch-mlir".update=none -c submodule."third_party/stablehlo".update=none -c submodule."src/runtime_src/core/common/aiebu".update=none submodule update --init --recursive --depth 1 --single-branch -j 10
          
      - name: Install deps
        run: |
          dnf install -y almalinux-release-devel epel-release
          yum remove -y openssl-devel zlib-devel || true
          yum install -y protobuf-devel protobuf-compiler tmate

      - name: Python deps
        run: |
          pip install "numpy<2" pyyaml "pybind11[global]==2.10.3" nanobind pytest

      - name: Run Pytest
        run: |
          pytest build_tools/ci

      - name: Enable cache
        uses: actions/cache/restore@v3
        with:
          path: ${{ env.CACHE_DIR }}
          key:  ${{ env.CACHE_KEY }}
          restore-keys: linux-build-test-cpp-

      - name: Build packages
        run: |
          export cache_dir="${{ env.CACHE_DIR }}"
          export CCACHE_COMPILERCHECK="string:$(clang --version)"
          bash build_tools/build_llvm.sh
          rm -rf llvm-build
          export llvm_install_dir=$PWD/llvm-install
          bash build_tools/build_test_cpp.sh

      - name: Create artifacts
        if: ${{ !cancelled() }}
        run: |
          pushd third_party/iree/third_party/llvm-project && llvm_sha_short=$(git rev-parse --short HEAD) && popd
          tar cf llvm-dist-linux-$llvm_sha_short.tar llvm-install
          tar cf iree-dist-linux.tar iree-install

      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: linux_x86_64_llvm_packages
          path: llvm-dist-*.tar
          if-no-files-found: warn

      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: linux_x86_64_iree_packages
          path: iree-dist-linux.tar
          if-no-files-found: warn

      - name: Save cache
        uses: actions/cache/save@v3
        if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }}
        with:
          path: ${{ env.CACHE_DIR }}
          key: ${{ env.CACHE_KEY }}


  test_linux:
    name: E2E Test linux
    needs: build_and_ctest
    strategy:
      fail-fast: false
      matrix:
        runs-on: [linux-phoenix]
    runs-on: ${{ matrix.runs-on }}
    env:
      XILINXD_LICENSE_FILE: /opt/xilinx/Xilinx.lic
    steps:
      - name: "Checking out repository" # for test scripts
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          submodules: false # not required for testbench

      - name: Download IREE artifacts
        uses: actions/download-artifact@v4
        with:
          name: linux_x86_64_iree_packages

      - name: Download LLVM artifacts
        uses: actions/download-artifact@v4
        with:
          name: linux_x86_64_llvm_packages

      - name: Extract artifact
        run: |
          tar -xvf llvm-dist-linux-*.tar
          echo "PATH=$PWD/llvm-install/bin:$PATH" >> $GITHUB_ENV
          tar -xvf iree-dist-linux.tar
          bash build_tools/download_peano.sh

      - name: Create venv and install dependencies
        run: |
          python3 -m venv .venv
          source .venv/bin/activate
          pip install -r tests/matmul/requirements.txt

      name: Unlimit locked memory for XRT
        run: |
          # Without this additional line an error like
          #
          # [XRT] ERROR: Failed to allocate host memory buffer (mmap(len=10616832, prot=3, flags=8193, offset=4294967296)
          # failed (err=11): Resource temporarily unavailable), make sure host bank is enabled (see xbutil configure --host-mem)
          # iree-amd-aie/runtime/src/iree-amd-aie/driver/xrt/direct_allocator.cc:179: RESOURCE_EXHAUSTED; could not allocate
          # memory for buffer; while invoking C++ function matmul_test.generate_random_matrix; while calling import;
          #
          # might be observed when too much memory is allocated. For example this
          # error was seen when running a bf16->f32 matmul with m=n=k=2304.
          #
          # This line was suggested at https://github.com/Xilinx/mlir-air/issues/566
          #
          # Note that this is only half of the fix. It is also necessary that
          # the machine that CI is running on has permission to run this line.
          #
          # This permission can be adding by adding the line
          # ```
          # %github ALL=(ALL) NOPASSWD: /usr/bin/prlimit *
          # ```
          #
          # to the file /etc/sudoers.d/github, which can be done by running
          # ```
          # sudo visudo -f /etc/sudoers.d/github
          # ```
          # on the guthub CI machine.
          sudo prlimit -lunlimited --pid $$

      - name : E2E comparison of AIE to llvm-cpu
        run: |
          source .venv/bin/activate
          source /opt/xilinx/xrt/setup.sh
          python3 build_tools/ci/cpu_comparison/run.py \
            test_aie_vs_cpu \
            $PWD/iree-install \
            $PWD/llvm-aie \
            --xrt-dir /opt/xilinx/xrt \
            --vitis-dir /opt/Xilinx/Vitis/2024.2 \
            --reset-npu-between-runs -v

      - name: E2E correctness matmul test
        run: |
          source .venv/bin/activate
          source /opt/xilinx/xrt/setup.sh
          bash build_tools/ci/run_matmul_test.sh \
            test_matmuls \
            iree-install \
            $PWD/llvm-aie \
            /opt/xilinx/xrt \
            /opt/Xilinx/Vitis/2024.2

      - name: Printing IR from aie2xclbin
        run: |
          source .venv/bin/activate
          source /opt/xilinx/xrt/setup.sh
          bash build_tools/ci/print_ir_aie2xclbin/print_ir_aie2xclbin.sh \
            iree-install \
            print_ir_aie2xclbin_results \
            $PWD/llvm-aie

      - name : Smoke E2E comparison flag test
        run: |
          source .venv/bin/activate
          source /opt/xilinx/xrt/setup.sh
          python3 build_tools/ci/cpu_comparison/run.py \
            test_aie_vs_cpu \
            iree-install \
            $PWD/llvm-aie \
            --xrt-dir /opt/xilinx/xrt \
            --test-set='Smoke' \
            --do-not-run-aie > smoke_output.log 2>&1

          # Assert that smoke_output.log is empty (because verbose=0)
          # First assert that smoke_output.log exists:
          if [ ! -f smoke_output.log ]; then
            echo "smoke_output.log does not exist."
            exit 1
          fi
          if [ -s smoke_output.log ]; then
            echo "smoke_output.log is not empty:"
            cat smoke_output.log
            exit 1
          else
            echo "smoke_output.log is empty"
          fi