From 6ba60e2b150221d6781ad8be27745e88bfb97611 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 10 Jan 2025 12:34:50 +0100 Subject: [PATCH] Add GH job with array API conformity run --- .github/workflows/array-api-skips.txt | 71 ++++++++++ .github/workflows/conda-package.yml | 188 +++++++++++++++++++++++++- 2 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/array-api-skips.txt diff --git a/.github/workflows/array-api-skips.txt b/.github/workflows/array-api-skips.txt new file mode 100644 index 00000000000..c4f5bc5d71e --- /dev/null +++ b/.github/workflows/array-api-skips.txt @@ -0,0 +1,71 @@ +# array API tests to be skipped + +# no 'uint8' dtype +array_api_tests/test_array_object.py::test_getitem_masking + +# no 'isdtype' function +array_api_tests/test_data_type_functions.py::test_isdtype +array_api_tests/test_has_names.py::test_has_names[data_type-isdtype] +array_api_tests/test_signatures.py::test_func_signature[isdtype] + +# missing unique-like functions +array_api_tests/test_has_names.py::test_has_names[set-unique_all] +array_api_tests/test_has_names.py::test_has_names[set-unique_counts] +array_api_tests/test_has_names.py::test_has_names[set-unique_inverse] +array_api_tests/test_has_names.py::test_has_names[set-unique_values] +array_api_tests/test_set_functions.py::test_unique_all +array_api_tests/test_set_functions.py::test_unique_counts +array_api_tests/test_set_functions.py::test_unique_inverse +array_api_tests/test_set_functions.py::test_unique_values +array_api_tests/test_signatures.py::test_func_signature[unique_all] +array_api_tests/test_signatures.py::test_func_signature[unique_counts] +array_api_tests/test_signatures.py::test_func_signature[unique_inverse] +array_api_tests/test_signatures.py::test_func_signature[unique_values] + +# no '__array_namespace_info__' function +array_api_tests/test_has_names.py::test_has_names[info-__array_namespace_info__] +array_api_tests/test_inspection_functions.py::test_array_namespace_info +array_api_tests/test_inspection_functions.py::test_array_namespace_info_dtypes +array_api_tests/test_searching_functions.py::test_searchsorted +array_api_tests/test_signatures.py::test_func_signature[__array_namespace_info__] +array_api_tests/test_signatures.py::test_info_func_signature[capabilities] +array_api_tests/test_signatures.py::test_info_func_signature[default_device] +array_api_tests/test_signatures.py::test_info_func_signature[default_dtypes] +array_api_tests/test_signatures.py::test_info_func_signature[devices] +array_api_tests/test_signatures.py::test_info_func_signature[dtypes] + +# do not return a namedtuple +array_api_tests/test_linalg.py::test_eigh +array_api_tests/test_linalg.py::test_slogdet +array_api_tests/test_linalg.py::test_svd + +# hypothesis found failures +array_api_tests/test_linalg.py::test_qr +array_api_tests/test_operators_and_elementwise_functions.py::test_clip + +# unexpected result is returned +array_api_tests/test_operators_and_elementwise_functions.py::test_asin +array_api_tests/test_operators_and_elementwise_functions.py::test_asinh + +# missing 'descending' keyword argument +array_api_tests/test_signatures.py::test_func_signature[argsort] +array_api_tests/test_signatures.py::test_func_signature[sort] +array_api_tests/test_sorting_functions.py::test_argsort +array_api_tests/test_sorting_functions.py::test_sort + +# missing 'correction' keyword argument +array_api_tests/test_signatures.py::test_func_signature[std] +array_api_tests/test_signatures.py::test_func_signature[var] + +# missing 'stream' keyword argument +array_api_tests/test_signatures.py::test_array_method_signature[to_device] + +# wrong shape is returned +array_api_tests/test_linalg.py::test_vecdot +array_api_tests/test_linalg.py::test_linalg_vecdot + +# tuple index out of range +array_api_tests/test_linalg.py::test_linalg_matmul + +# arrays have different values +array_api_tests/test_linalg.py::test_linalg_tensordot diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 3dd129d6b9a..d3b2ce1d665 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -210,7 +210,7 @@ jobs: - name: Smoke test run: | - python -c "import dpnp, dpctl; dpctl.lsplatform()" + python -c "import dpctl; dpctl.lsplatform()" python -c "import dpnp; print(dpnp.__version__)" - name: Run tests @@ -350,7 +350,7 @@ jobs: - name: Smoke test run: | - python -c "import dpnp, dpctl; dpctl.lsplatform()" + python -c "import dpctl; dpctl.lsplatform()" python -c "import dpnp; print(dpnp.__version__)" - name: Run tests @@ -432,6 +432,190 @@ jobs: env: ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + array-api-conformity: + name: Array API conformity + + needs: build + + permissions: + # Needed to add a comment to a pull request's issue + pull-requests: write + + strategy: + matrix: + python: ['3.12'] + os: [ubuntu-22.04] + + runs-on: ${{ matrix.os }} + + defaults: + run: + shell: bash -el {0} + + continue-on-error: true + + env: + array-api-tests-path: '${{ github.workspace }}/array-api-tests/' + json-report-file: '${{ github.workspace }}/.report.json' + dpnp-repo-path: '${{ github.workspace }}/dpnp/' + array-api-skips-file: '${{ github.workspace }}/dpnp/.github/workflows/array-api-skips.txt' + channel-path: '${{ github.workspace }}/channel/' + pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/' + extracted-pkg-path: '${{ github.workspace }}/pkg/' + ver-json-path: '${{ github.workspace }}/version.json' + + steps: + - name: Download artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} + path: ${{ env.pkg-path-in-channel }} + + - name: Extract package archive + run: | + mkdir -p ${{ env.extracted-pkg-path }} + tar -xvf ${{ env.pkg-path-in-channel }}/${{ env.PACKAGE_NAME }}-*.tar.bz2 -C ${{ env.extracted-pkg-path }} + + - name: Setup miniconda + id: setup_miniconda + continue-on-error: true + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 + with: + miniforge-version: latest + use-mamba: 'true' + channels: conda-forge + conda-remove-defaults: 'true' + python-version: ${{ matrix.python }} + activate-environment: 'array-api-conformity' + + - name: ReSetup miniconda + if: steps.setup_miniconda.outcome == 'failure' + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 + with: + miniforge-version: latest + use-mamba: 'true' + channels: conda-forge + conda-remove-defaults: 'true' + python-version: ${{ matrix.python }} + activate-environment: 'array-api-conformity' + + - name: Install conda-index + id: install_conda_index + continue-on-error: true + run: mamba install conda-index=${{ env.CONDA_INDEX_VERSION }} + + - name: ReInstall conda-index + if: steps.install_conda_index.outcome == 'failure' + run: mamba install conda-index=${{ env.CONDA_INDEX_VERSION }} + + - name: Create conda channel + run: | + python -m conda_index ${{ env.channel-path }} + + - name: Test conda channel + run: | + conda search ${{ env.PACKAGE_NAME }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} + cat ${{ env.ver-json-path }} + + - name: Get package version + run: | + export PACKAGE_VERSION=$(python -c "${{ env.VER_SCRIPT1 }} ${{ env.VER_SCRIPT2 }}") + + echo PACKAGE_VERSION=${PACKAGE_VERSION} + echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV + + - name: Install dpnp + id: install_dpnp + continue-on-error: true + run: | + mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} + env: + TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.CHANNELS }}' + + - name: ReInstall dpnp + if: steps.install_dpnp.outcome == 'failure' + run: | + mamba install ${{ env.PACKAGE_NAME }}=${{ env.PACKAGE_VERSION }} pytest python=${{ matrix.python }} ${{ env.TEST_CHANNELS }} + env: + TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.CHANNELS }}' + + - name: List installed packages + run: mamba list + + - name: Smoke test + run: | + python -c "import dpctl; dpctl.lsplatform()" + python -c "import dpnp; print(dpnp.__version__)" + + - name: Clone array API tests repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: 'data-apis/array-api-tests' + path: ${{ env.array-api-tests-path }} + fetch-depth: 0 + submodules: 'recursive' + + - name: Install array API test dependencies + run: | + pip install -r requirements.txt + working-directory: ${{ env.array-api-tests-path }} + + - name: Install jq + run: | + sudo apt-get install jq + + - name: List installed packages + run: mamba list + + - name: Smoke test + run: | + python -c "import dpctl; dpctl.lsplatform()" + python -c "import dpnp; print(dpnp.__version__)" + + # need to fetch array-api-skips.txt + - name: Checkout DPNP repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: ${{ env.dpnp-repo-path }} + + - name: Run array API conformance tests + run: | + python -m pytest --json-report --json-report-file=${{ env.json-report-file }} --disable-deadline --skips-file ${{ env.array-api-skips-file }} array_api_tests || true + env: + ARRAY_API_TESTS_MODULE: 'dpnp' + SYCL_CACHE_PERSISTENT: 1 + working-directory: ${{ env.array-api-tests-path }} + + - name: Set Github environment variables + run: | + FILE=${{ env.json-report-file }} + if test -f "$FILE"; then + PASSED_TESTS=$(jq '.summary | .passed // 0' $FILE) + FAILED_TESTS=$(jq '.summary | .failed // 0' $FILE) + SKIPPED_TESTS=$(jq '.summary | .skipped // 0' $FILE) + MESSAGE="Array API standard conformance tests for dpnp=$PACKAGE_VERSION ran successfully. + Passed: $PASSED_TESTS + Failed: $FAILED_TESTS + Skipped: $SKIPPED_TESTS" + echo "MESSAGE<> $GITHUB_ENV + echo "$MESSAGE" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + else + echo "Array API standard conformance tests failed to run for dpnp=$PACKAGE_VERSION." + exit 1 + fi + + - name: Output API summary + run: echo "::notice ${{ env.MESSAGE }}" + + - name: Post result to PR + if: ${{ github.event.pull_request && !github.event.pull_request.head.repo.fork }} + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 + with: + message: | + ${{ env.MESSAGE }} + allow-repeats: true + cleanup_packages: name: Clean up anaconda packages