From 733272b304563c2582c2c7d44196e218125b4938 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Sun, 26 Jan 2025 14:57:02 +0100 Subject: [PATCH] add tests for sources that depend on conda build config variants (recipe v0 & v1) (#3605) * add tests for recipe v0 polars update directly selecting source based on conda build config variant * add tests for recipe v0 polars update successfully reproduce polars issue * fix URL reference error * fix polars recipes --------- Co-authored-by: Matthew R. Becker --- conda_forge_tick/update_recipe/version.py | 1 + tests/test_migrators.py | 66 +++++++++-- tests/test_v1_yaml/version_polars.yaml | 105 ++++++++++++++++++ .../test_v1_yaml/version_polars_correct.yaml | 105 ++++++++++++++++++ .../test_v1_yaml/version_polars_variants.yaml | 17 +++ tests/test_version_migrator.py | 11 ++ .../test_yaml/version_polars_by_variant.yaml | 79 +++++++++++++ .../version_polars_by_variant_correct.yaml | 79 +++++++++++++ 8 files changed, 451 insertions(+), 12 deletions(-) create mode 100644 tests/test_v1_yaml/version_polars.yaml create mode 100644 tests/test_v1_yaml/version_polars_correct.yaml create mode 100644 tests/test_v1_yaml/version_polars_variants.yaml create mode 100644 tests/test_yaml/version_polars_by_variant.yaml create mode 100644 tests/test_yaml/version_polars_by_variant_correct.yaml diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 8b65fc988..5a1a42dfe 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -307,6 +307,7 @@ def _get_new_url_tmpl_and_hash( logger.info("initial rendered URL: %s", url) except jinja2.UndefinedError: logger.info("initial URL template does not render") + url = None pass if url != url_tmpl: diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 531071366..2302f1b06 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -1,3 +1,4 @@ +import itertools import json import os import re @@ -5,6 +6,7 @@ from pathlib import Path import pytest +import yaml from conda_forge_tick.contexts import ClonedFeedstockContext from conda_forge_tick.feedstock_parser import populate_feedstock_attributes @@ -471,27 +473,59 @@ def run_test_migration( should_filter: bool = False, make_body: bool = False, recipe_version: int = 0, + conda_build_config: str | None = None, ): tmpdir_p = Path(tmpdir) if mr_out: mr_out.update(bot_rerun=False) if recipe_version == 0: + if conda_build_config: + raise ValueError( + "conda_build_config is only supported for recipe version 1 in this test function" + ) tmpdir_p.joinpath("meta.yaml").write_text(inp) - recipe_dir = str(tmpdir_p) + recipe_dir_p = tmpdir_p elif recipe_version == 1: tmpdir_p.joinpath(".ci_support").mkdir() tmpdir_p.joinpath("recipe").mkdir() tmpdir_p.joinpath("recipe", "recipe.yaml").write_text(inp) - for target_platform in ("linux-64", "osx-arm64", "win-64"): - ( - tmpdir_p / ".ci_support" / f"{target_platform.replace('-', '_')}_.yaml" - ).write_text("target_platform:\n" f"- {target_platform}\n") - recipe_dir = str(tmpdir_p / "recipe") + build_variants = ( + yaml.safe_load(conda_build_config) if conda_build_config else {} + ) + + if "target_platform" not in build_variants: + build_variants["target_platform"] = ["linux-64", "osx-arm64", "win-64"] + + # move target_platform to the beginning of the keys + build_variants = { + "target_platform": build_variants.pop("target_platform"), + **build_variants, + } + + for assignment in itertools.product(*build_variants.values()): + assignment_map = dict(zip(build_variants.keys(), assignment)) + variant_name = "_".join( + str(value).replace("-", "_").replace("/", "") + for value in assignment_map.values() + ) + (tmpdir_p / ".ci_support" / f"{variant_name}_.yaml").write_text( + yaml.dump({k: [v] for k, v in assignment_map.items()}) + ) + + recipe_dir_p = tmpdir_p / "recipe" else: raise ValueError(f"Unsupported recipe version: {recipe_version}") + if conda_build_config: + conda_build_config_file = recipe_dir_p / "conda_build_config.yaml" + conda_build_config_file.write_text(conda_build_config) + else: + conda_build_config_file = None + + recipe_dir = str(recipe_dir_p) + # read the conda-forge.yml cf_yml_path = Path(tmpdir).parent / "conda-forge.yml" cf_yml = cf_yml_path.read_text() if cf_yml_path.exists() else "{}" @@ -499,7 +533,9 @@ def run_test_migration( # Load the meta.yaml (this is done in the graph) if recipe_version == 0: try: - name = parse_meta_yaml(inp)["package"]["name"] + name = parse_meta_yaml(inp, cbc_path=conda_build_config_file)["package"][ + "name" + ] except Exception: name = "blah" @@ -518,7 +554,9 @@ def run_test_migration( pmy.update(kwargs) else: try: - name = parse_recipe_yaml(inp)["package"]["name"] + name = parse_recipe_yaml(inp, cbc_path=conda_build_config_file)["package"][ + "name" + ] except Exception: name = "blah" @@ -720,7 +758,8 @@ def test_all_noarch(meta, is_all_noarch): "meta,is_all_noarch", [ ( - json.loads("""\ + json.loads( + """\ { "about": { "description": "NetworkX is a Python language software package for the creation,\\nmanipulation, and study of the structure, dynamics, and functions of complex\\nnetworks.", @@ -784,11 +823,13 @@ def test_all_noarch(meta, is_all_noarch): "sha256": "307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", "url": "https://pypi.org/packages/source/n/networkx/networkx-3.4.2.tar.gz" } - }"""), + }""" + ), True, ), ( - json.loads("""\ + json.loads( + """\ { "about": { "description": "This is a python extension ", @@ -857,7 +898,8 @@ def test_all_noarch(meta, is_all_noarch): "pytest" ] } - }"""), + }""" + ), False, ), ], diff --git a/tests/test_v1_yaml/version_polars.yaml b/tests/test_v1_yaml/version_polars.yaml new file mode 100644 index 000000000..bedc92c73 --- /dev/null +++ b/tests/test_v1_yaml/version_polars.yaml @@ -0,0 +1,105 @@ +context: + version: "1.17.1" + +# Note: This recipe is specifically designed to work well with the autotick bot. +# Also refer to https://github.com/conda-forge/rust-feedstock/blob/main/recipe/meta.yaml. +package: + name: ${{ polars_variant }} + version: ${{ version }} + +source: + - if: polars_variant == 'polars' + then: + url: https://pypi.org/packages/source/p/polars/polars-${{ version }}.tar.gz + sha256: 5a3dac3cb7cbe174d1fa898cba9afbede0c08e8728feeeab515554d762127019 + - if: polars_variant == 'polars-lts-cpu' + then: + url: https://pypi.org/packages/source/p/polars-lts-cpu/polars_lts_cpu-${{ version }}.tar.gz + sha256: d2717d17cd764223ea01e35ada2e3235327bc08040ecd41c71c803c7aad874fb + - if: polars_variant == 'polars-u64-idx' + then: + url: https://pypi.org/packages/source/p/polars-u64-idx/polars_u64_idx-${{ version }}.tar.gz + sha256: 5b47e993d3a73e40f674bc856dbac0e93eaf26c10bc7b1d6768f71faa6e023fe + +build: + number: 0 + +requirements: + build: + - if: build_platform != target_platform + then: + - python + # there is no cross-python for linux-64 -> win-64 + - if: target_platform != 'win-64' + then: cross-python_${{ target_platform }} + - crossenv + - maturin >=1.3.2,<2 + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + # clang_win-64 already adds all required run_exports for the windows build + - if: not (build_platform != target_platform and target_platform == "win-64") + then: + - ${{ stdlib('c') }} + else: + - cargo-feature + - if: build_platform == 'win-64' + then: + - posix + - ${{ compiler('rust') }} + - cmake + - if: unix + then: + - make + - cargo-bundle-licenses + host: + - python + - pip + - maturin >=1.3.2,<2 + - if: build_platform != target_platform and target_platform == "win-64" + then: + - mingw-w64-ucrt-x86_64-headers-git + run: + - python + - numpy >=1.16.0 + - if: python<3.11 + then: + - typing_extensions >=4.0.0 + - if: python >=3.10 + then: + - packaging + +tests: + - python: + pip_check: true + imports: + - polars + - script: + - python -c "from polars import DataFrame" + - package_contents: + site_packages: + - polars/polars.abi3.so + - polars/dataframe/__init__.py + - ${{ polars_variant | replace('-', '_') }}-${{ version }}.dist-info/METADATA + +about: + homepage: https://github.com/pola-rs/polars + license: MIT + license_file: + - LICENSE + - THIRDPARTY.yml + summary: Dataframes powered by a multithreaded, vectorized query engine, written in Rust + description: Polars is a DataFrame interface on top of an OLAP Query Engine implemented in Rust using Apache Arrow Columnar Format as the memory model. + documentation: https://docs.pola.rs + repository: https://github.com/pola-rs/polars + +extra: + recipe-maintainers: + - borchero + - Maxyme + - timkpaine + - ritchie46 + - sugatoray + - xhochy + - dhirschfeld + - pavelzw + - '0xbe7a' diff --git a/tests/test_v1_yaml/version_polars_correct.yaml b/tests/test_v1_yaml/version_polars_correct.yaml new file mode 100644 index 000000000..e35dcdda4 --- /dev/null +++ b/tests/test_v1_yaml/version_polars_correct.yaml @@ -0,0 +1,105 @@ +context: + version: "1.20.0" + +# Note: This recipe is specifically designed to work well with the autotick bot. +# Also refer to https://github.com/conda-forge/rust-feedstock/blob/main/recipe/meta.yaml. +package: + name: ${{ polars_variant }} + version: ${{ version }} + +source: + - if: polars_variant == 'polars' + then: + url: https://pypi.org/packages/source/p/polars/polars-${{ version }}.tar.gz + sha256: e8e9e3156fae02b58e276e5f2c16a5907a79b38617a9e2d731b533d87798f451 + - if: polars_variant == 'polars-lts-cpu' + then: + url: https://pypi.org/packages/source/p/polars-lts-cpu/polars_lts_cpu-${{ version }}.tar.gz + sha256: f8770fe1a752f60828ec73e6215c7dadcb2badd1f34dcb1def7a0f4ca0ac36f8 + - if: polars_variant == 'polars-u64-idx' + then: + url: https://pypi.org/packages/source/p/polars-u64-idx/polars_u64_idx-${{ version }}.tar.gz + sha256: a92fadacf59776bef2d777f99345c4d089cf4f8b3fd61f5728087bab27a46a75 + +build: + number: 0 + +requirements: + build: + - if: build_platform != target_platform + then: + - python + # there is no cross-python for linux-64 -> win-64 + - if: target_platform != 'win-64' + then: cross-python_${{ target_platform }} + - crossenv + - maturin >=1.3.2,<2 + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + # clang_win-64 already adds all required run_exports for the windows build + - if: not (build_platform != target_platform and target_platform == "win-64") + then: + - ${{ stdlib('c') }} + else: + - cargo-feature + - if: build_platform == 'win-64' + then: + - posix + - ${{ compiler('rust') }} + - cmake + - if: unix + then: + - make + - cargo-bundle-licenses + host: + - python + - pip + - maturin >=1.3.2,<2 + - if: build_platform != target_platform and target_platform == "win-64" + then: + - mingw-w64-ucrt-x86_64-headers-git + run: + - python + - numpy >=1.16.0 + - if: python<3.11 + then: + - typing_extensions >=4.0.0 + - if: python >=3.10 + then: + - packaging + +tests: + - python: + pip_check: true + imports: + - polars + - script: + - python -c "from polars import DataFrame" + - package_contents: + site_packages: + - polars/polars.abi3.so + - polars/dataframe/__init__.py + - ${{ polars_variant | replace('-', '_') }}-${{ version }}.dist-info/METADATA + +about: + homepage: https://github.com/pola-rs/polars + license: MIT + license_file: + - LICENSE + - THIRDPARTY.yml + summary: Dataframes powered by a multithreaded, vectorized query engine, written in Rust + description: Polars is a DataFrame interface on top of an OLAP Query Engine implemented in Rust using Apache Arrow Columnar Format as the memory model. + documentation: https://docs.pola.rs + repository: https://github.com/pola-rs/polars + +extra: + recipe-maintainers: + - borchero + - Maxyme + - timkpaine + - ritchie46 + - sugatoray + - xhochy + - dhirschfeld + - pavelzw + - '0xbe7a' diff --git a/tests/test_v1_yaml/version_polars_variants.yaml b/tests/test_v1_yaml/version_polars_variants.yaml new file mode 100644 index 000000000..ff14bbf38 --- /dev/null +++ b/tests/test_v1_yaml/version_polars_variants.yaml @@ -0,0 +1,17 @@ +channel_sources: + - conda-forge/label/rust_dev,conda-forge/label/cargo-patch-dev,conda-forge +channel_targets: + - conda-forge main +c_compiler: + - clang # [win] +c_compiler_version: + - 17 # [win] +cxx_compiler: + - clangxx # [win] +cxx_compiler_version: + - 17 # [win] + +polars_variant: + - polars # [not ppc64le] + - polars-lts-cpu + - polars-u64-idx # [not ppc64le] diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 7af900a01..5d2040fd0 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -81,6 +81,8 @@ "1.1.0", marks=pytest.mark.xfail(reason=VARIANT_SOURCES_NOT_IMPLEMENTED), ), + # use conda build config variants directly to select source + ("polars_by_variant", "1.20.0"), # upstream is not available # ("mumps", "5.2.1"), # ("cb3multi", "6.0.0"), @@ -127,6 +129,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): ("event_stream", "1.6.3"), ("selshaurl", "3.7.0"), ("libssh", "0.11.1"), + ("polars", "1.20.0"), ], ) @flaky @@ -139,6 +142,13 @@ def test_version_up_v1(case, new_ver, tmpdir, caplog): in_yaml = (YAML_V1_PATH / f"version_{case}.yaml").read_text() out_yaml = (YAML_V1_PATH / f"version_{case}_correct.yaml").read_text() + try: + conda_build_config = ( + YAML_V1_PATH / f"version_{case}_variants.yaml" + ).read_text() + except FileNotFoundError: + conda_build_config = None + kwargs = {"new_version": new_ver} if case == "sha1": kwargs["hash_type"] = "sha1" @@ -156,6 +166,7 @@ def test_version_up_v1(case, new_ver, tmpdir, caplog): }, tmpdir=tmpdir, recipe_version=1, + conda_build_config=conda_build_config, ) diff --git a/tests/test_yaml/version_polars_by_variant.yaml b/tests/test_yaml/version_polars_by_variant.yaml new file mode 100644 index 000000000..fd6a339ed --- /dev/null +++ b/tests/test_yaml/version_polars_by_variant.yaml @@ -0,0 +1,79 @@ +{% set version = "1.17.1" %} + +# Note: This recipe is specifically designed to work well with the autotick bot. +# Also refer to https://github.com/conda-forge/rust-feedstock/blob/main/recipe/meta.yaml. +package: + name: {{ polars_variant }} + version: {{ version }} + +source: + - url: https://pypi.org/packages/source/p/polars/polars-{{ version }}.tar.gz # [polars_variant == "polars"] + sha256: 5a3dac3cb7cbe174d1fa898cba9afbede0c08e8728feeeab515554d762127019 # [polars_variant == "polars"] + - url: https://pypi.org/packages/source/p/polars-lts-cpu/polars_lts_cpu-{{ version }}.tar.gz # [polars_variant == "polars-lts-cpu"] + sha256: d2717d17cd764223ea01e35ada2e3235327bc08040ecd41c71c803c7aad874fb # [polars_variant == "polars-lts-cpu"] + - url: https://pypi.org/packages/source/p/polars-u64-idx/polars_u64_idx-{{ version }}.tar.gz # [polars_variant == "polars-u64-idx"] + sha256: 5b47e993d3a73e40f674bc856dbac0e93eaf26c10bc7b1d6768f71faa6e023fe # [polars_variant == "polars-u64-idx"] + patches: + - 0001-Patch-libz-ng-sys.patch # [target_platform == "win-64"] + + +build: + number: 0 + +requirements: + build: + - python # [build_platform != target_platform] + # there is no cross-python for linux-64 -> win-64 + - cross-python_{{ target_platform }} # [build_platform != target_platform and not target_platform == "win-64"] + - crossenv # [build_platform != target_platform] + - maturin >=1.3.2,<2 # [build_platform != target_platform] + - {{ compiler('c') }} + - {{ compiler('cxx') }} # [win] + # clang_win-64 already adds all required run_exports for the windows build + - {{ stdlib("c") }} # [not (build_platform != target_platform and target_platform == "win-64")] + - {{ compiler('rust') }} + - posix # [build_platform == "win-64"] + - cmake + - make # [unix] + - cargo-feature # [build_platform != target_platform and target_platform == "win-64"] + - cargo-patch # [build_platform != target_platform and target_platform == "win-64"] + - cargo-bundle-licenses + host: + - python + - pip + - maturin >=1.3.2,<2 + run: + - python + - numpy >=1.16.0 + - backports.zoneinfo # [py<39] + - typing_extensions >=4.0.0 # [py<311] + - packaging # [py>=310] +test: + imports: + - polars + commands: + - pip check + - python -c "from polars import DataFrame" + requires: + - pip +about: + home: https://github.com/pola-rs/polars + license: MIT + license_family: MIT + license_file: + - LICENSE + - THIRDPARTY.yml + summary: Polars is a blazingly fast DataFrames library implemented in Rust using Apache Arrow(2) as memory model. + doc_url: https://pola-rs.github.io/polars-book/user-guide/index.html + dev_url: https://github.com/pola-rs/polars +extra: + recipe-maintainers: + - borchero + - Maxyme + - timkpaine + - ritchie46 + - sugatoray + - xhochy + - dhirschfeld + - pavelzw + - '0xbe7a' diff --git a/tests/test_yaml/version_polars_by_variant_correct.yaml b/tests/test_yaml/version_polars_by_variant_correct.yaml new file mode 100644 index 000000000..1c04926eb --- /dev/null +++ b/tests/test_yaml/version_polars_by_variant_correct.yaml @@ -0,0 +1,79 @@ +{% set version = "1.20.0" %} + +# Note: This recipe is specifically designed to work well with the autotick bot. +# Also refer to https://github.com/conda-forge/rust-feedstock/blob/main/recipe/meta.yaml. +package: + name: {{ polars_variant }} + version: {{ version }} + +source: + - url: https://pypi.org/packages/source/p/polars/polars-{{ version }}.tar.gz # [polars_variant == "polars"] + sha256: e8e9e3156fae02b58e276e5f2c16a5907a79b38617a9e2d731b533d87798f451 # [polars_variant == "polars"] + - url: https://pypi.org/packages/source/p/polars-lts-cpu/polars_lts_cpu-{{ version }}.tar.gz # [polars_variant == "polars-lts-cpu"] + sha256: f8770fe1a752f60828ec73e6215c7dadcb2badd1f34dcb1def7a0f4ca0ac36f8 # [polars_variant == "polars-lts-cpu"] + - url: https://pypi.org/packages/source/p/polars-u64-idx/polars_u64_idx-{{ version }}.tar.gz # [polars_variant == "polars-u64-idx"] + sha256: a92fadacf59776bef2d777f99345c4d089cf4f8b3fd61f5728087bab27a46a75 # [polars_variant == "polars-u64-idx"] + patches: + - 0001-Patch-libz-ng-sys.patch # [target_platform == "win-64"] + + +build: + number: 0 + +requirements: + build: + - python # [build_platform != target_platform] + # there is no cross-python for linux-64 -> win-64 + - cross-python_{{ target_platform }} # [build_platform != target_platform and not target_platform == "win-64"] + - crossenv # [build_platform != target_platform] + - maturin >=1.3.2,<2 # [build_platform != target_platform] + - {{ compiler('c') }} + - {{ compiler('cxx') }} # [win] + # clang_win-64 already adds all required run_exports for the windows build + - {{ stdlib("c") }} # [not (build_platform != target_platform and target_platform == "win-64")] + - {{ compiler('rust') }} + - posix # [build_platform == "win-64"] + - cmake + - make # [unix] + - cargo-feature # [build_platform != target_platform and target_platform == "win-64"] + - cargo-patch # [build_platform != target_platform and target_platform == "win-64"] + - cargo-bundle-licenses + host: + - python + - pip + - maturin >=1.3.2,<2 + run: + - python + - numpy >=1.16.0 + - backports.zoneinfo # [py<39] + - typing_extensions >=4.0.0 # [py<311] + - packaging # [py>=310] +test: + imports: + - polars + commands: + - pip check + - python -c "from polars import DataFrame" + requires: + - pip +about: + home: https://github.com/pola-rs/polars + license: MIT + license_family: MIT + license_file: + - LICENSE + - THIRDPARTY.yml + summary: Polars is a blazingly fast DataFrames library implemented in Rust using Apache Arrow(2) as memory model. + doc_url: https://pola-rs.github.io/polars-book/user-guide/index.html + dev_url: https://github.com/pola-rs/polars +extra: + recipe-maintainers: + - borchero + - Maxyme + - timkpaine + - ritchie46 + - sugatoray + - xhochy + - dhirschfeld + - pavelzw + - '0xbe7a'