From 658b1b8b3fd528d73b5d4f0ab0fb9cb175fbe28a Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Fri, 8 Dec 2023 08:45:11 +0100 Subject: [PATCH] feat: added python version option to yaml files and tests --- README.md | 13 ++++-- build_wheels.py | 103 +++++++++++++++++++++++++++++++++-------- exclude_list.yaml | 1 + include_list.yaml | 1 + test/test_list.yaml | 107 +++++++++++++++++++++++++++++++++++++++++++ test_build_wheels.py | 107 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 307 insertions(+), 25 deletions(-) create mode 100644 test/test_list.yaml create mode 100644 test_build_wheels.py diff --git a/README.md b/README.md index b910a2d..75ebce5 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,9 @@ The opposite logic of exclude_list is handled by the function itself, which mean For every `package_name` there are options: * `version` - - supports all logic operators defined by PEP508 for versions (<, >, !=, etc.) + - supports all logic operators defined by [PEP508](https://peps.python.org/pep-0508/) for versions (<, >, !=, etc.) * `platform` +* `python` which could be a string or a list of strings. @@ -45,18 +46,20 @@ exclude_list template: - package_name: '' version: '' / ['', ''] # optional platform: '' / ['', '', ''] # optional + python: '' / ['', '', ''] # optional -The syntax can be converted into a sentence: "From assembled **main requirements** exclude `package_name` with `version` on `platform`". +The syntax can be converted into a sentence: "From assembled **main requirements** exclude `package_name` with `version` on `platform` for `python` version". example: - package_name: 'pyserial' version: ['>=3.3', '<3.6'] platform: ['win32', 'linux', 'darwin'] + python: '>=3.9' -This would mean: "From assembled **main requirements** exclude `pyserial` with version `>=3.3` and `<3.6` on platform `win32`, `linux`, `darwin`". +This would mean: "From assembled **main requirements** exclude `pyserial` with version `>=3.3` and `<3.6` on platform `win32`, `linux`, `darwin` for `python` version `>=3.9`". -From the example above is clear that the `platform` could be left out (because all main platforms are specified) so the options `platform` or `version` are optional, one of them or both can be not specified and the key can be erased. When only `package_name` is given the package will be excluded from **main requirements**. +From the example above is clear that the `platform` could be left out (because all main platforms are specified) so the options `platform` or `version` or `python` are optional, one of them or both can be not specified and the key can be erased. When only `package_name` is given the package will be excluded from **main requirements**. ### include_list.yaml @@ -64,7 +67,7 @@ File for additional Python packages to the **main requirements** list. Built sep This YAML file uses the same mechanism such as **exclude_list** but without the opposite logic. -The syntax can be also converted into a sentence: "For assembled **main requirements** additionally include `package_name` with `version` on `platform`". +The syntax can be also converted into a sentence: "For assembled **main requirements** additionally include `package_name` with `version` on `platform` for `python` version". ### build_requirements.txt diff --git a/build_wheels.py b/build_wheels.py index a2e51ad..52bc310 100644 --- a/build_wheels.py +++ b/build_wheels.py @@ -195,19 +195,27 @@ def assemble_requirements(idf_branches: List[str], idf_constraints: List[str], m # --- exclude_list and include_list --- -def _change_specifier_logic(specifier: str) -> str: +def _change_specifier_logic(spec_with_text: str) -> tuple: """Change specifier logic to opposite - - e.g. "<3" will be ">3" + e.g. "<3" will be ">3" + - return (new_specifier, text_after, old_specifier) """ - new_spec = specifier + pattern = re.compile(r'(===|==|!=|~=|<=?|>=?|===?)\s*(.*)') + str_match = pattern.findall(spec_with_text) + specifier, text = str_match[0] replacements = (('<', '>'), ('>', '<'), ('!', '='), + ('~', '!'), ('==', '!=')) for old, new in replacements: - if old in specifier and '===' not in specifier: + if old in specifier and specifier != '===': new_spec = specifier.replace(old, new) - return new_spec + break + elif specifier == '===': + new_spec = '===' + break + return (new_spec, text, specifier) def yaml_to_requirement(yaml_file:str, exclude: bool = False) -> set: @@ -227,23 +235,30 @@ def yaml_to_requirement(yaml_file:str, exclude: bool = False) -> set: if 'version' in package: if not isinstance(package['version'], list): + new_spec, ver, old_spec = _change_specifier_logic(package['version']) requirement_str_list.append( - _change_specifier_logic(package['version']) if exclude else package['version'] + f'{new_spec}{ver}' if exclude else f'{old_spec}{ver}' ) else: - version_list = ( - [f'{_change_specifier_logic(ver)}' if exclude else f'{ver}' for ver in package['version']] - ) + version_list = [] + for elem in package['version']: + new_spec, ver, old_spec = _change_specifier_logic(elem) + if exclude: + version_list.append(f'{new_spec}{ver}') + else: + version_list.append(f'{old_spec}{ver}') requirement_str_list.append(','.join(version_list)) + if 'platform' in package or 'python' in package: + requirement_str_list.append('; ') if 'platform' in package and 'version' not in package: if not isinstance(package['platform'], list): requirement_str_list.append(( - f"; sys_platform != '{package['platform']}'" if exclude - else f"; sys_platform == '{package['platform']}'" + f"sys_platform != '{package['platform']}'" if exclude + else f"sys_platform == '{package['platform']}'" )) else: @@ -252,28 +267,76 @@ def yaml_to_requirement(yaml_file:str, exclude: bool = False) -> set: else f"sys_platform == '{plf}'" for plf in package['platform']] ) - requirement_str_list.append('; ' + ' or '.join(platform_list)) + requirement_str_list.append(' or '.join(platform_list)) + if ('platform' in package or 'python' in package) and 'version' in package and exclude: + requirement_old_str_list = [f"{package['package_name']}; "] if 'platform' in package and 'version' in package: - requirement_old_str_list = [f"{package['package_name']}"] - if not isinstance(package['platform'], list): - requirement_str_list.append(f"; sys_platform == '{package['platform']}'") + requirement_str_list.append(f"sys_platform == '{package['platform']}'") if exclude: - requirement_old_str_list.append(f"; sys_platform != '{package['platform']}'") - requirements_set.add(Requirement(''.join(requirement_old_str_list))) + requirement_old_str_list.append(f"sys_platform != '{package['platform']}'") else: platform_list = [f"sys_platform == '{plf}'" for plf in package['platform']] - requirement_str_list.append('; ' + ' or '.join(platform_list)) + requirement_str_list.append(' or '.join(platform_list)) if exclude: platform_list_old = [f"sys_platform != '{plf}'" for plf in package['platform']] - requirement_old_str_list.append('; ' + ' or '.join(platform_list_old)) - requirements_set.add(Requirement(''.join(requirement_old_str_list))) + requirement_old_str_list.append(' or '.join(platform_list_old)) + + if 'platform' in package and 'python' in package: + requirement_str_list.append(' and ') + + if ('platform' in package and 'python' in package) and 'version' in package and exclude: + requirement_old_str_list.append(' and ') + + if 'python' in package and 'version' not in package: + if not isinstance(package['python'], list): + new_spec, text_after, old_spec = _change_specifier_logic(package['python']) + requirement_str_list.append(( + f"python_version {new_spec} '{text_after}'" if exclude + else f"python_version {old_spec} '{text_after}'" + )) + + else: + python_list = [] + for elem in package['python']: + new_spec, text_after, old_spec = _change_specifier_logic(elem) + if exclude: + python_list.append(f"python_version {new_spec} '{text_after}'") + else: + python_list.append(f"python_version {old_spec} '{text_after}'") + + requirement_str_list.append(' and '.join(python_list)) + + if 'python' in package and 'version' in package: + + if not isinstance(package['python'], list): + new_spec, text_after, old_spec = _change_specifier_logic(package['python']) + requirement_str_list.append(f"python_version {old_spec} '{text_after}'") + + if exclude: + requirement_old_str_list.append(f"python_version {new_spec} '{text_after}'") + + else: + python_list = [] + python_list_old = [] + for elem in package['python']: + new_spec, text_after, old_spec = _change_specifier_logic(elem) + + python_list.append(f"python_version {old_spec} '{text_after}'") + if exclude: + python_list_old.append(f"python_version {new_spec} '{text_after}'") + requirement_str_list.append('' + ' and '.join(python_list)) + + if exclude: + requirement_old_str_list.append(' and '.join(python_list_old)) + if ('platform' in package or 'python' in package) and 'version' in package and exclude: + requirements_set.add(Requirement(''.join(requirement_old_str_list))) requirements_set.add(Requirement(''.join(requirement_str_list))) return requirements_set diff --git a/exclude_list.yaml b/exclude_list.yaml index 3a45cae..f7b07ce 100644 --- a/exclude_list.yaml +++ b/exclude_list.yaml @@ -5,6 +5,7 @@ #- package_name: '' # version: '' / ['', ''] # optional # platform: '' / ['', '', ''] # optional +# python: '' / ['', '', ''] # optional # dbus-python can not be build on Windows - package_name: 'dbus-python' diff --git a/include_list.yaml b/include_list.yaml index dca4bef..eed49fa 100644 --- a/include_list.yaml +++ b/include_list.yaml @@ -5,6 +5,7 @@ #- package_name: '' # version: '' / ['', ''] # optional # platform: '' / ['', '', ''] # optional +# python: '' / ['', '', ''] # optional # lxml is missing as a dependency of pytest-embedded # https://github.com/espressif/pytest-embedded/blob/main/pyproject.toml diff --git a/test/test_list.yaml b/test/test_list.yaml new file mode 100644 index 0000000..3ca6564 --- /dev/null +++ b/test/test_list.yaml @@ -0,0 +1,107 @@ +# List of Python packages for testing +# platform +- package_name: 'platform' + platform: 'win32' + +- package_name: 'platform' + platform: ['win32', 'linux'] +# version +- package_name: 'version' + version: '<42' + +- package_name: 'version' + version: ['<42', '>50'] +# Python +- package_name: 'python' + python: '>3.10' + +- package_name: 'python' + python: ['>3.10', '!=3.8'] +# version and platform +- package_name: 'version-platform' + version: '<=0.9.0.2' + platform: 'win32' + +- package_name: 'version-platform' + version: ['<=0.9.0.2', '>0.9.1'] + platform: 'win32' + +- package_name: 'version-platform' + version: '<=0.9.0.2' + platform: ['win32', 'linux'] + +- package_name: 'version-platform' + version: ['<=0.9.0.2', '>0.9.1'] + platform: ['win32', 'linux'] +# version and Python +- package_name: 'version-python' + version: '<=0.9.0.2' + python: '<3.8' + +- package_name: 'version-python' + version: ['<=0.9.0.2', '>0.9.1'] + python: '<3.8' + +- package_name: 'version-python' + version: '<=0.9.0.2' + python: ['<3.8', '>3.11'] + +- package_name: 'version-python' + version: ['<=0.9.0.2', '>0.9.1'] + python: ['<3.8', '>3.11'] +# platform and Python +- package_name: 'platform-python' + platform: 'win32' + python: '<3.8' + +- package_name: 'platform-python' + platform: ['win32', 'linux'] + python: '<3.8' + +- package_name: 'platform-python' + platform: 'win32' + python: ['<3.8', '>3.11'] + +- package_name: 'platform-python' + platform: ['win32', 'linux'] + python: ['<3.8', '>3.11'] +# version and platform and Python +- package_name: 'version-platform-python' + version: '<=0.9.0.2' + platform: 'win32' + python: '<3.8' + +- package_name: 'version-platform-python' + version: ['<=0.9.0.2', '>0.9.1'] + platform: 'win32' + python: '<3.8' + +- package_name: 'version-platform-python' + version: ['<=0.9.0.2', '>0.9.1'] + platform: ['win32', 'linux'] + python: '<3.8' + +- package_name: 'version-platform-python' + version: '<=0.9.0.2' + platform: ['win32', 'linux'] + python: '<3.8' + +- package_name: 'version-platform-python' + version: '<=0.9.0.2' + platform: 'win32' + python: ['<3.8', '>3.11'] + +- package_name: 'version-platform-python' + version: ['<=0.9.0.2', '>0.9.1'] + platform: 'win32' + python: ['<3.8', '>3.11'] + +- package_name: 'version-platform-python' + version: ['<=0.9.0.2', '>0.9.1'] + platform: ['win32', 'linux'] + python: ['<3.8', '>3.11'] + +- package_name: 'version-platform-python' + version: '<=0.9.0.2' + platform: ['win32', 'linux'] + python: ['<3.8', '>3.11'] diff --git a/test_build_wheels.py b/test_build_wheels.py new file mode 100644 index 0000000..0a57829 --- /dev/null +++ b/test_build_wheels.py @@ -0,0 +1,107 @@ +# ruff: noqa: E501 +# line too long skip in ruff for whole file (formatting would be worst than long lines) +import unittest + +from packaging.requirements import Requirement + +from build_wheels import _change_specifier_logic +from build_wheels import yaml_to_requirement + + +class TestYAMLtoRequirement(unittest.TestCase): + + def test_change_specifier_logic(self): + version_with_specifier = (('>0.9.0.2', '<0.9.0.2'), + ('<0.9.0.2', '>0.9.0.2'), + ('==0.9.0.2', '!=0.9.0.2'), + ('>=0.9.0.2', '<=0.9.0.2'), + ('<=0.9.0.2', '>=0.9.0.2'), + ('!=0.9.0.2', '==0.9.0.2'), + ('===0.9.0.2', '===0.9.0.2'), + ) + + for case in version_with_specifier: + self.assertEqual(f'{_change_specifier_logic(case[0])[0]}{_change_specifier_logic(case[0])[1]}', case[1]) + + def test_yaml_to_requirement(self): + test_requirements = {Requirement("platform;sys_platform == 'win32'"), + Requirement("platform;sys_platform == 'win32' or sys_platform == 'linux'"), + Requirement('version<42'), + Requirement('version<42,>50'), + Requirement("python;python_version > '3.10'"), + Requirement("python;python_version > '3.10' and python_version != '3.8'"), + Requirement("version-platform<=0.9.0.2;sys_platform == 'win32'"), + Requirement("version-platform<=0.9.0.2,>0.9.1;sys_platform == 'win32'"), + Requirement("version-platform<=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux'"), + Requirement("version-platform<=0.9.0.2,>0.9.1;sys_platform == 'win32' or sys_platform == 'linux'"), + Requirement("version-python<=0.9.0.2;python_version < '3.8'"), + Requirement("version-python<=0.9.0.2,>0.9.1;python_version < '3.8'"), + Requirement("version-python<=0.9.0.2;python_version < '3.8' and python_version > '3.11'"), + Requirement("version-python<=0.9.0.2,>0.9.1;python_version < '3.8' and python_version > '3.11'"), + Requirement("platform-python;sys_platform == 'win32' and python_version < '3.8'"), + Requirement("platform-python;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'"), + Requirement("platform-python;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'"), + Requirement("platform-python;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'"), + Requirement("version-platform-python<=0.9.0.2;sys_platform == 'win32' and python_version < '3.8'"), + Requirement("version-platform-python<=0.9.0.2,>0.9.1;sys_platform == 'win32' and python_version < '3.8'"), + Requirement("version-platform-python<=0.9.0.2,>0.9.1;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'"), + Requirement("version-platform-python<=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'"), + Requirement("version-platform-python<=0.9.0.2;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'"), + Requirement("version-platform-python<=0.9.0.2,>0.9.1;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'"), + Requirement("version-platform-python<=0.9.0.2,>0.9.1;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'"), + Requirement("version-platform-python<=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'"), + } + + self.assertEqual(yaml_to_requirement('test/test_list.yaml'), test_requirements) + + + def test_yaml_to_requirement_exclude(self): + test_requirements_exclude = {Requirement("platform;sys_platform != 'win32'"), + Requirement("platform;sys_platform != 'win32' or sys_platform != 'linux'"), + Requirement('version>42'), + Requirement('version>42,<50'), + Requirement("python;python_version < '3.10'"), + Requirement("python;python_version < '3.10' and python_version == '3.8'"), + Requirement("version-platform>=0.9.0.2;sys_platform == 'win32'"), + Requirement("version-platform;sys_platform != 'win32'"), + Requirement("version-platform>=0.9.0.2,<0.9.1;sys_platform == 'win32'"), + Requirement("version-platform;sys_platform != 'win32'"), + Requirement("version-platform>=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux'"), + Requirement("version-platform;sys_platform != 'win32' or sys_platform != 'linux'"), + Requirement("version-platform>=0.9.0.2,<0.9.1;sys_platform == 'win32' or sys_platform == 'linux'"), + Requirement("version-platform;sys_platform != 'win32' or sys_platform != 'linux'"), + Requirement("version-python>=0.9.0.2;python_version < '3.8'"), + Requirement("version-python;python_version > '3.8'"), + Requirement("version-python>=0.9.0.2,<0.9.1;python_version < '3.8'"), + Requirement("version-python;python_version > '3.8'"), + Requirement("version-python>=0.9.0.2;python_version < '3.8' and python_version > '3.11'"), + Requirement("version-python;python_version > '3.8' and python_version < '3.11'"), + Requirement("version-python>=0.9.0.2,<0.9.1;python_version < '3.8' and python_version > '3.11'"), + Requirement("version-python;python_version > '3.8' and python_version < '3.11'"), + Requirement("platform-python;sys_platform != 'win32' and python_version > '3.8'"), + Requirement("platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8'"), + Requirement("platform-python;sys_platform != 'win32' and python_version > '3.8' and python_version < '3.11'"), + Requirement("platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8' and python_version < '3.11'"), + Requirement("version-platform-python>=0.9.0.2;sys_platform == 'win32' and python_version < '3.8'"), + Requirement("version-platform-python;sys_platform != 'win32' and python_version > '3.8'"), + Requirement("version-platform-python>=0.9.0.2,<0.9.1;sys_platform == 'win32' and python_version < '3.8'"), + Requirement("version-platform-python;sys_platform != 'win32' and python_version > '3.8'"), + Requirement("version-platform-python>=0.9.0.2,<0.9.1;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'"), + Requirement("version-platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8'"), + Requirement("version-platform-python>=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'"), + Requirement("version-platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8'"), + Requirement("version-platform-python>=0.9.0.2;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'"), + Requirement("version-platform-python;sys_platform != 'win32' and python_version > '3.8' and python_version < '3.11'"), + Requirement("version-platform-python>=0.9.0.2,<0.9.1;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'"), + Requirement("version-platform-python;sys_platform != 'win32' and python_version > '3.8' and python_version < '3.11'"), + Requirement("version-platform-python>=0.9.0.2,<0.9.1;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'"), + Requirement("version-platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8' and python_version < '3.11'"), + Requirement("version-platform-python>=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'"), + Requirement("version-platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8' and python_version < '3.11'"), + } + + self.assertEqual(yaml_to_requirement('test/test_list.yaml', exclude=True), test_requirements_exclude) + + +if __name__ == '__main__': + unittest.main()