From 5122148f4ccb8cb5fd92f9c4950b8d6cb1ac73f9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 20 Aug 2024 19:16:12 +0000 Subject: [PATCH] drop python req from pip-build-install, move to pip-build-install, add tests There are lots of changes here, centered around removing all occurences of SETUPTOOLS_SCM_PRETEND_VERSION. That seems to be not necessary now, at least with the current pip-build-install workflow. * do not require 'python' setting in pip-build-install. Default to using the target of the 'python3' link. If not present, user has to supply it. Also use contextdir rather than subpkgdir so it can be used from "main" packages also. * replace any packages that had SETUPTOOLS_SCM_PRETEND_VERSION with py/pip-build-install. * Add tests to touched packages * Drop 'python3' runtime dep from touched packages. * Fix uninstallable py3-gcloud-aio-storage which conflicted with its dependency py3-gcloud-aio-auth as they both provided usr/lib/python3.12/site-packages/gcloud/py.typed * Fix the provides in py3-maturin-bin packages to provide py3-maturin-bin rather than just py3-maturin. That also fixes py3-pendulum which was failing to build because it actually needs 'maturin' executable but that was not being provided by the 'py3-maturin' dep. * Fix pendulum's dependency that had py3-pytzdata, it's deps list 'tzdata' * Drop the now-unused py3-pytzdata Note the 'python3' shell snippet in py3-rich and others is painful. See https://github.com/wolfi-dev/os/issues/26818 for more context. --- pipelines/py/pip-build-install.yaml | 12 +++++-- py3-antlr4-python3-runtime.yaml | 12 ++----- py3-cppy.yaml | 23 +++++------- py3-gcloud-aio-auth.yaml | 25 ++++++++----- py3-gcloud-aio-storage.yaml | 25 ++++++++----- py3-maturin.yaml | 14 +++++--- py3-pendulum.yaml | 33 +++++++++++------ py3-psycopg.yaml | 20 +++++------ py3-pytzdata.yaml | 55 ----------------------------- py3-rich.yaml | 32 +++++++---------- py3-tzdata.yaml | 24 +++++++++++-- 11 files changed, 127 insertions(+), 148 deletions(-) delete mode 100644 py3-pytzdata.yaml diff --git a/pipelines/py/pip-build-install.yaml b/pipelines/py/pip-build-install.yaml index 13d38b63b5..7a107d2051 100644 --- a/pipelines/py/pip-build-install.yaml +++ b/pipelines/py/pip-build-install.yaml @@ -3,16 +3,24 @@ name: Build and install python package with pip inputs: python: description: which python to use - required: true + default: DEFAULT dest: description: the destination - default: ${{targets.subpkgdir}} + default: ${{targets.contextdir}} pipeline: - name: "pip build ${{inputs.python}}" runs: | export SOURCE_DATE_EPOCH=315532800 py=${{inputs.python}} + if [ "$py" = "DEFAULT" ]; then + py=python3 + p=$(command -v $py) || { echo "No $py command found"; exit 1; } + if [ -L "$p" ]; then + py=$(readlink -f "$p") || + { echo "failed 'readlink -f $p'"; exit 1; } + fi + fi pyver=$("$py" -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') sitepkgd=$("$py" -c 'import site; print(site.getsitepackages()[0])') sitepkgd=${sitepkgd#/} diff --git a/py3-antlr4-python3-runtime.yaml b/py3-antlr4-python3-runtime.yaml index 1b9831a10a..ebf9d20bc6 100644 --- a/py3-antlr4-python3-runtime.yaml +++ b/py3-antlr4-python3-runtime.yaml @@ -26,16 +26,8 @@ pipeline: repository: https://github.com/antlr/antlr4 tag: ${{package.version}} - - runs: | - cd runtime/Python3 - export SETUPTOOLS_SCM_PRETEND_VERSION=${{package.version}} - python3=$(readlink -f `which python3`) - $python3 -m pip install -U poetry - $python3 -m gpep517 build-wheel \ - --wheel-dir dist \ - --output-fd 3 3>&1 >&2 - $python3 -m installer -d "${{targets.destdir}}" \ - dist/*.whl + - uses: py/pip-build-install + working-directory: runtime/Python3 - uses: strip diff --git a/py3-cppy.yaml b/py3-cppy.yaml index fdf303aa02..7d0c6323a6 100644 --- a/py3-cppy.yaml +++ b/py3-cppy.yaml @@ -6,7 +6,7 @@ package: - license: BSD-3-Clause dependencies: runtime: - - python3 + - py3-setuptools environment: contents: @@ -30,22 +30,17 @@ pipeline: expected-commit: 13a67972ca9063951440c911270f204aa7fe631a tag: ${{package.version}} - - runs: | - git fetch --prune --unshallow - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - - name: Python Build - runs: | - export SETUPTOOLS_SCM_PRETEND_VERSION=${{package.version}} - python3 -m gpep517 build-wheel \ - --wheel-dir .dist \ - --output-fd 3 3>&1 >&2 - - - name: Python Install - runs: python3 -m installer -d "${{targets.destdir}}" .dist/cppy*.whl + - uses: py/pip-build-install - uses: strip +test: + pipeline: + - uses: python/import + with: + imports: | + import cppy + update: enabled: true github: diff --git a/py3-gcloud-aio-auth.yaml b/py3-gcloud-aio-auth.yaml index 91873be7c1..adcf4ea3c0 100644 --- a/py3-gcloud-aio-auth.yaml +++ b/py3-gcloud-aio-auth.yaml @@ -13,7 +13,6 @@ package: - py3-chardet - py3-cryptography - py3-pyjwt - - python3 environment: contents: @@ -23,6 +22,7 @@ environment: - ca-certificates-bundle - poetry - py3-gpep517 + - py3-pip - py3-setuptools - py3-wheel - python3 @@ -35,19 +35,26 @@ pipeline: repository: https://github.com/talkiq/gcloud-aio tag: auth-${{package.version}} - - name: Python Build + - name: remove py.typed to avoid conflict with py3-gcloud-aio-* runs: | - cd auth - export SETUPTOOLS_SCM_PRETEND_VERSION=${{package.version}} - python3 -m gpep517 build-wheel \ - --wheel-dir dist \ - --output-fd 3 3>&1 >&2 + rm auth/gcloud/py.typed auth/gcloud/aio/py.typed - python3 -m installer -d "${{targets.destdir}}" \ - dist/*.whl + - uses: py/pip-build-install + working-directory: auth - uses: strip +test: + environment: + contents: + packages: + - openssl-provider-legacy # needed for 'from gcloud.aio import auth' + pipeline: + - uses: python/import + with: + imports: | + from gcloud.aio import auth + update: enabled: true github: diff --git a/py3-gcloud-aio-storage.yaml b/py3-gcloud-aio-storage.yaml index 7004ace769..a24c204c4b 100644 --- a/py3-gcloud-aio-storage.yaml +++ b/py3-gcloud-aio-storage.yaml @@ -12,7 +12,6 @@ package: - py3-gcloud-aio-auth - py3-asn1-modules - py3-rsa - - python3 environment: contents: @@ -22,6 +21,7 @@ environment: - ca-certificates-bundle - poetry - py3-gpep517 + - py3-pip - py3-setuptools - py3-wheel - python3 @@ -34,19 +34,26 @@ pipeline: repository: https://github.com/talkiq/gcloud-aio tag: storage-${{package.version}} - - name: Python Build + - name: remove py.typed to avoid conflict with py3-gcloud-aio-* runs: | - cd storage - export SETUPTOOLS_SCM_PRETEND_VERSION=${{package.version}} - python3 -m gpep517 build-wheel \ - --wheel-dir dist \ - --output-fd 3 3>&1 >&2 + rm storage/gcloud/py.typed storage/gcloud/aio/py.typed - python3 -m installer -d "${{targets.destdir}}" \ - dist/*.whl + - uses: py/pip-build-install + working-directory: storage - uses: strip +test: + environment: + contents: + packages: + - openssl-provider-legacy # needed for gcloud.aio.auth imports to work. + pipeline: + - uses: python/import + with: + imports: | + from gcloud.aio import storage + update: enabled: true github: diff --git a/py3-maturin.yaml b/py3-maturin.yaml index 237c7b8e0c..4be439ebed 100644 --- a/py3-maturin.yaml +++ b/py3-maturin.yaml @@ -55,6 +55,12 @@ subpackages: with: python: python${{range.key}} - uses: strip + test: + pipeline: + - uses: python/import + with: + python: python${{range.key}} + import: ${{vars.pypi-package}} - range: py-versions name: py${{range.key}}-${{vars.pypi-package}}-bin @@ -63,7 +69,7 @@ subpackages: runtime: - py${{range.key}}-${{vars.pypi-package}} provides: - - py3-${{vars.pypi-package}} + - py3-${{vars.pypi-package}}-bin provider-priority: ${{range.value}} pipeline: - runs: | @@ -71,10 +77,8 @@ subpackages: mv ${{targets.contextdir}}/../py${{range.key}}-${{vars.pypi-package}}/usr/bin ${{targets.contextdir}}/usr test: pipeline: - - uses: python/import - with: - python: python${{range.key}} - import: ${{vars.pypi-package}} + - runs: | + maturin --help - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. diff --git a/py3-pendulum.yaml b/py3-pendulum.yaml index 1ab0dea04d..d40750c057 100644 --- a/py3-pendulum.yaml +++ b/py3-pendulum.yaml @@ -8,9 +8,8 @@ package: dependencies: runtime: - py3-dateutil - - py3-pytzdata - py3-typing - - python3 + - py3-tzdata environment: contents: @@ -22,7 +21,7 @@ environment: - poetry - py3-gpep517 - py3-installer - - py3-maturin + - py3-maturin-bin - py3-pip - py3-setuptools - py3-wheel @@ -38,17 +37,29 @@ pipeline: tag: ${{package.version}} expected-commit: 0fcd10217af0469b3edda072f2b152d5273f3d58 - - runs: | - export SETUPTOOLS_SCM_PRETEND_VERSION=${{package.version}} - python3 -m gpep517 build-wheel \ - --wheel-dir dist \ - --output-fd 3 3>&1 >&2 - - python3 -m installer -d "${{targets.destdir}}" \ - dist/*.whl + - uses: py/pip-build-install - uses: strip +test: + pipeline: + - uses: python/import + with: + imports: | + import pendulum + - runs: | + python=$(set +x; m=/usr/bin/python3.*; set -- $m + [ $# -eq 1 -a -x "$1" ] && echo "$1" && exit 0 + echo "found $# matches to $m"; exit 1;) + # test verifies that timezone data is present. + $python -c ' + from pendulum import datetime + tor = datetime(2012, 1, 1, tz="America/Toronto") + van = datetime(2012, 1, 1, tz="America/Vancouver") + assert 3 == van.diff(tor).in_hours() + print("PASS: time zones check out") + ' + update: enabled: true github: diff --git a/py3-psycopg.yaml b/py3-psycopg.yaml index b16079463f..52b06cb03d 100644 --- a/py3-psycopg.yaml +++ b/py3-psycopg.yaml @@ -10,7 +10,6 @@ package: - libpq-16 - py3-typing-extensions - py3-tzdata - - python3 environment: contents: @@ -20,6 +19,7 @@ environment: - ca-certificates-bundle - poetry - py3-gpep517 + - py3-pip - py3-setuptools - py3-wheel - python3 @@ -32,16 +32,8 @@ pipeline: tag: ${{package.version}} expected-commit: bb47d3944d1c65d9baf83808696aba1b2dfed9af - - name: Python Build - runs: | - cd psycopg - export SETUPTOOLS_SCM_PRETEND_VERSION=${{package.version}} - python3 -m gpep517 build-wheel \ - --wheel-dir dist \ - --output-fd 3 3>&1 >&2 - - python3 -m installer -d "${{targets.destdir}}" \ - dist/*.whl + - uses: py/pip-build-install + working-directory: psycopg - uses: strip @@ -113,4 +105,8 @@ test: if __name__ == "__main__": main() EOF - python3 /tmp/test.py + + python=$(m=/usr/bin/python3.*; set +x; set -- $m + [ $# -eq 1 -a -x "$1" ] && echo "$1" && exit 0 + echo "found $# matches to $m"; exit 1;) + $python /tmp/test.py diff --git a/py3-pytzdata.yaml b/py3-pytzdata.yaml deleted file mode 100644 index 9a97876349..0000000000 --- a/py3-pytzdata.yaml +++ /dev/null @@ -1,55 +0,0 @@ -package: - name: py3-pytzdata - version: "2020.1" - epoch: 2 - description: The Olson timezone database for Python. - copyright: - - license: MIT - dependencies: - runtime: - - py3-installer - - python3 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - poetry - - py3-gpep517 - - py3-installer - - py3-pip - - py3-setuptools - - py3-wheel - - python3 - - python3-dev - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/sdispater/pytzdata - tag: ${{package.version}} - expected-commit: 11dda19bc50a2d4e246326450bd2e86389cf67c8 - - - name: Python Build - runs: | - export SETUPTOOLS_SCM_PRETEND_VERSION=${{package.version}} - pip install -U poetry - python3 -m gpep517 build-wheel \ - --wheel-dir dist \ - --output-fd 3 3>&1 >&2 - - python3 -m installer -d "${{targets.destdir}}" \ - dist/*.whl - - - uses: strip - -update: - enabled: true - ignore-regex-patterns: - - pool-* - github: - identifier: sdispater/pytzdata - use-tag: true diff --git a/py3-rich.yaml b/py3-rich.yaml index fea155e734..bd0c109684 100644 --- a/py3-rich.yaml +++ b/py3-rich.yaml @@ -9,7 +9,6 @@ package: runtime: - py3-markdown-it-py - py3-pygments - - python3 environment: contents: @@ -18,6 +17,7 @@ environment: - ca-certificates-bundle - py3-gpep517 - py3-pip + - py3-poetry - py3-setuptools - python3 - wolfi-base @@ -29,14 +29,9 @@ pipeline: tag: v${{package.version}} expected-commit: 7f580bdcf07a3b269a0e786b6a3aa9c804f393cf + - uses: py/pip-build-install + - runs: | - export SETUPTOOLS_SCM_PRETEND_VERSION=${{package.version}} - python3 -m pip install -U poetry - python3 -m gpep517 build-wheel \ - --wheel-dir dist \ - --output-fd 3 3>&1 >&2 - python3 -m installer -d "${{targets.destdir}}" \ - dist/*.whl install -Dm644 LICENSE \ "${{targets.destdir}}"/usr/share/licenses/${{package.name}}/LICENSE @@ -50,19 +45,18 @@ update: test: environment: - contents: - packages: - - py3-pip environment: RICH_TEST_STRING: "This is a test string for Rich!" pipeline: - name: Test Rich Installation runs: | - python3 -m pip install rich - python3 -c "from rich import print; print('${RICH_TEST_STRING}')" - - name: Test Rich Formatting Capabilities - runs: | - python3 -c "from rich.console import Console; console = Console(); console.print('${RICH_TEST_STRING}', style='bold red')" - - name: Test Rich JSON Rendering - runs: | - python3 -c "from rich import print; print({'json': ['Renders', 'well', 'in', 'Rich']})" + python=$(set +x; m=/usr/bin/python3.*; set -- $m + [ $# -eq 1 -a -x "$1" ] && echo "$1" && exit 0 + echo "found $# matches to $m"; exit 1;) + $python -c "from rich import print; print('${RICH_TEST_STRING}')" + + # Test Rich Formatting Capabilities + $python -c "from rich.console import Console; console = Console(); console.print('${RICH_TEST_STRING}', style='bold red')" + + # Test Rich JSON Rendering + $python -c "from rich import print; print({'json': ['Renders', 'well', 'in', 'Rich']})" diff --git a/py3-tzdata.yaml b/py3-tzdata.yaml index 5f681242a6..1ae3e86d1f 100644 --- a/py3-tzdata.yaml +++ b/py3-tzdata.yaml @@ -6,8 +6,6 @@ package: copyright: - license: Apache-2.0 dependencies: - runtime: - - python3 provider-priority: 0 vars: @@ -54,6 +52,28 @@ subpackages: with: python: python${{range.key}} import: tzdata + - runs: | + # adapted from https://tzdata.readthedocs.io/en/latest/ + python${{range.key}} -c ' + from importlib.resources import files + from tzdata.zoneinfo import America + with open(files(America).joinpath("New_York"), "rb") as f: + assert f.read(4) == b"TZif" + ' + +test: + pipeline: + - runs: | + python=$(set +x; m=/usr/bin/python3.*; set -- $m + [ $# -eq 1 -a -x "$1" ] && echo "$1" && exit 0 + echo "found $# matches to $m"; exit 1;) + # adapted from https://tzdata.readthedocs.io/en/latest/ + $python -c ' + from importlib.resources import files + from tzdata.zoneinfo import America + with open(files(America).joinpath("New_York"), "rb") as f: + assert f.read(4) == b"TZif" + ' update: enabled: true