From 04a872cb0efcb8be14dedd91470fb0fe44a7d319 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 22 Sep 2022 16:45:56 +0800 Subject: [PATCH 01/21] Remove the thirdparty directory Signed-off-by: Chin Yeung Li --- thirdparty/README.rst | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 thirdparty/README.rst diff --git a/thirdparty/README.rst b/thirdparty/README.rst deleted file mode 100644 index b31482f..0000000 --- a/thirdparty/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python dependency wheels to be vendored in this directory. - From 07cd7360ec38a3a43b53d7f697d225e3487ec725 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 22 Sep 2022 16:48:21 +0800 Subject: [PATCH 02/21] Update the azure-piprlines The following images are deprecated in GitHub actions and Azure DevOps: * `ubuntu-18.04` : actions/runner-images#6002 * `macos-10.15` : actions/runner-images#5583 Due to this there was failing tests due to planned brownouts. Updated ubuntu18 to ubuntu22 Updated macos-1015 to macos12 Signed-off-by: Chin Yeung Li --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ca19c4..7f1720c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,8 +9,8 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu18_cpython - image_name: ubuntu-18.04 + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -25,8 +25,8 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: macos1015_cpython - image_name: macos-10.15 + job_name: macos12_cpython + image_name: macos-12 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs From 509b03210d3965eacf60914f1dee068063c138de Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 18 Oct 2022 12:05:56 -0700 Subject: [PATCH 03/21] Add missing os import in utils_requirements.py * Rename references to etc/release with etc/scripts Signed-off-by: Jono Yang --- etc/scripts/README.rst | 8 ++++---- etc/scripts/utils_requirements.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index edf82e4..5e54a2c 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -21,7 +21,7 @@ Pre-requisites virtualenv or in the the main configured development virtualenv. These requireements need to be installed:: - pip install --requirement etc/release/requirements.txt + pip install --requirement etc/scripts/requirements.txt TODO: we need to pin the versions of these tools @@ -34,7 +34,7 @@ Scripts ~~~~~~~ **gen_requirements.py**: create/update requirements files from currently - installed requirements. + installed requirements. **gen_requirements_dev.py** does the same but can subtract the main requirements to get extra requirements used in only development. @@ -50,7 +50,7 @@ The sequence of commands to run are: ./configure --clean ./configure - python etc/release/gen_requirements.py --site-packages-dir + python etc/scripts/gen_requirements.py --site-packages-dir * You can optionally install or update extra main requirements after the ./configure step such that these are included in the generated main requirements. @@ -59,7 +59,7 @@ The sequence of commands to run are: ./configure --clean ./configure --dev - python etc/release/gen_requirements_dev.py --site-packages-dir + python etc/scripts/gen_requirements_dev.py --site-packages-dir * You can optionally install or update extra dev requirements after the ./configure step such that these are included in the generated dev diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7c99a33..db7e0ee 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,7 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import os import re import subprocess From c6bba072eba17c092cbe17b3d553828a307365a6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 25 Nov 2022 17:23:10 +0100 Subject: [PATCH 04/21] Reinstate Ubuntu 18 and drop Python 3.6 and 3.7 3.7 is not available anymore on newer OSes and is retired in 2023. Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7f1720c..e796fce 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,9 +9,9 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu22_cpython - image_name: ubuntu-22.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + job_name: ubuntu18_cpython + image_name: ubuntu-18.04 + python_versions: ['3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -19,7 +19,15 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +35,7 @@ jobs: parameters: job_name: macos12_cpython image_name: macos-12 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +43,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +51,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -51,6 +59,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 1bd7a2ff959f311fb00772982deaae4efd8355e5 Mon Sep 17 00:00:00 2001 From: swastik Date: Fri, 6 Jan 2023 03:35:13 +0530 Subject: [PATCH 05/21] Replace packaging with packvers * import update in src/scripts/utils_dejacode * Packvers replacing packaging in other src/scripts * Added packvers in src/scripts/requirments.txt Signed-off-by: swastik --- etc/scripts/requirements.txt | 3 ++- etc/scripts/utils_dejacode.py | 2 +- etc/scripts/utils_pip_compatibility_tags.py | 2 +- etc/scripts/utils_thirdparty.py | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index ebb404b..7c514da 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -8,4 +8,5 @@ pip setuptools twine wheel -build \ No newline at end of file +build +packvers diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index f28e247..c42e6c9 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -15,7 +15,7 @@ import requests import saneyaml -from packaging import version as packaging_version +from packvers import version as packaging_version """ Utility to create and retrieve package and ABOUT file data from DejaCode. diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index 5d5eb34..af42a0c 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,7 +27,7 @@ import re -from packaging.tags import ( +from packvers.tags import ( compatible_tags, cpython_tags, generic_tags, diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 53f2d33..121af38 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -28,8 +28,8 @@ from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name -from packaging import tags as packaging_tags -from packaging import version as packaging_version +from packvers import tags as packaging_tags +from packvers import version as packaging_version import utils_pip_compatibility_tags From 6f21d2b7b97b0a81741089ae9d6271b131a0ef58 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:13:17 +0100 Subject: [PATCH 06/21] Ignore egginfo Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 339dca5..2d48196 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.py[cod] # virtualenv and other misc bits +/src/*.egg-info *.egg-info /dist /build From f841c2f04732070b38810c1f03fcda26dd6ce339 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:13:45 +0100 Subject: [PATCH 07/21] Drop Python 3.7 add Python 3.11 Also test on latest Ubuntu and macOS Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ca19c4..fc5a41e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -19,7 +19,15 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +35,7 @@ jobs: parameters: job_name: macos1015_cpython image_name: macos-10.15 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +43,15 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos12_cpython + image_name: macos-12 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +59,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -51,6 +67,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From bd2a464557273dcc57bd37a0f113596c3b45a1e4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:13:58 +0100 Subject: [PATCH 08/21] Clean .cache and .eggs Signed-off-by: Philippe Ombredanne --- configure | 2 +- configure.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 32e02f5..926a894 100755 --- a/configure +++ b/configure @@ -36,7 +36,7 @@ DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" VIRTUALENV_DIR=venv # Cleanable files and directories to delete with the --clean option -CLEANABLE="build venv" +CLEANABLE="build dist venv .cache .eggs" # extra arguments passed to pip PIP_EXTRA_ARGS=" " diff --git a/configure.bat b/configure.bat index 41547cc..5e95b31 100644 --- a/configure.bat +++ b/configure.bat @@ -34,7 +34,7 @@ set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" set "VIRTUALENV_DIR=venv" @rem # Cleanable files and directories to delete with the --clean option -set "CLEANABLE=build venv" +set "CLEANABLE=build dist venv .cache .eggs" @rem # extra arguments passed to pip set "PIP_EXTRA_ARGS= " From 6270a8805c7fb964e545a56ca8a92829d240a96a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:14:27 +0100 Subject: [PATCH 09/21] Add COC to redistributed license-like files Signed-off-by: Philippe Ombredanne --- setup.cfg | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 12d6654..006b322 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ license_files = NOTICE AUTHORS.rst CHANGELOG.rst + CODE_OF_CONDUCT.rst [options] package_dir = @@ -37,7 +38,7 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.6.* +python_requires = >=3.7 install_requires = @@ -50,8 +51,10 @@ where = src testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 - aboutcode-toolkit >= 6.0.0 + aboutcode-toolkit >= 7.0.2 + twine black + isort docs = Sphinx >= 3.3.1 From d3a19bdcc126b51149f4226323158e843a6cfcad Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:14:38 +0100 Subject: [PATCH 10/21] Add new Makefile Signed-off-by: Philippe Ombredanne --- Makefile | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cc36c35 --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +# Python version can be specified with `$ PYTHON_EXE=python3.x make conf` +PYTHON_EXE?=python3 +VENV=venv +ACTIVATE?=. ${VENV}/bin/activate; + +dev: + @echo "-> Configure the development envt." + ./configure --dev + +isort: + @echo "-> Apply isort changes to ensure proper imports ordering" + ${VENV}/bin/isort --sl -l 100 src tests setup.py + +black: + @echo "-> Apply black code formatter" + ${VENV}/bin/black -l 100 src tests setup.py + +doc8: + @echo "-> Run doc8 validation" + @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ + +valid: isort black + +check: + @echo "-> Run pycodestyle (PEP8) validation" + @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . + @echo "-> Run isort imports ordering validation" + @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . + @echo "-> Run black validation" + @${ACTIVATE} black --check --check -l 100 src tests setup.py + +clean: + @echo "-> Clean the Python env" + ./configure --clean + +test: + @echo "-> Run the test suite" + ${VENV}/bin/pytest -vvs + +docs: + rm -rf docs/_build/ + @${ACTIVATE} sphinx-build docs/ docs/_build/ + +.PHONY: conf dev check valid black isort clean test docs From 91f561334ed4cbe9b003ec9d3a7e61cfa649dfd6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:15:03 +0100 Subject: [PATCH 11/21] Align scripts with latest ScanCode Toolkit Signed-off-by: Philippe Ombredanne --- etc/scripts/fetch_thirdparty.py | 87 +++++++++++++++------ etc/scripts/gen_pypi_simple.py | 6 +- etc/scripts/requirements.txt | 3 +- etc/scripts/utils_dejacode.py | 2 +- etc/scripts/utils_pip_compatibility_tags.py | 2 +- etc/scripts/utils_requirements.py | 2 + etc/scripts/utils_thirdparty.py | 20 +++-- 7 files changed, 82 insertions(+), 40 deletions(-) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 89d17de..eedf05c 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -12,6 +12,7 @@ import itertools import os import sys +from collections import defaultdict import click @@ -110,6 +111,39 @@ is_flag=True, help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", ) +@click.option( + "--sdist-only", + "sdist_only", + type=str, + metavar="SDIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in sdist format (no wheels). " + "The command will not fail and exit if no wheel exists for these names", +) +@click.option( + "--wheel-only", + "wheel_only", + type=str, + metavar="WHEEL", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in wheel format (no sdist). " + "The command will not fail and exit if no sdist exists for these names", +) +@click.option( + "--no-dist", + "no_dist", + type=str, + metavar="DIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that do not come either in wheel or sdist format. " + "The command will not fail and exit if no distribution exists for these names", +) @click.help_option("-h", "--help") def fetch_thirdparty( requirements_files, @@ -122,6 +156,9 @@ def fetch_thirdparty( sdists, index_urls, use_cached_index, + sdist_only, + wheel_only, + no_dist, ): """ Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, @@ -204,58 +241,62 @@ def fetch_thirdparty( ) repos.append(repo) - wheels_fetched = [] - wheels_not_found = [] - - sdists_fetched = [] - sdists_not_found = [] + wheels_or_sdist_not_found = defaultdict(list) for name, version in sorted(required_name_versions): nv = name, version print(f"Processing: {name} @ {version}") if wheels: for environment in environments: + if TRACE: print(f" ==> Fetching wheel for envt: {environment}") - fwfns = utils_thirdparty.download_wheel( + + fetched = utils_thirdparty.download_wheel( name=name, version=version, environment=environment, dest_dir=dest_dir, repos=repos, ) - if fwfns: - wheels_fetched.extend(fwfns) - else: - wheels_not_found.append(f"{name}=={version} for: {environment}") + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: print(f" NOT FOUND") - if sdists: + if (sdists or + (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) + ): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") + fetched = utils_thirdparty.download_sdist( name=name, version=version, dest_dir=dest_dir, repos=repos, ) - if fetched: - sdists_fetched.append(fetched) - else: - sdists_not_found.append(f"{name}=={version}") + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") if TRACE: print(f" NOT FOUND") - if wheels and wheels_not_found: - print(f"==> MISSING WHEELS") - for wh in wheels_not_found: - print(f" {wh}") + mia = [] + for nv, dists in wheels_or_sdist_not_found.items(): + name, _, version = nv.partition("==") + if name in no_dist: + continue + sdist_missing = sdists and "sdist" in dists and not name in wheel_only + if sdist_missing: + mia.append(f"SDist missing: {nv} {dists}") + wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + if wheels_missing: + mia.append(f"Wheels missing: {nv} {dists}") - if sdists and sdists_not_found: - print(f"==> MISSING SDISTS") - for sd in sdists_not_found: - print(f" {sd}") + if mia: + for m in mia: + print(m) + raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 03312ab..214d90d 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -118,7 +118,7 @@ def build_per_package_index(pkg_name, packages, base_url): """ document.append(header) - for package in packages: + for package in sorted(packages, key=lambda p: p.archive_file): document.append(package.simple_index_entry(base_url)) footer = """ @@ -141,8 +141,8 @@ def build_links_package_index(packages_by_package_name, base_url): """ document.append(header) - for _name, packages in packages_by_package_name.items(): - for package in packages: + for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]): + for package in sorted(packages, key=lambda p: p.archive_file): document.append(package.simple_index_entry(base_url)) footer = """ diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index ebb404b..7c514da 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -8,4 +8,5 @@ pip setuptools twine wheel -build \ No newline at end of file +build +packvers diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index f28e247..c42e6c9 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -15,7 +15,7 @@ import requests import saneyaml -from packaging import version as packaging_version +from packvers import version as packaging_version """ Utility to create and retrieve package and ABOUT file data from DejaCode. diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index 5d5eb34..af42a0c 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,7 +27,7 @@ import re -from packaging.tags import ( +from packvers.tags import ( compatible_tags, cpython_tags, generic_tags, diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7c99a33..0fc25a3 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,8 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # + +import os import re import subprocess diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 53f2d33..addf8e5 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -28,8 +28,8 @@ from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name -from packaging import tags as packaging_tags -from packaging import version as packaging_version +from packvers import tags as packaging_tags +from packvers import version as packaging_version import utils_pip_compatibility_tags @@ -115,10 +115,9 @@ TRACE_ULTRA_DEEP = False # Supported environments -PYTHON_VERSIONS = "36", "37", "38", "39", "310" +PYTHON_VERSIONS = "37", "38", "39", "310" PYTHON_DOT_VERSIONS_BY_VER = { - "36": "3.6", "37": "3.7", "38": "3.8", "39": "3.9", @@ -134,7 +133,6 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "36": ["cp36", "cp36m", "abi3"], "37": ["cp37", "cp37m", "abi3"], "38": ["cp38", "cp38m", "abi3"], "39": ["cp39", "cp39m", "abi3"], @@ -912,7 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): declared_license = [raw_data["License"]] + [ c for c in classifiers if c.startswith("License") ] - license_expression = compute_normalized_license_expression(declared_license) + license_expression = get_license_expression(declared_license) other_classifiers = [c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] @@ -1337,10 +1335,10 @@ def package_from_dists(cls, dists): For example: >>> w1 = Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], + ... python_versions=['cp38'], abis=['cp38m'], ... platforms=['linux_x86_64']) >>> w2 = Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], + ... python_versions=['cp38'], abis=['cp38m'], ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']) >>> sd = Sdist(name='bitarray', version='0.8.1') >>> package = PypiPackage.package_from_dists(dists=[w1, w2, sd]) @@ -2274,16 +2272,16 @@ def find_problems( check_about(dest_dir=dest_dir) -def compute_normalized_license_expression(declared_licenses): +def get_license_expression(declared_licenses): """ Return a normalized license expression or None. """ if not declared_licenses: return try: - from packagedcode import pypi + from packagedcode.licensing import get_only_expression_from_extracted_license - return pypi.compute_normalized_license(declared_licenses) + return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses lics = [python_safe_name(l).lower() for l in declared_licenses] From d661e13c743994d30593d914f567461efe66adc4 Mon Sep 17 00:00:00 2001 From: Arnav Mandal Date: Fri, 24 Mar 2023 23:38:15 +0530 Subject: [PATCH 12/21] Add pycodestyle in setup.cfg Signed-off-by: Arnav Mandal --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 006b322..edc16ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 + pycodestyle >= 2.8.0 twine black isort From e98549283f763b7097956d401df9a7191876ed16 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 28 Mar 2023 18:37:15 +0530 Subject: [PATCH 13/21] Remove deprecated github-actions/azure images - remove deprecated `ubuntu-18.04` image - remove deprecated `macos-10.15` image Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fc5a41e..5067fd4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,14 +7,6 @@ jobs: - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu18_cpython - image_name: ubuntu-18.04 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu20_cpython @@ -31,14 +23,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos1015_cpython - image_name: macos-10.15 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos11_cpython From 7e354456b35e59c6fad1f9c196994736d0ec613b Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 8 May 2023 12:37:46 -0700 Subject: [PATCH 14/21] Pin Sphinx version to 6.2.1 Signed-off-by: Jono Yang --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index edc16ba..18bfbde 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,6 @@ testing = isort docs = - Sphinx >= 3.3.1 + Sphinx == 6.2.1 sphinx-rtd-theme >= 0.5.0 doc8 >= 0.8.1 From e3aaf6309e65ad8d814d9713a20d4e930b07cbe9 Mon Sep 17 00:00:00 2001 From: Arijit De Date: Wed, 31 May 2023 20:50:06 +0530 Subject: [PATCH 15/21] Publish PDF version of RTD documentation Made changes in the .readthedocs.yaml to enable format for downloading pdf and epub versions of the documentation and added latex_elements in the conf.py file which generates the pdf without blank pages. The minimum version requirement for sphinx was 6.2.1 which was causing build failure in read the docs, hence changing it 3.3.1 as written in setup.cfg of nexB/aboutcode Signed-off-by: Arijit De --- .readthedocs.yml | 5 +++++ docs/source/conf.py | 6 ++++++ setup.cfg | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 1b71cd9..2a7dc0b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,11 @@ # Required version: 2 +# Build PDF & ePub +formats: + - epub + - pdf + # Where the Sphinx conf.py file is located sphinx: configuration: docs/source/conf.py diff --git a/docs/source/conf.py b/docs/source/conf.py index d5435e7..39835c6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -95,3 +95,9 @@ .. role:: img-title-para """ + +# -- Options for LaTeX output ------------------------------------------------- + +latex_elements = { + 'classoptions': ',openany,oneside' +} \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 18bfbde..b02fd34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,6 @@ testing = isort docs = - Sphinx == 6.2.1 + Sphinx == 5.1.0 sphinx-rtd-theme >= 0.5.0 doc8 >= 0.8.1 From 5be7a24d3f6b581f3dd9aef193923a4a23420dea Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 6 Jun 2023 21:46:49 +0530 Subject: [PATCH 16/21] Bump github actions versions #75 Update the following actions: * actions/checkout * actions/setup-python Reference: https://github.com/nexB/skeleton/issues/75 Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 4 ++-- .github/workflows/pypi-release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 18a44aa..511b7c2 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 22315ff..4ebe10d 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -24,9 +24,9 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: 3.9 From 78a64e5348f0bca0cb0fa5ead20ae4e4e16e5766 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 10 Jul 2023 15:17:09 +0530 Subject: [PATCH 17/21] Update github actions Reference: https://github.com/nexB/skeleton/issues/75 Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 4ebe10d..9585730 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -78,6 +78,6 @@ jobs: - name: Publish to PyPI if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} From a4d8628e1fc3c956f28ac8104e2d895bfad7f592 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 10 Jul 2023 15:22:22 +0530 Subject: [PATCH 18/21] Update RTD buil Signed-off-by: Ayan Sinha Mahapatra --- .readthedocs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 2a7dc0b..8ab2368 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,12 @@ # Required version: 2 +# Build in latest ubuntu/python +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # Build PDF & ePub formats: - epub From 4c68fba913f5ebd7598200e14b8085e5d38865a2 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 10 Jul 2023 15:34:19 +0530 Subject: [PATCH 19/21] Fix unordered lists issue Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index b02fd34..ae1043e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,6 @@ testing = isort docs = - Sphinx == 5.1.0 - sphinx-rtd-theme >= 0.5.0 - doc8 >= 0.8.1 + Sphinx>=5.0.2 + sphinx-rtd-theme>=1.0.0 + doc8>=0.11.2 \ No newline at end of file From c33241d5f0407f740e0a49280ffc65a60f1ab247 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 25 May 2023 23:25:58 +0530 Subject: [PATCH 20/21] Add redirects for docs Signed-off-by: Ayan Sinha Mahapatra --- docs/source/conf.py | 6 ++++++ setup.cfg | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 39835c6..918d62c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -29,8 +29,14 @@ # ones. extensions = [ "sphinx.ext.intersphinx", + "sphinx_reredirects", ] + +# Redirects for olds pages +# See https://documatt.gitlab.io/sphinx-reredirects/usage.html +redirects = {} + # This points to aboutcode.readthedocs.io # In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot # Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 diff --git a/setup.cfg b/setup.cfg index ae1043e..d6c7da7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,4 +60,6 @@ testing = docs = Sphinx>=5.0.2 sphinx-rtd-theme>=1.0.0 - doc8>=0.11.2 \ No newline at end of file + sphinx-reredirects >= 0.1.2 + doc8>=0.11.2 + From 8c042228dbd0f2f8e61852e7fb60e848d9dd4371 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 18 Jul 2023 10:05:03 -0700 Subject: [PATCH 21/21] Add macOS-13 job in azure-pipelines.yml Signed-off-by: Jono Yang --- README.rst | 5 ++++- azure-pipelines.yml | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index be65734..6cbd839 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ A Simple Python Project Skeleton ================================ This repo attempts to standardize the structure of the Python-based project's -repositories using modern Python packaging and configuration techniques. +repositories using modern Python packaging and configuration techniques. Using this `blog post`_ as inspiration, this repository serves as the base for all new Python projects and is mergeable in existing repositories as well. @@ -41,6 +41,9 @@ More usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= +- 2023-07-18: + - Add macOS-13 job in azure-pipelines.yml + - 2022-03-04: - Synchronize configure and configure.bat scripts for sanity - Update CI operating system support with latest Azure OS images diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5067fd4..764883d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,7 +26,7 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos11_cpython - image_name: macos-11 + image_name: macOS-11 python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -34,7 +34,15 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos12_cpython - image_name: macos-12 + image_name: macOS-12 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos13_cpython + image_name: macOS-13 python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs