Align scripts with latest ScanCode Toolkit

Signed-off-by: Philippe Ombredanne <[email protected]>

etc/scripts/fetch_thirdparty.py

@@ -12,6 +12,7 @@
 import itertools
 import os
 import sys
+from collections import defaultdict
 
 import click
 
@@ -110,6 +111,39 @@
     is_flag=True,
     help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.",
 )
+@click.option(
+    "--sdist-only",
+    "sdist_only",
+    type=str,
+    metavar="SDIST",
+    default=tuple(),
+    show_default=False,
+    multiple=True,
+    help="Package name(s) that come only in sdist format (no wheels). "
+         "The command will not fail and exit if no wheel exists for these names",
+)
+@click.option(
+    "--wheel-only",
+    "wheel_only",
+    type=str,
+    metavar="WHEEL",
+    default=tuple(),
+    show_default=False,
+    multiple=True,
+    help="Package name(s) that come only in wheel format (no sdist). "
+         "The command will not fail and exit if no sdist exists for these names",
+)
+@click.option(
+    "--no-dist",
+    "no_dist",
+    type=str,
+    metavar="DIST",
+    default=tuple(),
+    show_default=False,
+    multiple=True,
+    help="Package name(s) that do not come either in wheel or sdist format. "
+         "The command will not fail and exit if no distribution exists for these names",
+)
 @click.help_option("-h", "--help")
 def fetch_thirdparty(
     requirements_files,
@@ -122,6 +156,9 @@ def fetch_thirdparty(
     sdists,
     index_urls,
     use_cached_index,
+    sdist_only,
+    wheel_only,
+    no_dist,
 ):
     """
     Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions,
@@ -204,58 +241,62 @@ def fetch_thirdparty(
             )
             repos.append(repo)
 
-    wheels_fetched = []
-    wheels_not_found = []
-
-    sdists_fetched = []
-    sdists_not_found = []
+    wheels_or_sdist_not_found = defaultdict(list)
 
     for name, version in sorted(required_name_versions):
         nv = name, version
         print(f"Processing: {name} @ {version}")
         if wheels:
             for environment in environments:
+
                 if TRACE:
                     print(f"  ==> Fetching wheel for envt: {environment}")
-                fwfns = utils_thirdparty.download_wheel(
+
+                fetched = utils_thirdparty.download_wheel(
                     name=name,
                     version=version,
                     environment=environment,
                     dest_dir=dest_dir,
                     repos=repos,
                 )
-                if fwfns:
-                    wheels_fetched.extend(fwfns)
-                else:
-                    wheels_not_found.append(f"{name}=={version} for: {environment}")
+                if not fetched:
+                    wheels_or_sdist_not_found[f"{name}=={version}"].append(environment)
                     if TRACE:
                         print(f"      NOT FOUND")
 
-        if sdists:
+        if (sdists or
+            (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only)
+         ):
             if TRACE:
                 print(f"  ==> Fetching sdist: {name}=={version}")
+
             fetched = utils_thirdparty.download_sdist(
                 name=name,
                 version=version,
                 dest_dir=dest_dir,
                 repos=repos,
             )
-            if fetched:
-                sdists_fetched.append(fetched)
-            else:
-                sdists_not_found.append(f"{name}=={version}")
+            if not fetched:
+                wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist")
                 if TRACE:
                     print(f"      NOT FOUND")
 
-    if wheels and wheels_not_found:
-        print(f"==> MISSING WHEELS")
-        for wh in wheels_not_found:
-            print(f"  {wh}")
+    mia = []
+    for nv, dists in wheels_or_sdist_not_found.items():
+        name, _, version = nv.partition("==")
+        if name in no_dist:
+            continue
+        sdist_missing = sdists and "sdist" in dists and not name in wheel_only
+        if sdist_missing:
+            mia.append(f"SDist missing: {nv} {dists}")
+        wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only
+        if wheels_missing:
+            mia.append(f"Wheels missing: {nv} {dists}")
 
-    if sdists and sdists_not_found:
-        print(f"==> MISSING SDISTS")
-        for sd in sdists_not_found:
-            print(f"  {sd}")
+    if mia:
+        for m in mia:
+            print(m)
+        raise Exception(mia)
 
     print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES")
     utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index)

etc/scripts/gen_pypi_simple.py

@@ -118,7 +118,7 @@ def build_per_package_index(pkg_name, packages, base_url):
   <body>"""
     document.append(header)
 
-    for package in packages:
+    for package in sorted(packages, key=lambda p: p.archive_file):
         document.append(package.simple_index_entry(base_url))
 
     footer = """  </body>
@@ -141,8 +141,8 @@ def build_links_package_index(packages_by_package_name, base_url):
   <body>"""
     document.append(header)
 
-    for _name, packages in packages_by_package_name.items():
-        for package in packages:
+    for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]):
+        for package in sorted(packages, key=lambda p: p.archive_file):
             document.append(package.simple_index_entry(base_url))
 
     footer = """  </body>

etc/scripts/requirements.txt

@@ -8,4 +8,5 @@ pip
 setuptools
 twine
 wheel
-build
+build
+packvers

etc/scripts/utils_dejacode.py

@@ -15,7 +15,7 @@
 import requests
 import saneyaml
 
-from packaging import version as packaging_version
+from packvers import version as packaging_version
 
 """
 Utility to create and retrieve package and ABOUT file data from DejaCode.

etc/scripts/utils_pip_compatibility_tags.py

@@ -27,7 +27,7 @@
 
 import re
 
-from packaging.tags import (
+from packvers.tags import (
     compatible_tags,
     cpython_tags,
     generic_tags,

etc/scripts/utils_requirements.py

@@ -8,6 +8,8 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+
+import os
 import re
 import subprocess
 

etc/scripts/utils_thirdparty.py

@@ -28,8 +28,8 @@
 from commoncode import fileutils
 from commoncode.hash import multi_checksums
 from commoncode.text import python_safe_name
-from packaging import tags as packaging_tags
-from packaging import version as packaging_version
+from packvers import tags as packaging_tags
+from packvers import version as packaging_version
 
 import utils_pip_compatibility_tags
 
@@ -115,10 +115,9 @@
 TRACE_ULTRA_DEEP = False
 
 # Supported environments
-PYTHON_VERSIONS = "36", "37", "38", "39", "310"
+PYTHON_VERSIONS = "37", "38", "39", "310"
 
 PYTHON_DOT_VERSIONS_BY_VER = {
-    "36": "3.6",
     "37": "3.7",
     "38": "3.8",
     "39": "3.9",
@@ -134,7 +133,6 @@ def get_python_dot_version(version):
 
 
 ABIS_BY_PYTHON_VERSION = {
-    "36": ["cp36", "cp36m", "abi3"],
     "37": ["cp37", "cp37m", "abi3"],
     "38": ["cp38", "cp38m", "abi3"],
     "39": ["cp39", "cp39m", "abi3"],
@@ -912,7 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR):
         declared_license = [raw_data["License"]] + [
             c for c in classifiers if c.startswith("License")
         ]
-        license_expression = compute_normalized_license_expression(declared_license)
+        license_expression = get_license_expression(declared_license)
         other_classifiers = [c for c in classifiers if not c.startswith("License")]
 
         holder = raw_data["Author"]
@@ -1337,10 +1335,10 @@ def package_from_dists(cls, dists):
 
         For example:
         >>> w1 = Wheel(name='bitarray', version='0.8.1', build='',
-        ...    python_versions=['cp36'], abis=['cp36m'],
+        ...    python_versions=['cp38'], abis=['cp38m'],
         ...    platforms=['linux_x86_64'])
         >>> w2 = Wheel(name='bitarray', version='0.8.1', build='',
-        ...    python_versions=['cp36'], abis=['cp36m'],
+        ...    python_versions=['cp38'], abis=['cp38m'],
         ...    platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64'])
         >>> sd = Sdist(name='bitarray', version='0.8.1')
         >>> package = PypiPackage.package_from_dists(dists=[w1, w2, sd])
@@ -2274,16 +2272,16 @@ def find_problems(
     check_about(dest_dir=dest_dir)
 
 
-def compute_normalized_license_expression(declared_licenses):
+def get_license_expression(declared_licenses):
     """
     Return a normalized license expression or None.
     """
     if not declared_licenses:
         return
     try:
-        from packagedcode import pypi
+        from packagedcode.licensing import get_only_expression_from_extracted_license
 
-        return pypi.compute_normalized_license(declared_licenses)
+        return get_only_expression_from_extracted_license(declared_licenses)
     except ImportError:
         # Scancode is not installed, clean and join all the licenses
         lics = [python_safe_name(l).lower() for l in declared_licenses]