feat: Add Field.codelist and open_codelist

.github/workflows/ci.yml

@@ -40,5 +40,5 @@ jobs:
           BENCHMARK_SAVE: 1
         run: pytest --benchmark-save=libcove --benchmark-only
       - if: matrix.python-version != 'pypy-3.10'
-        run: pytest --benchmark-only --benchmark-compare=0001 --benchmark-compare-fail=mean:1%
+        run: pytest --benchmark-only --benchmark-compare=0001 --benchmark-compare-fail=mean:10%
       - uses: coverallsapp/github-action@v2

docs/changelog.rst

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+1.3.1 (2024-12-16)
+------------------
+
+Added
+~~~~~
+
+-  :class:`ocdskit.schema.Field`: Add ``codelist``, ``open_codelist``.
+
 1.3.0 (2024-12-15)
 ------------------
 

docs/conf.py

@@ -23,7 +23,7 @@
 author = "Open Contracting Partnership"
 
 # The short X.Y version
-version = "1.3.0"
+version = "1.3.1"
 # The full version, including alpha/beta/rc tags
 release = version
 

manage.py

@@ -7,7 +7,12 @@
 from unittest.mock import Mock
 
 import click
-from libcove.lib.common import _get_schema_deprecated_paths, _get_schema_non_required_ids, schema_dict_fields_generator
+from libcove.lib.common import (
+    _get_schema_deprecated_paths,
+    _get_schema_non_required_ids,
+    get_schema_codelist_paths,
+    schema_dict_fields_generator,
+)
 from ocdsextensionregistry import ExtensionRegistry, ProfileBuilder
 from ocdsextensionregistry.exceptions import VersionedReleaseTypeWarning
 from ocdsextensionregistry.util import replace_refs
@@ -218,6 +223,7 @@ def main(file, tag, clobber, keep, verbose):
         for package_type in ("release", "record"):
             package_schema_path = directory / f"{package_type}-package-schema-dereferenced.json"
             additional_path = directory / f"{package_type}-additional.json"
+            codelist_path = directory / f"{package_type}-codelist.json"
             deprecated_path = directory / f"{package_type}-deprecated.json"
             missing_ids_path = directory / f"{package_type}-missing-ids.json"
 
@@ -253,6 +259,8 @@ def main(file, tag, clobber, keep, verbose):
 
             if overwrite or not additional_path.exists():
                 write(additional_path, sorted(set(schema_dict_fields_generator(package_schema))))
+            if overwrite or not codelist_path.exists():
+                write(codelist_path, [[key, value] for key, value in get_schema_codelist_paths(mock).items()])
             if overwrite or not deprecated_path.exists():
                 write(deprecated_path, _get_schema_deprecated_paths(mock))
             if overwrite or not missing_ids_path.exists():

ocdskit/schema.py

@@ -33,6 +33,10 @@ class Field:
     required: bool = False
     #: Whether the field's name is ``id`` and isn't under a ``wholeListMerge`` array.
     merge_by_id: bool = False
+    #: The field's codelist.
+    codelist: str = ''
+    #: Whether the field's codelist is open.
+    open_codelist: bool = False
     #: The separator to use in string representations of paths.
     sep = '.'
 
@@ -147,7 +151,9 @@ def get_schema_fields(
             prop_pointer = f'{pointer}/properties/{name}'
             prop_path_components = (*path_components, name)
             prop_deprecated = _deprecated(subschema)
+            prop_items = subschema.get('items', {})
 
+            # codelist and openCodelist in OCDS appear under `properties`, not `items`.
             yield Field(
                 name=name,
                 schema=subschema,
@@ -156,6 +162,8 @@ def get_schema_fields(
                 definition=definition,
                 deprecated_self=prop_deprecated,
                 deprecated=deprecated or prop_deprecated,
+                codelist=subschema.get('codelist') or prop_items.get('codelist', ''),
+                open_codelist=subschema.get('openCodelist') or prop_items.get('openCodelist', False),
                 multilingual=name in multilingual,
                 required=name in required,
                 merge_by_id=name == 'id' and array and not whole_list_merge,
@@ -173,9 +181,11 @@ def get_schema_fields(
 
     # Yield `patternProperties` last, to be interpreted in the context of `properties`.
     for name, subschema in nonmultilingual_pattern_properties.items():
+        # The duplication across `properties` and `patternProperties` can be avoided, but is >5% slower.
         prop_pointer = f'{pointer}/patternProperties/{name}'
         prop_path_components = (*path_components, name)
         prop_deprecated = _deprecated(subschema)
+        prop_items = subschema.get('items', {})
 
         yield Field(
             name=name,
@@ -185,6 +195,8 @@ def get_schema_fields(
             definition=definition,
             deprecated_self=prop_deprecated,
             deprecated=deprecated or prop_deprecated,
+            codelist=subschema.get('codelist') or prop_items.get('codelist', ''),
+            open_codelist=subschema.get('openCodelist') or prop_items.get('openCodelist', False),
             pattern=True,
             # `patternProperties` can't be multilingual, required, or "id".
         )

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ocdskit"
-version = "1.3.0"
+version = "1.3.1"
 authors = [{name = "Open Contracting Partnership", email = "[email protected]"}]
 description = "A suite of command-line tools for working with OCDS data"
 readme = "README.rst"

tests/test_regression.py

@@ -6,6 +6,7 @@
 from libcove.lib.common import (
     _get_schema_deprecated_paths,
     _get_schema_non_required_ids,
+    get_schema_codelist_paths,
     schema_dict_fields_generator,
 )
 
@@ -15,8 +16,23 @@
 # Remove parts of paths to reduce repetition in EXCEPTIONS.
 REMOVE = {"compiledRelease", "records", "releases", "value", "versionedRelease"}
 
+# Some extensions set `type` to "object" but sets `items`.
 EXCEPTIONS = {
-    # guatecompras/ocds_partyDetails_publicEntitiesLevelDetails_extension sets `type` to "object" but sets `items`.
+    # devgateway/ocds_certificate_extension
+    # devgateway/ocds_progress_extension
+    ('certificates', 'certificateAmount', 'currency'),
+    ('certificates', 'certificateAmount', 'exchangeRates', 'currency'),
+    ('certificates', 'certificateAmount', 'exchangeRates', 'source'),
+    ('certificates', 'totalAmount', 'currency'),
+    ('certificates', 'totalAmount', 'exchangeRates', 'currency'),
+    ('certificates', 'totalAmount', 'exchangeRates', 'source'),
+    ('progress', 'actualValue', 'currency'),
+    ('progress', 'actualValue', 'exchangeRates', 'currency'),
+    ('progress', 'actualValue', 'exchangeRates', 'source'),
+    ('progress', 'investmentValue', 'currency'),
+    ('progress', 'investmentValue', 'exchangeRates', 'currency'),
+    ('progress', 'investmentValue', 'exchangeRates', 'source'),
+    # guatecompras/ocds_partyDetails_publicEntitiesLevelDetails_extension
     ("complaints", "intervenients", "details", "entityType", "id"),
     ("complaints", "intervenients", "details", "legalEntityTypeDetail", "id"),
     ("complaints", "intervenients", "details", "level", "id"),
@@ -47,6 +63,7 @@ def old(schema, mock):
     set(schema_dict_fields_generator(schema))
     _get_schema_non_required_ids(mock)
     _get_schema_deprecated_paths(mock)
+    get_schema_codelist_paths(mock)
 
 
 def new(schema):
@@ -58,10 +75,17 @@ def new(schema):
     {field.path_components for field in fields if field.merge_by_id and not field.required}
     # Deprecated fields.
     {
-        field.path_components: [field.deprecated["deprecatedVersion"], field.deprecated["description"]]
+        field.path_components: (field.deprecated["deprecatedVersion"], field.deprecated["description"])
         for field in fields
         if field.deprecated
     }
+    # Codelist fields.
+    {
+        field.path_components: (field.codelist, field.open_codelist)
+        for field in fields
+        if field.codelist
+    }
+
 
 
 def test_benchmark(scenario, package_type, benchmark):
@@ -125,7 +149,7 @@ def test_get_schema_deprecated_paths(scenario, package_type):
     expected = dumpload(_get_schema_deprecated_paths(mock))
 
     actual = {
-        field.path_components: [field.deprecated_self["deprecatedVersion"], field.deprecated_self["description"]]
+        field.path_components: (field.deprecated_self["deprecatedVersion"], field.deprecated_self["description"])
         for field in get_schema_fields(schema)
         if field.deprecated
         # libcove doesn't support `oneOf`.
@@ -135,4 +159,25 @@ def test_get_schema_deprecated_paths(scenario, package_type):
     }
 
     assert expected == load(scenario, f"{package_type}-deprecated.json")
-    assert actual == {tuple(components): value for components, value in expected}
+    assert actual == {tuple(components): tuple(value) for components, value in expected}
+
+
+# libcove: get_additional_codelist_values uses get_schema_codelist_paths and calls _generate_data_path.
+def test_get_schema_codelist_paths(scenario, package_type):
+    schema = load(scenario, f"{package_type}-package-schema-dereferenced.json")
+    mock = Mock()
+    mock.get_pkg_schema_obj = Mock(return_value=schema)
+
+    expected = dumpload([[key, value] for key, value in get_schema_codelist_paths(mock).items()])
+
+    # {("tender", "status"): ("tenderStatus.csv", False), ...}
+    actual = {
+        field.path_components: (field.codelist, field.open_codelist)
+        for field in get_schema_fields(schema)
+        if field.codelist
+        # libcove trusts `type`, instead of using `properties` and `items`.
+        and tuple(c for c in field.path_components if c not in REMOVE) not in EXCEPTIONS
+    }
+
+    assert expected == load(scenario, f"{package_type}-codelist.json")
+    assert actual == {tuple(components): tuple(value) for components, value in expected}