Merge pull request #47 from regro/odd-dftbplus

BUG correct for missing jinja2 pin functions

conda_forge_feedstock_check_solvable/check_solvable.py

@@ -1,8 +1,10 @@
 import glob
 import os
+import pprint
 from typing import Dict, List, Tuple
 
-import conda_build.api
+import conda_build.config
+import conda_build.variants
 import psutil
 from ruamel.yaml import YAML
 
@@ -14,12 +16,14 @@
     TimeoutTimer,
     TimeoutTimerException,
     apply_pins,
+    conda_build_api_render,
     get_run_exports,
     override_env_var,
     print_debug,
     print_info,
     print_warning,
     remove_reqs_by_name,
+    replace_pin_compatible,
     suppress_output,
 )
 from conda_forge_feedstock_check_solvable.virtual_packages import (
@@ -253,7 +257,7 @@ def _is_recipe_solvable_on_platform(
         timeout_timer.raise_for_timeout()
 
         # now we render the meta.yaml into an actual recipe
-        metas = conda_build.api.render(
+        metas = conda_build_api_render(
             recipe_dir,
             platform=platform,
             arch=arch,
@@ -378,12 +382,18 @@ def _is_recipe_solvable_on_platform(
                     | host_rx["strong_constrains"]
                 )
 
+        pin_compat_req = (host_req or []) if m.is_cross else (build_req or [])
+
         run_constrained = apply_pins(
             run_constrained, host_req or [], build_req or [], outnames, m
         )
         if run_req:
+            print_debug("run reqs before pins:\n\n%s\n" % pprint.pformat(run_req))
             run_req = apply_pins(run_req, host_req or [], build_req or [], outnames, m)
             run_req = remove_reqs_by_name(run_req, outnames)
+            run_req = replace_pin_compatible(run_req, pin_compat_req)
+            print_debug("run reqs after pins:\n\n%s\n" % pprint.pformat(run_req))
+
             _solvable, _err, _ = solver.solve(
                 run_req,
                 constraints=run_constrained,
@@ -405,7 +415,10 @@ def _is_recipe_solvable_on_platform(
             + run_req
         )
         if tst_req:
+            print_debug("test reqs before pins:\n\n%s\n" % pprint.pformat(tst_req))
             tst_req = remove_reqs_by_name(tst_req, outnames)
+            tst_req = replace_pin_compatible(tst_req, pin_compat_req)
+            print_debug("test reqs after pins:\n\n%s\n" % pprint.pformat(tst_req))
             _solvable, _err, _ = solver.solve(
                 tst_req,
                 constraints=run_constrained,

conda_forge_feedstock_check_solvable/utils.py

@@ -7,14 +7,17 @@
 import tempfile
 import time
 import traceback
+import unittest.mock
 from collections.abc import Mapping
 
+import conda_build.api
 import conda_package_handling.api
 import rapidjson as json
 import requests
 import wurlitzer
 import zstandard
 from conda.models.match_spec import MatchSpec
+from conda_build.jinja_context import context_processor as conda_build_context_processor
 from conda_build.utils import download_channeldata
 from conda_forge_metadata.artifact_info import get_artifact_info_as_json
 
@@ -519,10 +522,144 @@ def apply_pins(reqs, host_req, build_req, outnames, m):
             pinned_req.append(
                 get_pin_from_build(m, dep, full_build_dep_versions),
             )
-        except Exception:
+        except Exception as e:
+            print_critical(
+                "Failed to apply pin for {}, falling back to req: {}".format(
+                    dep, repr(e)
+                ),
+            )
             # in case we couldn't apply pins for whatever
             # reason, fall back to the req
             pinned_req.append(dep)
 
     pinned_req = _filter_problematic_reqs(pinned_req)
     return pinned_req
+
+
+def _render_with_name(name, *args, **kwargs):
+    out = name + "("
+    for arg in args:
+        out += repr(arg) + ","
+    for k, v in kwargs.items():
+        out += k + "=" + repr(v) + ","
+    out = out[:-1]
+    out += ")"
+    return out
+
+
+def _custom_context_processor(*args, **kwargs):
+    """Custom context processor for conda_build that changes pin_compatible."""
+    ctx = conda_build_context_processor(*args, **kwargs)
+    ctx["pin_compatible"] = lambda *args, **kwargs: _render_with_name(
+        "pin_compatible", *args, **kwargs
+    )
+    return ctx
+
+
+def conda_build_api_render(*args, **kwargs):
+    """Run conda_build.api.render with a patched jinja2 context for pin_compatible."""
+    with unittest.mock.patch(
+        "conda_build.jinja_context.context_processor", new=_custom_context_processor
+    ):
+        return conda_build.api.render(*args, **kwargs)
+
+
+def _apply_pin_compatible(
+    version,
+    build,
+    lower_bound=None,
+    upper_bound=None,
+    min_pin="x.x.x.x.x.x",
+    max_pin="x",
+    exact=False,
+):
+    from conda_build.utils import apply_pin_expressions
+
+    if exact:
+        return (version + " " + build).strip()
+    else:
+        _version = lower_bound or version
+        if _version:
+            if upper_bound:
+                if min_pin or lower_bound:
+                    compatibility = ">=" + str(_version) + ","
+                compatibility += f"<{upper_bound}"
+            else:
+                compatibility = apply_pin_expressions(_version, min_pin, max_pin)
+            return compatibility.strip()
+        else:
+            raise ValueError("No version or lower bound found for pin_compatible!")
+
+
+def _strip_quotes(s):
+    if s.startswith('"') and s.endswith('"'):
+        return s[1:-1]
+    elif s.startswith("'") and s.endswith("'"):
+        return s[1:-1]
+    else:
+        return s
+
+
+def replace_pin_compatible(reqs, host_reqs):
+    host_lookup = {req.split(" ")[0]: req.split(" ")[1:] for req in host_reqs}
+
+    new_reqs = []
+    for req in reqs:
+        if "pin_compatible(" in req:
+            if not req.startswith("pin_compatible("):
+                raise ValueError("Very odd pinning: %s!" % req)
+            parts = req.rsplit(")", 1)
+            if len(parts) == 2:
+                build = parts[1].strip()
+            else:
+                build = ""
+
+            parts = parts[0].split("pin_compatible(")[1]
+            parts = parts.split(",")
+            name = _strip_quotes(parts[0].strip())
+            parts = parts[1:]
+
+            if name not in host_lookup:
+                raise ValueError(
+                    "Very odd pinning: %s! Package %s not found in host %r!"
+                    % (req, name, host_lookup)
+                )
+            if not host_lookup[name]:
+                raise ValueError(
+                    "Very odd pinning: %s! Package found in host but no version %r!"
+                    % (req, host_lookup[name])
+                )
+
+            host_version = host_lookup[name][0]
+            if len(host_lookup[name]) > 1:
+                host_build = host_lookup[name][1]
+                if build and "exact=true" in req.lower():
+                    raise ValueError(
+                        "Build string cannot be given for pin_compatible with exact=True! %r"
+                        % req
+                    )
+            else:
+                host_build = ""
+
+            args = []
+            kwargs = {}
+            for part in parts:
+                if "=" in part:
+                    k, v = part.split("=")
+                    kwargs[k.strip()] = _strip_quotes(v.strip())
+                else:
+                    args.append(part.strip())
+
+            new_reqs.append(
+                (
+                    name
+                    + " "
+                    + _apply_pin_compatible(host_version, host_build, *args, **kwargs)
+                    + " "
+                    + build
+                ).strip()
+            )
+        else:
+            new_reqs.append(req)
+
+    return new_reqs

tests/test_check_solvable.py

@@ -258,6 +258,25 @@ def test_grpcio_solvable(tmp_path, solver):
     assert solvable, pprint.pformat(errors)
 
 
+@flaky
+def test_dftbplus_solvable(tmp_path, solver):
+    """grpcio has a runtime dep on openssl which has strange pinning things in it"""
+    feedstock_dir = clone_and_checkout_repo(
+        tmp_path,
+        "https://github.com/conda-forge/dftbplus-feedstock",
+        ref="main",
+    )
+    solvable, errors, solvable_by_variant = is_recipe_solvable(
+        feedstock_dir,
+        solver=solver,
+        verbosity=VERB,
+        timeout=None,
+        fail_fast=True,
+    )
+    pprint.pprint(solvable_by_variant)
+    assert solvable, pprint.pformat(errors)
+
+
 @flaky
 def test_cupy_solvable(tmp_path, solver):
     """grpcio has a runtime dep on openssl which has strange pinning things in it"""

tests/test_utils.py

@@ -2,13 +2,15 @@
 
 from conda_forge_feedstock_check_solvable.utils import (
     DEFAULT_RUN_EXPORTS,
+    _apply_pin_compatible,
     _convert_run_exports_to_canonical_form,
     _get_run_exports_from_artifact_info,
     _get_run_exports_from_download,
     _get_run_exports_from_run_exports_json,
     _has_run_exports_in_channel_data,
     convert_spec_to_conda_build,
     get_run_exports,
+    replace_pin_compatible,
 )
 
 
@@ -95,3 +97,69 @@ def test_utils_get_run_exports(full_channel_url, filename, expected):
         )
         == expected
     )
+
+
+def test_apply_pin_compatible():
+    version = "1.2.3"
+    build = "h24524543_0"
+    assert _apply_pin_compatible(version, build) == ">=1.2.3,<2.0a0"
+    assert (
+        _apply_pin_compatible(version, build, lower_bound="1.2.4") == ">=1.2.4,<2.0a0"
+    )
+    assert (
+        _apply_pin_compatible(version, build, upper_bound="1.2.4") == ">=1.2.3,<1.2.4"
+    )
+    assert (
+        _apply_pin_compatible(version, build, lower_bound="1.2.4", upper_bound="1.2.5")
+        == ">=1.2.4,<1.2.5"
+    )
+    assert _apply_pin_compatible(version, build, exact=True) == f"{version} {build}"
+    assert _apply_pin_compatible(version, build, max_pin="x.x") == ">=1.2.3,<1.3.0a0"
+    assert (
+        _apply_pin_compatible(version, build, min_pin="x.x", max_pin="x")
+        == ">=1.2,<2.0a0"
+    )
+
+
+def test_replace_pin_compatible():
+    host_reqs = [
+        "foo 1.2.3 5",
+        "bar 2.3 1",
+        "baz 3.4 h5678_1",
+    ]
+    reqs = [
+        "pin_compatible('foo') mpi_*",
+        "pin_compatible('bar', exact=True)",
+        "pin_compatible('baz', upper_bound='3.8')",
+        "pin_compatible('baz', lower_bound=3.5, upper_bound='3.8')",
+        "pin_compatible('foo', max_pin='x.x')",
+    ]
+    assert replace_pin_compatible(reqs, host_reqs) == [
+        "foo >=1.2.3,<2.0a0 mpi_*",
+        "bar 2.3 1",
+        "baz >=3.4,<3.8",
+        "baz >=3.5,<3.8",
+        "foo >=1.2.3,<1.3.0a0",
+    ]
+
+
+def test_replace_pin_compatible_raises():
+    with pytest.raises(ValueError) as e:
+        replace_pin_compatible(["pin_compatible('foo') mpi_*"], [])
+    assert "Package foo not found in host" in str(e.value)
+
+    with pytest.raises(ValueError) as e:
+        replace_pin_compatible(["pin_compatible('foo') mpi_*"], ["foo"])
+    assert "Package found in host but no version" in str(e.value)
+
+    with pytest.raises(ValueError) as e:
+        replace_pin_compatible(
+            ["pin_compatible('foo', exact=True) mpi_*"], ["foo 14 dfgdfs"]
+        )
+    assert "Build string cannot be given for pin_compatible with exact=True!" in str(
+        e.value
+    )
+
+    with pytest.raises(ValueError) as e:
+        replace_pin_compatible(["5 pin_compatible('foo') mpi_*"], ["foo 14 dfgdfs"])
+    assert "Very odd" in str(e.value)