From ad07de3d9cfad4c33008d973a71ab8892871965e Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Fri, 16 Oct 2020 21:47:49 -0400 Subject: [PATCH 1/3] start work on type things --- conda_forge_tick/auto_tick.py | 35 +++++++++---------- conda_forge_tick/migrators/core.py | 36 +++++++++++++------- conda_forge_tick/migrators/migration_yaml.py | 8 ++--- conda_forge_tick/migrators_types.pyi | 15 +++++--- conda_forge_tick/utils.py | 8 ++--- 5 files changed, 59 insertions(+), 43 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index a094e3bf9..91fad0608 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -152,12 +152,6 @@ def run( branch_name = migrator.remote_branch(feedstock_ctx) + "_h" + uuid4().hex[0:6] - if hasattr(migrator, "name"): - assert isinstance(migrator.name, str) - migrator_name = migrator.name.lower().replace(" ", "") - else: - migrator_name = migrator.__class__.__name__.lower() - # TODO: run this in parallel feedstock_dir, repo = get_repo( ctx=migrator.ctx.session, @@ -173,9 +167,7 @@ def run( # migrate the feedstock migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs) - # TODO - make a commit here if the repo changed - migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs, **kwargs) if not migrate_return: @@ -237,9 +229,11 @@ def run( if ( migrator.check_solvable # we always let stuff in cycles go - and feedstock_ctx.attrs["name"] not in getattr(migrator, "cycles", set()) + and feedstock_ctx.attrs["feedstock_name"] + not in getattr(migrator, "cycles", set()) # we always let stuff at the top go - and feedstock_ctx.attrs["name"] not in getattr(migrator, "top_level", set()) + and feedstock_ctx.attrs["feedstock_name"] + not in getattr(migrator, "top_level", set()) # for solveability always assume automerge is on. and feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get("automerge", True) ) or feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get( @@ -251,7 +245,9 @@ def run( pre_key = "pre_pr_migrator_status" if pre_key not in feedstock_ctx.attrs: feedstock_ctx.attrs[pre_key] = {} - feedstock_ctx.attrs[pre_key][migrator_name] = "not solvable: %s" % sorted( + feedstock_ctx.attrs[pre_key][ + migrator.sanitized_name + ] = "not solvable: %s" % sorted( set(errors), ) eval_cmd(f"rm -rf {feedstock_dir}") @@ -262,7 +258,7 @@ def run( if ( isinstance(migrator, MigrationYaml) and not diffed_files - and feedstock_ctx.attrs["name"] != "conda-forge-pinning" + and feedstock_ctx.attrs["feedstock_name"] != "conda-forge-pinning" ): # spoof this so it looks like the package is done pr_json = { @@ -274,7 +270,7 @@ def run( # push up try: # TODO: remove this hack, but for now this is the only way to get - # the feedstock dir into pr_body + # the feedstock dir into pr_body feedstock_ctx.feedstock_dir = feedstock_dir pr_json = push_repo( session_ctx=migrator.ctx.session, @@ -305,8 +301,6 @@ def run( # from .dynamo_models import PRJson # PRJson.dump(pr_json) - # If we've gotten this far then the node is good - feedstock_ctx.attrs["bad"] = False logger.info("Removing feedstock dir") eval_cmd(f"rm -rf {feedstock_dir}") return migrate_return, ljpr @@ -331,7 +325,7 @@ def add_replacement_migrator( old_pkg: "PackageName", new_pkg: "PackageName", rationale: str, - alt_migrator: Union[Migrator, None] = None, + alt_migrator: Optional[Migrator] = None, ) -> None: """Adds a migrator to replace one package with another. @@ -584,7 +578,7 @@ def migration_factory( logger.warning("skipping migration %s because it is paused", __mname) -def _outside_pin_range(pin_spec, current_pin, new_version): +def _outside_pin_range(pin_spec: str, current_pin: str, new_version: str) -> bool: pin_level = len(pin_spec.split(".")) current_split = current_pin.split(".") new_split = new_version.split(".") @@ -597,7 +591,10 @@ def _outside_pin_range(pin_spec, current_pin, new_version): return False -def create_migration_yaml_creator(migrators: MutableSequence[Migrator], gx: nx.DiGraph): +def create_migration_yaml_creator( + migrators: MutableSequence[Migrator], + gx: nx.DiGraph, +) -> None: with indir(os.environ["CONDA_PREFIX"]): pinnings = parse_config_file( "conda_build_config.yaml", @@ -710,7 +707,7 @@ def initialize_migrators( gx, "matplotlib", "matplotlib-base", - ("Unless you need `pyqt`, recipes should depend only on " "`matplotlib-base`."), + "Unless you need `pyqt`, recipes should depend only on " "`matplotlib-base`.", alt_migrator=MatplotlibBase, ) create_migration_yaml_creator(migrators=MIGRATORS, gx=gx) diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 5019abd8c..6315c4a30 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -27,7 +27,9 @@ AttrsTypedDict, MigrationUidTypedDict, PackageName, - ) + FeedstockName, + PRedElementTypedDict, OutputsLUT, +) from conda_forge_tick.utils import JsonFriendly @@ -113,13 +115,15 @@ def __init__( # TODO: Validate this? obj_version: Optional[int] = None, piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None, - check_solvable=True, + check_solvable: bool = True, ): self.piggy_back_migrations = piggy_back_migrations or [] self.pr_limit = pr_limit self.obj_version = obj_version self.ctx: MigratorContext = None self.check_solvable = check_solvable + self.name = self.__class__.__name__.lower() + self.sanitized_name = self.name.lower().replace(" ", "") def bind_to_ctx(self, migrator_ctx: MigratorContext) -> None: self.ctx = migrator_ctx @@ -405,17 +409,19 @@ def migrator_label(cls) -> dict: class GraphMigrator(Migrator): + outputs_lut: OutputsLUT + def __init__( self, *, name: Optional[str] = None, graph: nx.DiGraph = None, pr_limit: int = 0, - top_level: Set["PackageName"] = None, - cycles: Optional[Sequence["packageName"]] = None, + top_level: Set[FeedstockName] = None, + cycles: Optional[Sequence[FeedstockName]] = None, obj_version: Optional[int] = None, piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None, - check_solvable=True, + check_solvable: bool = True, ): super().__init__( pr_limit, @@ -439,7 +445,9 @@ def __init__( for k in node.get("payload", {}).get("outputs_names", []) } - self.name = name + if name: + self.name = name + self.sanitized_name = self.name.lower().replace(" ", "") self.top_level = top_level or set() self.cycles = set(chain.from_iterable(cycles or [])) @@ -451,27 +459,31 @@ def predecessors_not_yet_built(self, attrs: "AttrsTypedDict") -> bool: except KeyError as e: print(node) raise e + + # Any archived nodes don't count so we skip them + if payload.get("archived", False): + continue + muid = frozen_to_json_friendly(self.migrator_uid(payload)) if ( muid not in _sanitized_muids( payload.get("PRed", []), ) - and not payload.get("archived", False) ): return True + # This is due to some PRed_json loss due to bad graph deploy outage + m_pred_json: PRedElementTypedDict for m_pred_json in payload.get("PRed", []): if m_pred_json["data"] == muid["data"]: break + # If we can't find the uid data in the pred json then we haven't made the pr else: - m_pred_json = None + return False # note that if the bot is missing the PR we assume it is open # so that errors halt the migration and can be fixed - if ( - m_pred_json - and m_pred_json.get("PR", {"state": "open"}).get("state", "") == "open" - ): + if m_pred_json.get("PR", {"state": "open"}).get("state", "") == "open": return True return False diff --git a/conda_forge_tick/migrators/migration_yaml.py b/conda_forge_tick/migrators/migration_yaml.py index 22a05467e..c50de9b31 100644 --- a/conda_forge_tick/migrators/migration_yaml.py +++ b/conda_forge_tick/migrators/migration_yaml.py @@ -21,8 +21,8 @@ from ..migrators_types import ( MigrationUidTypedDict, AttrsTypedDict, - PackageName, - ) + PackageName, RequirementsTypedDict, +) logger = logging.getLogger("conda_forge_tick.migrators.migration_yaml") @@ -425,7 +425,7 @@ def _req_is_python(req): return pin_sep_pat.split(req)[0].strip().lower() == "python" -def _all_noarch(attrs, only_python=False): +def _all_noarch(attrs: AttrsTypedDict, only_python: bool=False) -> bool: meta_yaml = attrs.get("meta_yaml", {}) or {} if not only_python: @@ -478,7 +478,7 @@ def create_rebuild_graph( if node == "conda-forge-pinning": continue attrs: "AttrsTypedDict" = node_attrs["payload"] - requirements = attrs.get("requirements", {}) + requirements: RequirementsTypedDict = attrs.get("requirements", {}) host = requirements.get("host", set()) build = requirements.get("build", set()) bh = host or build diff --git a/conda_forge_tick/migrators_types.pyi b/conda_forge_tick/migrators_types.pyi index ea17e5052..a37b16d91 100644 --- a/conda_forge_tick/migrators_types.pyi +++ b/conda_forge_tick/migrators_types.pyi @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Set, Tuple, Union, Optional from mypy_extensions import TypedDict PackageName = typing.NewType("PackageName", str) +FeedstockName = typing.NewType("FeedstockName", str) class AboutTypedDict(TypedDict, total=False): description: str @@ -86,9 +87,10 @@ class PackageTypedDict(TypedDict): version: str class RequirementsTypedDict(TypedDict, total=False): - build: List[str] - host: List[str] - run: List[str] + build: Set[PackageName] + host: Set[PackageName] + run: Set[PackageName] + test: Set[PackageName] class SourceTypedDict(TypedDict, total=False): fn: str @@ -106,7 +108,7 @@ class AttrsTypedDict_(TypedDict, total=False): about: AboutTypedDict build: BuildTypedDict extra: ExtraTypedDict - feedstock_name: str + feedstock_name: FeedstockName meta_yaml: MetaYamlTypedDict package: PackageTypedDict raw_meta_yaml: str @@ -122,9 +124,11 @@ class AttrsTypedDict_(TypedDict, total=False): bad: Union[bool, str] # TODO: ADD in # "conda-forge.yml": + pre_pr_migrator_status: Dict[str, str] class CondaForgeYamlContents(TypedDict, total=False): provider: Dict[str, str] + bot: Dict[str, str] CondaForgeYaml = TypedDict( "CondaForgeYaml", {"conda-forge.yml": CondaForgeYamlContents} @@ -132,3 +136,6 @@ CondaForgeYaml = TypedDict( class AttrsTypedDict(AttrsTypedDict_, CondaForgeYaml): pass + + +OutputsLUT = Dict[PackageName, FeedstockName] \ No newline at end of file diff --git a/conda_forge_tick/utils.py b/conda_forge_tick/utils.py index d7914aef5..8b5c20c89 100644 --- a/conda_forge_tick/utils.py +++ b/conda_forge_tick/utils.py @@ -35,7 +35,7 @@ if typing.TYPE_CHECKING: from mypy_extensions import TypedDict, TestTypedDict - from .migrators_types import PackageName, RequirementsTypedDict + from .migrators_types import PackageName, RequirementsTypedDict, AttrsTypedDict_ from conda_forge_tick.migrators_types import MetaYamlTypedDict @@ -78,7 +78,7 @@ ) -def eval_cmd(cmd, **kwargs): +def eval_cmd(cmd: str, **kwargs: Any) -> str: """run a command capturing stdout stderr is printed for debugging @@ -679,12 +679,12 @@ def _fetch_static_repo(name, dest): def populate_feedstock_attributes( name: str, - sub_graph: typing.MutableMapping, + sub_graph: AttrsTypedDict_, meta_yaml: typing.Union[str, Response] = "", conda_forge_yaml: typing.Union[str, Response] = "", mark_not_archived=False, feedstock_dir=None, -) -> typing.MutableMapping: +) -> AttrsTypedDict_: """Parse the various configuration information into something usable Notes From 32698afae92b249a8de8f1123e03c65bf57fed8d Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Fri, 16 Oct 2020 22:22:13 -0400 Subject: [PATCH 2/3] more types --- conda_forge_tick/migrators/core.py | 12 +++++------ conda_forge_tick/migrators/migration_yaml.py | 22 ++++++++++---------- conda_forge_tick/migrators_types.pyi | 4 +++- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 6315c4a30..0c5d82e56 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -28,8 +28,9 @@ MigrationUidTypedDict, PackageName, FeedstockName, - PRedElementTypedDict, OutputsLUT, -) + PRedElementTypedDict, + OutputsLUT, + ) from conda_forge_tick.utils import JsonFriendly @@ -465,11 +466,8 @@ def predecessors_not_yet_built(self, attrs: "AttrsTypedDict") -> bool: continue muid = frozen_to_json_friendly(self.migrator_uid(payload)) - if ( - muid - not in _sanitized_muids( - payload.get("PRed", []), - ) + if muid not in _sanitized_muids( + payload.get("PRed", []), ): return True diff --git a/conda_forge_tick/migrators/migration_yaml.py b/conda_forge_tick/migrators/migration_yaml.py index c50de9b31..729690090 100644 --- a/conda_forge_tick/migrators/migration_yaml.py +++ b/conda_forge_tick/migrators/migration_yaml.py @@ -13,21 +13,21 @@ from conda_forge_tick.contexts import FeedstockContext from conda_forge_tick.migrators.core import GraphMigrator, MiniMigrator, Migrator +from conda_forge_tick.utils import eval_cmd, pluck, pin_sep_pat from conda_forge_tick.xonsh_utils import indir -from conda_forge_tick.utils import eval_cmd -from ..utils import pluck, pin_sep_pat if typing.TYPE_CHECKING: from ..migrators_types import ( MigrationUidTypedDict, AttrsTypedDict, - PackageName, RequirementsTypedDict, -) + PackageName, + RequirementsTypedDict, + ) logger = logging.getLogger("conda_forge_tick.migrators.migration_yaml") -def _patch_dict(cfg, patches): +def _patch_dict(cfg: dict, patches: dict) -> None: """Patch a dictionary using a set of patches. Given a dict like @@ -77,7 +77,7 @@ def _patch_dict(cfg, patches): logger.warning("conda-forge.yml patch %s: %s did not work!", k, v) -def merge_migrator_cbc(migrator_yaml: str, conda_build_config_yaml: str): +def merge_migrator_cbc(migrator_yaml: str, conda_build_config_yaml: str) -> str: """Merge a migrator_yaml with the conda_build_config_yaml""" migrator_keys = defaultdict(list) current_key = None @@ -129,8 +129,8 @@ def __init__( bump_number: int = 1, piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None, automerge: bool = False, - check_solvable=True, - conda_forge_yml_patches=None, + check_solvable: bool = True, + conda_forge_yml_patches: dict = None, **kwargs: Any, ): super().__init__( @@ -421,11 +421,11 @@ def order( ) -def _req_is_python(req): +def _req_is_python(req: str) -> bool: return pin_sep_pat.split(req)[0].strip().lower() == "python" -def _all_noarch(attrs: AttrsTypedDict, only_python: bool=False) -> bool: +def _all_noarch(attrs: AttrsTypedDict, only_python: bool = False) -> bool: meta_yaml = attrs.get("meta_yaml", {}) or {} if not only_python: @@ -478,7 +478,7 @@ def create_rebuild_graph( if node == "conda-forge-pinning": continue attrs: "AttrsTypedDict" = node_attrs["payload"] - requirements: RequirementsTypedDict = attrs.get("requirements", {}) + requirements = attrs.get("requirements", {}) host = requirements.get("host", set()) build = requirements.get("build", set()) bh = host or build diff --git a/conda_forge_tick/migrators_types.pyi b/conda_forge_tick/migrators_types.pyi index a37b16d91..ad72bcee0 100644 --- a/conda_forge_tick/migrators_types.pyi +++ b/conda_forge_tick/migrators_types.pyi @@ -81,6 +81,8 @@ class MigrationUidTypedDict(TypedDict, total=False): migrator_object_version: int # Used by version migrators version: str + # Used by MigrationYamlCreator + pin_version: str class PackageTypedDict(TypedDict): name: str @@ -138,4 +140,4 @@ class AttrsTypedDict(AttrsTypedDict_, CondaForgeYaml): pass -OutputsLUT = Dict[PackageName, FeedstockName] \ No newline at end of file +OutputsLUT = Dict[PackageName, FeedstockName] From 2430e27d5a7ed03abb8a8ef428e6dfc54d800f8f Mon Sep 17 00:00:00 2001 From: "Christopher J. 'CJ' Wright" Date: Tue, 27 Oct 2020 19:10:40 -0400 Subject: [PATCH 3/3] Update conda_forge_tick/utils.py Co-authored-by: Marius van Niekerk --- conda_forge_tick/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/utils.py b/conda_forge_tick/utils.py index 8b5c20c89..7e4d1b121 100644 --- a/conda_forge_tick/utils.py +++ b/conda_forge_tick/utils.py @@ -679,7 +679,7 @@ def _fetch_static_repo(name, dest): def populate_feedstock_attributes( name: str, - sub_graph: AttrsTypedDict_, + sub_graph: "AttrsTypedDict_", meta_yaml: typing.Union[str, Response] = "", conda_forge_yaml: typing.Union[str, Response] = "", mark_not_archived=False,