From 94fd022746835762ae5592917029d4be1ce1edc7 Mon Sep 17 00:00:00 2001 From: Yannik Tausch Date: Thu, 8 Feb 2024 18:03:42 +0100 Subject: [PATCH] correct typing for load_graph handling of empty/nonexistent graph file --- conda_forge_tick/auto_tick.py | 7 ++--- conda_forge_tick/cli.py | 2 +- conda_forge_tick/deploy.py | 4 +-- conda_forge_tick/pypi_name_mapping.py | 4 +-- conda_forge_tick/update_prs.py | 4 +-- conda_forge_tick/update_upstream_versions.py | 4 +-- conda_forge_tick/utils.py | 27 +++++++++++++++++--- 7 files changed, 36 insertions(+), 16 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index de0253b31a..22b8cee742 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -100,6 +100,7 @@ fold_log_lines, frozen_to_json_friendly, get_keys_default, + load_existing_graph, load_graph, parse_meta_yaml, parse_munged_run_export, @@ -982,7 +983,7 @@ def initialize_migrators( dry_run: bool = False, ) -> Tuple[MigratorSessionContext, list, MutableSequence[Migrator]]: temp = glob.glob("/tmp/*") - gx = load_graph() + gx = load_existing_graph() smithy_version = eval_cmd("conda smithy --version").strip() pinning_version = json.loads(eval_cmd("conda list conda-forge-pinning --json"))[0][ "version" @@ -1409,7 +1410,7 @@ def _setup_limits(): resource.setrlimit(resource.RLIMIT_AS, (limit_int, limit_int)) -def _update_nodes_with_bot_rerun(gx): +def _update_nodes_with_bot_rerun(gx: nx.DiGraph): """Go through all the open PRs and check if they are rerun""" print("processing bot-rerun labels", flush=True) @@ -1552,7 +1553,7 @@ def _remove_closed_pr_json(): def _update_graph_with_pr_info(): _remove_closed_pr_json() - gx = load_graph() + gx = load_existing_graph() _update_nodes_with_bot_rerun(gx) _update_nodes_with_new_versions(gx) dump_graph(gx) diff --git a/conda_forge_tick/cli.py b/conda_forge_tick/cli.py index 73a72cb669..4d569cfce0 100644 --- a/conda_forge_tick/cli.py +++ b/conda_forge_tick/cli.py @@ -4,7 +4,7 @@ from typing import Optional import click -from click import IntRange, Context +from click import Context, IntRange from conda_forge_tick import lazy_json_backends diff --git a/conda_forge_tick/deploy.py b/conda_forge_tick/deploy.py index 4b5a1a1468..047706e60a 100644 --- a/conda_forge_tick/deploy.py +++ b/conda_forge_tick/deploy.py @@ -6,7 +6,7 @@ from . import sensitive_env from .cli_context import CliContext from .lazy_json_backends import CF_TICK_GRAPH_DATA_HASHMAPS, get_lazy_json_backends -from .utils import load_graph +from .utils import load_existing_graph BUILD_URL_KEY = "CIRCLE_BUILD_URL" @@ -93,7 +93,7 @@ def deploy(ctx: CliContext): # make sure the graph can load, if not we will error try: - gx = load_graph() + gx = load_existing_graph() # TODO: be more selective about which json to check for node, attrs in gx.nodes.items(): attrs["payload"]._load() diff --git a/conda_forge_tick/pypi_name_mapping.py b/conda_forge_tick/pypi_name_mapping.py index fe4b3e9801..46d42de502 100644 --- a/conda_forge_tick/pypi_name_mapping.py +++ b/conda_forge_tick/pypi_name_mapping.py @@ -20,7 +20,7 @@ from packaging.utils import canonicalize_name as canonicalize_pypi_name from .lazy_json_backends import LazyJson, dump, get_all_keys_for_hashmap, loads -from .utils import as_iterable, load_graph +from .utils import as_iterable, load_existing_graph, load_graph class Mapping(TypedDict): @@ -298,7 +298,7 @@ def determine_best_matches_for_pypi_import( map_by_conda_name[conda_name] = m graph_file = str(pathlib.Path(".") / "graph.json") - gx = load_graph(graph_file) + gx = load_existing_graph(graph_file) # TODO: filter out archived feedstocks? try: diff --git a/conda_forge_tick/update_prs.py b/conda_forge_tick/update_prs.py index 966669baa6..683c25fecb 100644 --- a/conda_forge_tick/update_prs.py +++ b/conda_forge_tick/update_prs.py @@ -18,7 +18,7 @@ from .executors import executor from .make_graph import ghctx -from .utils import github_client, load_graph +from .utils import github_client, load_existing_graph # from conda_forge_tick.profiler import profiling @@ -164,7 +164,7 @@ def close_dirty_prs( # @profiling def main(ctx: CliContext, job: int = 1, n_jobs: int = 1) -> None: - gx = load_graph() + gx = load_existing_graph() gx = close_labels(gx, ctx.dry_run, job=job, n_jobs=n_jobs) gx = update_graph_pr_status(gx, ctx.dry_run, job=job, n_jobs=n_jobs) diff --git a/conda_forge_tick/update_upstream_versions.py b/conda_forge_tick/update_upstream_versions.py index 120d5da25f..ddc41db594 100644 --- a/conda_forge_tick/update_upstream_versions.py +++ b/conda_forge_tick/update_upstream_versions.py @@ -35,7 +35,7 @@ RawURL, ROSDistro, ) -from .utils import get_keys_default, load_graph +from .utils import get_keys_default, load_existing_graph T = TypeVar("T") @@ -393,7 +393,7 @@ def main( """ logger.info("Reading graph") # Graph enabled for inspection - gx = load_graph() + gx = load_existing_graph() # Check if 'versions' folder exists or create a new one; os.makedirs("versions", exist_ok=True) diff --git a/conda_forge_tick/utils.py b/conda_forge_tick/utils.py index f981649312..1822566f2f 100644 --- a/conda_forge_tick/utils.py +++ b/conda_forge_tick/utils.py @@ -74,6 +74,8 @@ def _munge_dict_repr(d): datetime=datetime, ) +DEFAULT_GRAPH_FILENAME = "graph.json" + @contextlib.contextmanager def fold_log_lines(title): @@ -431,7 +433,6 @@ def dump_graph_json(gx: nx.DiGraph, filename: str = "graph.json") -> None: links = nld["links"] links2 = sorted(links, key=lambda x: f'{x["source"]}{x["target"]}') nld["links"] = links2 - from conda_forge_tick.lazy_json_backends import LazyJson lzj = LazyJson(filename) with lzj as attrs: @@ -447,14 +448,32 @@ def dump_graph( dump_graph_json(gx, filename) -def load_graph(filename: str = "graph.json") -> nx.DiGraph: - from conda_forge_tick.lazy_json_backends import LazyJson +def load_existing_graph(filename: str = DEFAULT_GRAPH_FILENAME) -> nx.DiGraph: + """ + Load the graph from a file using the lazy json backend. + If a non-existing or empty file is encountered, a FileNotFoundError is raised. + If you expect the graph to possibly not exist, use load_graph. + + :return: the graph + """ + gx = load_graph(filename) + if gx is None: + raise FileNotFoundError(f"Graph file {filename} does not exist or is empty") + return gx + +def load_graph(filename: str = DEFAULT_GRAPH_FILENAME) -> Optional[nx.DiGraph]: + """ + Load the graph from a file using the lazy json backend. + If you expect the graph to exist, use load_existing_graph. + + :return: the graph, or None if the file does not exist or is empty JSON + """ dta = copy.deepcopy(LazyJson(filename).data) if dta: return nx.node_link_graph(dta) else: - raise FileNotFoundError("Graph file not found.") + return None # TODO: This type does not support generics yet sadly