Refactor Logging, Add GitHub Graph Backend

README.md

@@ -21,6 +21,31 @@ in the `autotick-bot` subdirectory.
 If you want to stop the worker, rename the file to `please.stop` and it will not restart
 itself on the next round.
 
+
+## Debugging Locally
+You can use the CLI of the bot to debug it locally. To do so, install the bot with the following command:
+```bash
+pip install -e .
+```
+
+Then you can use the CLI like this:
+```bash
+conda-forge-tick --help
+```
+
+For debugging, use the `--debug` flag. This enables debug logging and disables multiprocessing.
+
+Note that the bot expects the [conda-forge dependency graph](https://github.com/regro/cf-graph-countyfair) to be
+present in the current working directory by default, unless the `--online` flag is used.
+
+> [!TIP]
+> Use the `--online` flag when debugging the bot locally to avoid having to clone the whole
+> dependency graph.
+
+The local debugging functionality is still work in progress and might not work for all commands.
+Currently, the following commands are supported and tested:
+- `update-upstream-versions`
+
 ## What has the bot done recently?
 
 Check out its [PRs](https://github.com/pulls?utf8=%E2%9C%93&q=is%3Aopen+is%3Apr+author%3Aregro-cf-autotick-bot+archived%3Afalse+), its currently [running jobs](https://github.com/regro/cf-scripts/actions?query=is%3Ain_progress++), and the [status page](https://conda-forge.org/status/#current_migrations)!

conda_forge_tick/all_feedstocks.py

@@ -1,16 +1,14 @@
 import logging
-from typing import Any, List
+from typing import List
 
 import github
 import tqdm
 
 from conda_forge_tick import sensitive_env
 
-from .cli_context import CliContext
 from .lazy_json_backends import dump, load
-from .utils import setup_logger
 
-logger = logging.getLogger("conda_forge_tick.all_feedstocks")
+logger = logging.getLogger(__name__)
 
 
 def get_all_feedstocks_from_github():
@@ -67,11 +65,7 @@ def get_archived_feedstocks(cached: bool = False) -> List[str]:
     return names
 
 
-def main(ctx: CliContext) -> None:
-    if ctx.debug:
-        setup_logger(logger, level="debug")
-    else:
-        setup_logger(logger)
+def main() -> None:
     logger.info("fetching active feedstocks from github")
     data = get_all_feedstocks_from_github()
     with open("all_feedstocks.json", "w") as fp:

conda_forge_tick/auto_tick.py

@@ -27,6 +27,12 @@
 import tqdm
 
 from .cli_context import CliContext
+from .lazy_json_backends import (
+    LazyJson,
+    get_all_keys_for_hashmap,
+    lazy_json_transaction,
+    remove_key_for_hashmap,
+)
 
 if typing.TYPE_CHECKING:
     from .migrators_types import MetaYamlTypedDict, MigrationUidTypedDict, PackageName
@@ -54,12 +60,6 @@
     is_github_api_limit_reached,
     push_repo,
 )
-from conda_forge_tick.lazy_json_backends import (
-    LazyJson,
-    get_all_keys_for_hashmap,
-    lazy_json_transaction,
-    remove_key_for_hashmap,
-)
 from conda_forge_tick.migrators import (
     ArchRebuild,
     Build2HostMigrator,
@@ -100,19 +100,18 @@
     fold_log_lines,
     frozen_to_json_friendly,
     get_keys_default,
-    load_graph,
+    load_existing_graph,
     parse_meta_yaml,
     parse_munged_run_export,
     pluck,
     sanitize_string,
-    setup_logger,
     yaml_safe_load,
 )
 
 # not using this right now
 # from conda_forge_tick.deploy import deploy
 
-logger = logging.getLogger("conda_forge_tick.auto_tick")
+logger = logging.getLogger(__name__)
 
 PR_LIMIT = 5
 MAX_PR_LIMIT = 50
@@ -983,7 +982,7 @@
     dry_run: bool = False,
 ) -> Tuple[MigratorSessionContext, list, MutableSequence[Migrator]]:
     temp = glob.glob("/tmp/*")
-    gx = load_graph()
+    gx = load_existing_graph()
     smithy_version = eval_cmd("conda smithy --version").strip()
     pinning_version = json.loads(eval_cmd("conda list conda-forge-pinning --json"))[0][
         "version"
@@ -1410,7 +1409,7 @@
         resource.setrlimit(resource.RLIMIT_AS, (limit_int, limit_int))
 
 
-def _update_nodes_with_bot_rerun(gx):
+def _update_nodes_with_bot_rerun(gx: nx.DiGraph):
     """Go through all the open PRs and check if they are rerun"""
 
     print("processing bot-rerun labels", flush=True)
@@ -1553,7 +1552,7 @@
 
 def _update_graph_with_pr_info():
     _remove_closed_pr_json()
-    gx = load_graph()
+    gx = load_existing_graph()
     _update_nodes_with_bot_rerun(gx)
     _update_nodes_with_new_versions(gx)
     dump_graph(gx)
@@ -1566,12 +1565,6 @@
     global BOT_HOME_DIR
     BOT_HOME_DIR = os.getcwd()
 
-    # logging
-    if ctx.debug:
-        setup_logger(logging.getLogger("conda_forge_tick"), level="debug")
-    else:
-        setup_logger(logging.getLogger("conda_forge_tick"))
-
     with fold_log_lines("updating graph with PR info"):
         _update_graph_with_pr_info()
 

conda_forge_tick/backend_settings.py

@@ -0,0 +1,3 @@
+GITHUB_GRAPH_BACKEND_BASE_URL = (
+    "https://github.com/regro/cf-graph-countyfair/raw/master"
+)

conda_forge_tick/cli.py

@@ -1,12 +1,18 @@
+import logging
 import os
 import time
 from typing import Optional
 
 import click
-from click import IntRange
+from click import Context, IntRange
+
+from conda_forge_tick import lazy_json_backends
+from conda_forge_tick.utils import setup_logging
 
 from .cli_context import CliContext
 
+logger = logging.getLogger(__name__)
+
 pass_context = click.make_pass_decorator(CliContext, ensure=True)
 job_option = click.option(
     "--job",
@@ -46,21 +52,43 @@
     default=False,
     help="dry run: don't push changes to PRs or graph to Github",
 )
+@click.option(
+    "--online/--offline",
+    default=False,
+    help="online: Make requests to GitHub for accessing the dependency graph. This is useful for local testing. Note "
+    "however that any write operations will not be performed. Important: The current working directory will be "
+    "used to cache JSON files. Local files will be used if they exist.",
+)
 @pass_context
-def main(ctx: CliContext, debug: bool, dry_run: bool) -> None:
+@click.pass_context
+def main(
+    click_context: Context,
+    ctx: CliContext,
+    debug: bool,
+    dry_run: bool,
+    online: bool,
+) -> None:
+    log_level = "debug" if debug else "info"
+    setup_logging(log_level)
+
     ctx.debug = debug
     ctx.dry_run = dry_run
 
     if ctx.debug:
         os.environ["CONDA_FORGE_TICK_DEBUG"] = "1"
 
+    if online:
+        logger.info("Running in online mode")
+        click_context.with_resource(
+            lazy_json_backends.lazy_json_override_backends(["github"]),
+        )
+
 
 @main.command(name="gather-all-feedstocks")
-@pass_context
-def gather_all_feedstocks(ctx: CliContext) -> None:
+def gather_all_feedstocks() -> None:
     from . import all_feedstocks
 
-    all_feedstocks.main(ctx)
+    all_feedstocks.main()
 
 
 @main.command(name="make-graph")

conda_forge_tick/depfinder_api.py

@@ -40,7 +40,7 @@
 from depfinder.stdliblist import builtin_modules as _builtin_modules
 from depfinder.utils import SKETCHY_TYPES_TABLE
 
-logger = logging.getLogger("conda_forge_tick.depfinder_api")
+logger = logging.getLogger(__name__)
 
 
 def extract_pkg_from_import(name):

conda_forge_tick/deploy.py

@@ -6,7 +6,7 @@
 from . import sensitive_env
 from .cli_context import CliContext
 from .lazy_json_backends import CF_TICK_GRAPH_DATA_HASHMAPS, get_lazy_json_backends
-from .utils import load_graph
+from .utils import load_existing_graph
 
 BUILD_URL_KEY = "CIRCLE_BUILD_URL"
 
@@ -93,7 +93,7 @@
 
         # make sure the graph can load, if not we will error
         try:
-            gx = load_graph()
+            gx = load_existing_graph()
             # TODO: be more selective about which json to check
             for node, attrs in gx.nodes.items():
                 attrs["payload"]._load()

conda_forge_tick/executors.py

@@ -19,7 +19,7 @@ def __exit__(self, *args, **kwargs):
 DLOCK = DummyLock()
 
 
-logger = logging.getLogger("conda_forge_tick.executor")
+logger = logging.getLogger(__name__)
 
 
 def _init_process(lock):

conda_forge_tick/feedstock_parser.py

@@ -22,7 +22,7 @@
 
 from .utils import as_iterable, parse_meta_yaml
 
-logger = logging.getLogger("conda_forge_tick.feedstock_parser")
+logger = logging.getLogger(__name__)
 
 PIN_SEP_PAT = re.compile(r" |>|<|=|\[")