From cd5e127a567d48eb4493a2c7425ba843cb966879 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 19 Oct 2023 07:02:58 -0600 Subject: [PATCH] graph (#4) --- README.md | 493 ++++++++++++++++++++-------------- img/teatime-0.svg | 199 ++++++++++++++ img/teatime-1.svg | 199 ++++++++++++++ img/teatime-2.svg | 199 ++++++++++++++ img/teatime-3.svg | 139 ++++++++++ img/teatime-4.svg | 115 ++++++++ img/teatime-5.svg | 67 +++++ recipe/meta.json | 4 +- recipe/meta.yaml | 2 +- src/iotaa/__init__.py | 184 +++++++++---- src/iotaa/demo.py | 79 +++--- src/iotaa/tests/test_iotaa.py | 191 ++++++++++--- 12 files changed, 1553 insertions(+), 318 deletions(-) create mode 100644 img/teatime-0.svg create mode 100644 img/teatime-1.svg create mode 100644 img/teatime-2.svg create mode 100644 img/teatime-3.svg create mode 100644 img/teatime-4.svg create mode 100644 img/teatime-5.svg diff --git a/README.md b/README.md index 9cd4272..9bddcaa 100644 --- a/README.md +++ b/README.md @@ -14,33 +14,33 @@ Workflows comprise: ## Assets -An `asset` object has two attributes: +An asset (an instance of class `iotaa.Asset`) has two attributes: 1. `ref`: A value, of any type, uniquely referring to the observable state this asset represents (e.g. a POSIX filesytem path, an S3 URI, an ISO8601 timestamp) 2. `ready`: A 0-arity (no-argument) function returning a `bool` indicating whether or not the asset is ready to use -Create an `asset` by calling `asset()`. +Create an asset by calling `iotaa.asset()`. ## Tasks -Task are functions that declare, by `yield`ing values to `iotaa`, a description of the assets represented by the task (aka the task's name), plus -- depending on task type -- one or more of: the `asset`s themselves, other tasks that the task requires, and/or executable logic to make the task's asset ready. `iotaa` provides three Python decorators to define tasks: +Task are functions that declare, by `yield`ing values to `iotaa`, a description of the assets represented by the task (aka the task's name), plus -- depending on task type -- one or more of: the assets themselves, other tasks that the task requires, and/or executable logic to make the task's asset ready. `iotaa` provides three Python decorators to define tasks: ### `@task` The essential workflow function type. A `@task` function `yield`s, in order: 1. A task name describing the assets being readied, for logging -2. An `asset` -- or an `asset` `list`, or a `dict` mapping `str` keys to `asset` values, or `None` -- that the task is responsible for making ready +2. An asset -- or an asset `list`, or a `dict` mapping `str` keys to assets, or `None` -- that the task is responsible for making ready 3. A task-function call (e.g. `t(args)` for task `t`) -- or a `list` or `dict` of such calls, or `None` -- that this task requires before it can ready its own assets Arbitrary Python statements may appear before and interspersed between the `yield` statements. If the assets of all required tasks are ready, the statements following the third and final `yield` will be executed, with the expectation that they will make the task's assets ready. ### `@external` -A function type representing a required `asset` that `iotaa` cannot make ready, or a `list` or `dict` of such assets. An `@external` function `yield`s, in order: +A function type representing a required asset that `iotaa` cannot make ready, or a `list` or `dict` of such assets. An `@external` function `yield`s, in order: 1. A task name describing the assets being readied, for logging -2. A required `asset` -- or an `asset` `list`, or a `dict` mapping `str` keys to `asset` values, or `None` -- that must become ready via external means not under workflow control. (Specifying `None` may be nonsensical.) +2. A required asset -- or an asset `list`, or a `dict` mapping `str` keys to assets, or `None` -- that must become ready via external means not under workflow control. (Specifying `None` may be nonsensical.) As with `@task` functions, arbitrary Python statements may appear before and interspersed between these `yield` statements. However, no statements should follow the second and final `yield`: They will never execute since `@external` tasks are intended as passive wrappers around external state. @@ -65,7 +65,7 @@ As with `@external` tasks, no statements should follow the second and final `yie ``` % iotaa --help -usage: iotaa [-d] [-h] [-v] module function [args ...] +usage: iotaa [-d] [-h] [-g] [-v] module function [args ...] positional arguments: module @@ -80,6 +80,8 @@ optional arguments: run in dry-run mode -h, --help show help and exit + -g, --graph + emit Graphviz dot to stdout -v, --verbose verbose logging ``` @@ -102,9 +104,9 @@ Use the CLI `--dry-mode` switch (or call `dryrun()` programmatically) to run `io Several public helper callables are available in the `iotaa` module: -- `asset()` creates an asset, to be returned from task functions. +- `asset()` instantiates an asset to return from a task function. - `dryrun()` enables dry-run mode. -- `ref()` accepts a task object and returns a `dict` mapping `int` indexes (if the task `yield`s its assets as a `list` or as a single `asset`) or `str` (if the task `yield`s its assets as a `dict`) keys to the `ref` attributes of the task's assets. +- `refs()` accepts a task object and returns a `dict` mapping `int` indexes (if the task `yield`s its assets as a `list` or as a single asset) or `str` (if the task `yield`s its assets as a `dict`) keys to the `ref` attributes of the task's assets. - `logcfg()` configures Python's root logger to support `logging.info()` et al calls, which `iotaa` itself makes. It is called when the `iotaa` CLI is used, but could also be called by standalone applications with simple logging needs. - `main()` is the entry-point function for CLI use. - `run()` runs a command in a subshell -- functionality commonly needed in workflows. @@ -127,360 +129,443 @@ In the base environment of a conda installation ([Miniforge](https://github.com/ Consider the source code of the [demo application](src/iotaa/demo.py), which simulates making a cup of tea (according to [the official recipe](https://www.google.com/search?q=masters+of+reality+t.u.s.a.+lyrics)). -The first `@tasks` method defines the end result: A cup of tea, steeped, with sugar: +The first `@tasks` method defines the end result: A cup of tea, steeped, with sugar -- and a spoon to stir in the sugar: ``` python @tasks def a_cup_of_tea(basedir): - yield "A cup of steeped tea with sugar" - cupdir = ref(cup(basedir)) - yield [cup(basedir), steeped_tea_with_sugar(cupdir)] + # A cup of steeped tea with sugar, and a spoon. + yield "The perfect cup of tea" + yield [spoon(basedir), steeped_tea_with_sugar(basedir)] ``` -As described above, a `@tasks` function must `yield` its name and the assets it requires: In this case, a cup to make the tea in; then the steeped tea with sugar, in that cup. Knowledge of the location of the directory representing the cup belongs to `cup()`, and the expression `ref(cup(basedir))[0]` 1. Calls `cup()`, which returns a list of the assets it makes ready; 2. Passes those returned assets into `ref()`, which extracts the unique references to the assets (a filesystem path in this case); and 3. Retrieves the first (and in this case only) ref, which is the cup directory. (Compare to the definition of `@task` `cup`, below.) The function then declares that it requires this `cup()`, as well as steeped tea with sugar in the cup, by `yield`ing these task-function calls. +As described above, a `@tasks` function must `yield` its name and the assets it requires: In this case, the steeped tea with sugar, and a spoon. Since this function is a `@tasks` connection, no executable statements follow the final `yield.` -Note that the function could have equivalently +The `spoon()` and `cup()` `@task` functions are straightforward: ``` python - the_cup = cup(basedir) - cupdir = ref(the_cup) - yield [the_cup, steeped_tea_with_sugar(cupdir)] +@task +def spoon(basedir): + # A spoon to stir the tea. + path = Path(basedir) / "spoon" + yield "A spoon" + yield asset(path, path.exists) + yield None + path.parent.mkdir(parents=True) + path.touch() ``` -to avoid repeating the `cup(basedir)` call. But since `iotaa` caches task-function calls, repeating the call does not change the workflow's behavior. - -Since this function is a `@tasks` connection, to executable statements follow the final `yield.` - -The `cup()` `@task` function is straightforward: - ``` python @task def cup(basedir): - # Get a cup to make the tea in. + # A cup for the tea. path = Path(basedir) / "cup" - yield f"The cup: {path}" + yield "A cup" yield asset(path, path.exists) yield None path.mkdir(parents=True) ``` -It `yield`s its name; the asset it is responsible for making ready; and its requirements (it has none). Following the final `yield`, it does what is necessary to ready its asset: Creates the cup directory. +They `yield` their names; the asset each is responsible for making ready; and the tasks they require -- `None` in this case, since they have no requirements. Following the final `yield`, they do what is necessary to ready their assets: `spoon()` ensures that the base directory exists, then creates the `spoon` file therein; `cup()` creates the `cup` directory that will contain the tea ingredients. + +Note that, while `pathlib`'s `Path.mkdir()` would normally raise an exception if the specified directory already exists (unless an `exist_ok=True` argument is supplied), the workflow need not explicitly account for this possibility because `iotaa` checks for the readiness of assets before executing code that would ready them. That is, `iotaa` will not execute the `path.mkdir()` statement if it determines that the asset represented by that directoy is already ready (i.e. exists). This check is provided by the `path.exists` function supplied as the second argument to `asset()` in `cup()`. The `steeped_tea_with_sugar()` `@task` function is next: ``` python @task -def steeped_tea_with_sugar(cupdir): +def steeped_tea_with_sugar(basedir): # Add sugar to the steeped tea. Requires tea to have steeped. - for x in ingredient(cupdir, "sugar", "Steeped tea with sugar", steeped_tea): + for x in ingredient(basedir, "sugar", "Sugar", steeped_tea): yield x ``` Two new ideas are demonstrated here. -First, a task function can call other non-task logic to help it carry out its duties. In this case, it calls an `ingredient()` helper function defined thus: +First, a task function can call arbitrary logic to help it carry out its duties. In this case, it calls an `ingredient()` helper function defined thus: ``` python -def ingredient(cupdir, fn, name, req=None): - path = Path(cupdir) / fn - path.parent.mkdir(parents=True, exist_ok=True) - yield f"{name} in {cupdir}" +def ingredient(basedir, fn, name, req=None): + yield f"{name} in cup" + path = refs(cup(basedir)) / fn yield {fn: asset(path, path.exists)} - yield req(cupdir) if req else None + yield [cup(basedir)] + ([req(basedir)] if req else []) + logging.info("Adding %s to cup", fn) path.touch() ``` -This helper is called by other task functions in the workflow. It simulates adding an ingredient (tea, boiling water, sugar) to the tea cup, and handles `yield`ing the necessary values to `iotaa`. +This helper is called by other task functions in the workflow, too. It simulates adding an ingredient (tea, water, sugar) to the tea cup, and `yield`s values that the caller can re-`yield` to `iotaa`. -Second, `steeped_tea_with_sugar()` `yield`s (indirectly, by passing it to `ingredient()`) a requirement: Sugar is added as a last step after the tea is steeped, so `steeped_tea_with_sugar()` requires `steeped_tea()`. Note that it passes the function _name_ rather than a call (i.e. `steeped_tea` instead of `steeped_tea(cupdir)`) so that it can be called at the right time by `ingredient()`. +Second, `steeped_tea_with_sugar()` `yield`s (indirectly, by passing it to `ingredient()`) a requirement: Sugar is added as a last step after the tea is steeped, so `steeped_tea_with_sugar()` requires `steeped_tea()`. Note that it passes the function _name_ rather than a call (i.e. `steeped_tea` instead of `steeped_tea(basedir)`) so that it can be called at the right time by `ingredient()`. -Next up, the `steeped_tea()` `@task` function, which is somewhat more complex: +Next up, the `steeped_tea()` function, which is more complex: ``` python @task -def steeped_tea(cupdir): +def steeped_tea(basedir): # Give tea time to steep. - yield f"Steeped tea in {cupdir}" - ready = False - water = ref(steeping_tea(cupdir))["water"] + yield "Steeped tea" + water = refs(steeping_tea(basedir))["water"] + steep_time = lambda x: asset("elapsed time", lambda: x) + t = 10 # seconds if water.exists(): water_poured_time = dt.datetime.fromtimestamp(water.stat().st_mtime) - ready_time = water_poured_time + dt.timedelta(seconds=10) + ready_time = water_poured_time + dt.timedelta(seconds=t) now = dt.datetime.now() ready = now >= ready_time - yield asset(None, lambda: ready) - if not ready: - logging.info("Tea needs to steep for %ss", int((ready_time - now).total_seconds())) + remaining = int((ready_time - now).total_seconds()) + yield steep_time(ready) else: - yield asset(None, lambda: False) - yield steeping_tea(cupdir) + ready = False + remaining = t + yield steep_time(False) + yield steeping_tea(basedir) + if not ready: + logging.warning("Tea needs to steep for %ss", remaining) ``` -Here, the asset being `yield`ed is abstract: It represents a certain amount of time having passed since the boiling water was poured over the tea. (The observant reader will note that 10 seconds is insufficient, though useful for a demo. Try 3 minutes for black tea.) The path to the `water` file is located by calling `ref()` on the return value of `steeping_tea()` and taking the item with key `water`. (Because `ingredient()` `yield`s its assets as `{fn: asset(path, path.exists)}`, where `fn` if the filename, e.g. `tea`, `water`, `sugar`.) If the water was poured long enough ago, `steeped_tea` is ready; if not, it should be during some future execution of this workflow. The function finally `yield`s the `steeping_tea` task it requires. There are no post-`yield` statements, because there's nothing this task can do to make its asset (time passed) ready. It can only wait. +Here, the asset being `yield`ed is more abstract: It represents a certain amount of time having passed since the boiling water was poured over the tea. (The observant reader will note that 10 seconds is insufficient, if handy for a demo. Try 3 minutes for black tea IRL.) The path to the `water` file is located by calling `refs()` on the return value of `steeping_tea()` and taking the item with key `water` (because `ingredient()` `yield`s its assets as `{fn: asset(path, path.exists)}`, where `fn` is the filename, e.g. `sugar`, `teabag`, `water`.) If the water was poured long enough ago, `steeped_tea` is ready; if not, it should be during some future execution of this workflow. Note that the executable code following the final `yield` only logs information: There's nothing this task can do to make its asset (time passed) ready: It can only wait. -The `steeping_tea()` and `tea_bad()` functions are again straightforward `@task`s, leveraging the `ingredient()` helper: +Note the statement + +``` python +water = refs(steeping_tea(basedir))["water"] +``` + +Here, `steeped_tea()` needs to know the path to the `water` file, and obtains it by calling the `steeping_tea()` task, extracting the references to its assets with `iotaa`'s `refs()` function, and selecting the `"water"` item's reference, which is the path to the `water` file. This is a useful way to delegate ownership of knowledge about assets to those assets, but note that the function call `steeping_tea(basedir)` effectively transfers workflow control to that task. This can be seen in the execution traces shown later in this document, where the task responsible for the `water` file (as well as its requirements) are evaluated before the steep-time task. + +The `steeping_tea()` and `teabag()` functions are again straightforward `@task`s, leveraging the `ingredient()` helper: ``` python @task -def steeping_tea(cupdir): - # Pour boiling water over the tea. Requires tea bag in cup. - for x in ingredient(cupdir, "water", "Boiling water over the tea", tea_bag): +def steeping_tea(basedir): + # Pour boiling water over the tea. Requires teabag in cup. + for x in ingredient(basedir, "water", "Boiling water", teabag): yield x ``` ``` python @task -def tea_bag(cupdir): - # Place tea bag in the cup. Requires box of tea bags. - for x in ingredient(cupdir, "tea", "Tea bag", box_of_tea_bags): +def teabag(basedir): + # Place tea bag in the cup. Requires box of teabags. + for x in ingredient(basedir, "teabag", "Teabag", box_of_teabags): yield x ``` -Finally, we have this workflow's only `@external` task, `box_of_tea_bags()`. The idea here is that this is something that simply must exist, and no action by the workflow can create it: +Finally, we have this workflow's only `@external` task, `box_of_teabags()`. The idea here is that this is something that simply must exist (think: someone must have simply bought the box of teabags at the store), and no action by the workflow can create it. Unlike other task types, the `@external` `yield`s, after its name, only the _assets_ that it represents. It `yield`s no task requirements, and has no executable statements to make the asset ready: ``` python @external -def box_of_tea_bags(cupdir): - path = Path(cupdir).parent / "box-of-tea" - yield f"Tea from store: {path}" +def box_of_teabags(basedir): + path = Path(basedir) / "box-of-teabags" + yield f"Box of teabags {path}" yield asset(path, path.exists) ``` -Unlike other task types, the `@external` `yield`s, after its name, only the _assets_ that it represents. It `yield`s no task requirements, and has no executable statements to make the asset ready. - Let's run this workflow with the `iotaa` command-line tool, requesting that the workflow start with the `a_cup_of_tea` task: ``` % iotaa iotaa.demo a_cup_of_tea ./teatime -[2023-09-19T22:44:21] INFO A cup of steeped tea with sugar: Checking required tasks -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Initial state: Pending -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Checking required tasks -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Ready -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Executing -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Final state: Ready -[2023-09-19T22:44:21] INFO Steeped tea with sugar in teatime/cup: Initial state: Pending -[2023-09-19T22:44:21] INFO Steeped tea with sugar in teatime/cup: Checking required tasks -[2023-09-19T22:44:21] INFO Boiling water over the tea in teatime/cup: Initial state: Pending -[2023-09-19T22:44:21] INFO Boiling water over the tea in teatime/cup: Checking required tasks -[2023-09-19T22:44:21] INFO Tea bag in teatime/cup: Initial state: Pending -[2023-09-19T22:44:21] INFO Tea bag in teatime/cup: Checking required tasks -[2023-09-19T22:44:21] WARNING Tea from store: teatime/box-of-tea: Final state: Pending (EXTERNAL) -[2023-09-19T22:44:21] INFO Tea bag in teatime/cup: Pending -[2023-09-19T22:44:21] WARNING Tea bag in teatime/cup: Final state: Pending -[2023-09-19T22:44:21] INFO Boiling water over the tea in teatime/cup: Pending -[2023-09-19T22:44:21] WARNING Boiling water over the tea in teatime/cup: Final state: Pending -[2023-09-19T22:44:21] INFO Steeped tea in teatime/cup: Initial state: Pending -[2023-09-19T22:44:21] INFO Steeped tea in teatime/cup: Checking required tasks -[2023-09-19T22:44:21] INFO Steeped tea in teatime/cup: Pending -[2023-09-19T22:44:21] WARNING Steeped tea in teatime/cup: Final state: Pending -[2023-09-19T22:44:21] INFO Steeped tea with sugar in teatime/cup: Pending -[2023-09-19T22:44:21] WARNING Steeped tea with sugar in teatime/cup: Final state: Pending -[2023-09-19T22:44:21] WARNING A cup of steeped tea with sugar: Final state: Pending -``` - -There's lots to see during the first invocation. Most of the tasks start and end in a pending state. Only the `cup()` task makes progress from pending to ready state: - -``` -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Initial state: Pending -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Checking required tasks -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Ready -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Executing -[2023-09-19T22:44:21] INFO The cup: teatime/cup: Final state: Ready -``` +[2023-10-19T11:49:43] INFO The perfect cup of tea: Initial state: Pending +[2023-10-19T11:49:43] INFO The perfect cup of tea: Checking requirements +[2023-10-19T11:49:43] INFO A spoon: Initial state: Pending +[2023-10-19T11:49:43] INFO A spoon: Checking requirements +[2023-10-19T11:49:43] INFO A spoon: Requirement(s) ready +[2023-10-19T11:49:43] INFO A spoon: Executing +[2023-10-19T11:49:43] INFO A spoon: Final state: Ready +[2023-10-19T11:49:43] INFO A cup: Initial state: Pending +[2023-10-19T11:49:43] INFO A cup: Checking requirements +[2023-10-19T11:49:43] INFO A cup: Requirement(s) ready +[2023-10-19T11:49:43] INFO A cup: Executing +[2023-10-19T11:49:43] INFO A cup: Final state: Ready +[2023-10-19T11:49:43] INFO Sugar in cup: Initial state: Pending +[2023-10-19T11:49:43] INFO Sugar in cup: Checking requirements +[2023-10-19T11:49:43] INFO Boiling water in cup: Initial state: Pending +[2023-10-19T11:49:43] INFO Boiling water in cup: Checking requirements +[2023-10-19T11:49:43] INFO Teabag in cup: Initial state: Pending +[2023-10-19T11:49:43] INFO Teabag in cup: Checking requirements +[2023-10-19T11:49:43] WARNING Box of teabags teatime/box-of-teabags: State: Pending (EXTERNAL) +[2023-10-19T11:49:43] INFO Teabag in cup: Requirement(s) pending +[2023-10-19T11:49:43] WARNING Teabag in cup: Final state: Pending +[2023-10-19T11:49:43] INFO Boiling water in cup: Requirement(s) pending +[2023-10-19T11:49:43] WARNING Boiling water in cup: Final state: Pending +[2023-10-19T11:49:43] INFO Steeped tea: Initial state: Pending +[2023-10-19T11:49:43] INFO Steeped tea: Checking requirements +[2023-10-19T11:49:43] INFO Steeped tea: Requirement(s) pending +[2023-10-19T11:49:43] WARNING Steeped tea: Final state: Pending +[2023-10-19T11:49:43] INFO Sugar in cup: Requirement(s) pending +[2023-10-19T11:49:43] WARNING Sugar in cup: Final state: Pending +[2023-10-19T11:49:43] WARNING The perfect cup of tea: Final state: Pending +``` + +There's lots to see during the first invocation. Most of the tasks start and end in a pending state. Only the `spoon()` and `cup()` tasks make progress from `Pending` to `Ready` states: + +``` +[2023-10-19T11:49:43] INFO A spoon: Initial state: Pending +[2023-10-19T11:49:43] INFO A spoon: Checking requirements +[2023-10-19T11:49:43] INFO A spoon: Requirement(s) ready +[2023-10-19T11:49:43] INFO A spoon: Executing +[2023-10-19T11:49:43] INFO A spoon: Final state: Ready +``` + +``` +[2023-10-19T11:49:43] INFO A cup: Initial state: Pending +[2023-10-19T11:49:43] INFO A cup: Checking requirements +[2023-10-19T11:49:43] INFO A cup: Requirement(s) ready +[2023-10-19T11:49:43] INFO A cup: Executing +[2023-10-19T11:49:43] INFO A cup: Final state: Ready +``` + +We will see in subsequent workflow invocations that these tasks are not revisited, as their assets will be found to be ready. The on-disk workflow state is: ``` -% tree teatime/ -teatime/ -└── cup +% tree teatime +teatime +├── cup +└── spoon ``` Note the blocker: ``` -[2023-09-19T22:44:21] WARNING Tea from store: teatime/box-of-tea: Final state: Pending (EXTERNAL) - +[2023-10-19T11:49:43] WARNING Box of teabags teatime/box-of-teabags: State: Pending (EXTERNAL) ``` -The file `teatime/box-of-tea` cannot be created by the workflow, as it is declared `@external`. Let's create it externally: +The file `teatime/box-of-teabags` cannot be created by the workflow, as it is declared `@external`. Let's create it externally: ``` -% touch teatime/box-of-tea +% touch teatime/box-of-teabags % tree teatime/ teatime/ -├── box-of-tea -└── cup +├── box-of-teabags +├── cup +└── spoon ``` Now let's iterate the workflow: ``` % iotaa iotaa.demo a_cup_of_tea ./teatime -[2023-09-19T22:46:33] INFO A cup of steeped tea with sugar: Checking required tasks -[2023-09-19T22:46:33] INFO Steeped tea with sugar in teatime/cup: Initial state: Pending -[2023-09-19T22:46:33] INFO Steeped tea with sugar in teatime/cup: Checking required tasks -[2023-09-19T22:46:33] INFO Boiling water over the tea in teatime/cup: Initial state: Pending -[2023-09-19T22:46:33] INFO Boiling water over the tea in teatime/cup: Checking required tasks -[2023-09-19T22:46:33] INFO Tea bag in teatime/cup: Initial state: Pending -[2023-09-19T22:46:33] INFO Tea bag in teatime/cup: Checking required tasks -[2023-09-19T22:46:33] INFO Tea bag in teatime/cup: Ready -[2023-09-19T22:46:33] INFO Tea bag in teatime/cup: Executing -[2023-09-19T22:46:33] INFO Tea bag in teatime/cup: Final state: Ready -[2023-09-19T22:46:33] INFO Boiling water over the tea in teatime/cup: Ready -[2023-09-19T22:46:33] INFO Boiling water over the tea in teatime/cup: Executing -[2023-09-19T22:46:33] INFO Boiling water over the tea in teatime/cup: Final state: Ready -[2023-09-19T22:46:33] INFO Steeped tea in teatime/cup: Initial state: Pending -[2023-09-19T22:46:33] INFO Steeped tea in teatime/cup: Checking required tasks -[2023-09-19T22:46:33] INFO Tea needs to steep for 9s -[2023-09-19T22:46:33] INFO Steeped tea in teatime/cup: Ready -[2023-09-19T22:46:33] INFO Steeped tea in teatime/cup: Executing -[2023-09-19T22:46:33] INFO Steeped tea with sugar in teatime/cup: Pending -[2023-09-19T22:46:33] WARNING Steeped tea with sugar in teatime/cup: Final state: Pending -[2023-09-19T22:46:33] WARNING A cup of steeped tea with sugar: Final state: Pending +[2023-10-19T11:52:09] INFO The perfect cup of tea: Initial state: Pending +[2023-10-19T11:52:09] INFO The perfect cup of tea: Checking requirements +[2023-10-19T11:52:09] INFO Sugar in cup: Initial state: Pending +[2023-10-19T11:52:09] INFO Sugar in cup: Checking requirements +[2023-10-19T11:52:09] INFO Boiling water in cup: Initial state: Pending +[2023-10-19T11:52:09] INFO Boiling water in cup: Checking requirements +[2023-10-19T11:52:09] INFO Teabag in cup: Initial state: Pending +[2023-10-19T11:52:09] INFO Teabag in cup: Checking requirements +[2023-10-19T11:52:09] INFO Teabag in cup: Requirement(s) ready +[2023-10-19T11:52:09] INFO Teabag in cup: Executing +[2023-10-19T11:52:09] INFO Adding teabag to cup +[2023-10-19T11:52:09] INFO Teabag in cup: Final state: Ready +[2023-10-19T11:52:09] INFO Boiling water in cup: Requirement(s) ready +[2023-10-19T11:52:09] INFO Boiling water in cup: Executing +[2023-10-19T11:52:09] INFO Adding water to cup +[2023-10-19T11:52:09] INFO Boiling water in cup: Final state: Ready +[2023-10-19T11:52:09] INFO Steeped tea: Initial state: Pending +[2023-10-19T11:52:09] INFO Steeped tea: Checking requirements +[2023-10-19T11:52:09] INFO Steeped tea: Requirement(s) ready +[2023-10-19T11:52:09] INFO Steeped tea: Executing +[2023-10-19T11:52:09] WARNING Tea needs to steep for 9s +[2023-10-19T11:52:09] INFO Sugar in cup: Requirement(s) pending +[2023-10-19T11:52:09] WARNING Sugar in cup: Final state: Pending +[2023-10-19T11:52:09] WARNING The perfect cup of tea: Final state: Pending ``` On-disk workflow state now: ``` -% tree teatime/ -teatime/ -├── box-of-tea -└── cup - ├── tea - └── water +% tree teatime +teatime +├── box-of-teabags +├── cup +│ ├── teabag +│ └── water +└── spoon ``` -Since the box of tea became available, the workflow could add tea to the cup and pour boiling water over it. Note the informative message `Tea needs to steep for 9s`. If we iterate the workflow again quickly, we can see the steep time decreasing: +Since the box of tea became available, the workflow was able to add tea to the cup and pour boiling water over it. Note the message `Tea needs to steep for 9s`. If we iterate the workflow again after a few seconds, we can see the steep time decreasing: ``` % iotaa iotaa.demo a_cup_of_tea ./teatime ... -[2023-09-19T22:46:37] INFO Tea needs to steep for 5s +[2023-10-19T11:52:12] WARNING Tea needs to steep for 6s ... ``` -If we wait a few seconds more and iterate: +If we wait a bit longer and iterate: ``` % iotaa iotaa.demo a_cup_of_tea ./teatime -[2023-09-19T22:47:11] INFO A cup of steeped tea with sugar: Checking required tasks -[2023-09-19T22:47:11] INFO Steeped tea with sugar in teatime/cup: Initial state: Pending -[2023-09-19T22:47:11] INFO Steeped tea with sugar in teatime/cup: Checking required tasks -[2023-09-19T22:47:11] INFO Steeped tea with sugar in teatime/cup: Ready -[2023-09-19T22:47:11] INFO Steeped tea with sugar in teatime/cup: Executing -[2023-09-19T22:47:11] INFO Steeped tea with sugar in teatime/cup: Final state: Ready -[2023-09-19T22:47:11] INFO A cup of steeped tea with sugar: Final state: Ready +[2023-10-19T11:53:49] INFO The perfect cup of tea: Initial state: Pending +[2023-10-19T11:53:49] INFO The perfect cup of tea: Checking requirements +[2023-10-19T11:53:49] INFO Sugar in cup: Initial state: Pending +[2023-10-19T11:53:49] INFO Sugar in cup: Checking requirements +[2023-10-19T11:53:49] INFO Sugar in cup: Requirement(s) ready +[2023-10-19T11:53:49] INFO Sugar in cup: Executing +[2023-10-19T11:53:49] INFO Adding sugar to cup +[2023-10-19T11:53:49] INFO Sugar in cup: Final state: Ready +[2023-10-19T11:53:49] INFO The perfect cup of tea: Final state: Ready ``` Now that the tea has steeped long enough, the sugar has been added: ``` -% tree teatime/ -teatime/ -├── box-of-tea -└── cup - ├── sugar - ├── tea - └── water +% tree teatime +teatime +├── box-of-teabags +├── cup +│ ├── sugar +│ ├── teabag +│ └── water +└── spoon ``` One more iteration and we see that the workflow has reached its final state and takes no more action: ``` % iotaa iotaa.demo a_cup_of_tea ./teatime -[2023-09-19T22:48:22] INFO A cup of steeped tea with sugar: Checking required tasks -[2023-09-19T22:48:22] INFO A cup of steeped tea with sugar: Final state: Ready +[2023-10-19T11:54:32] INFO The perfect cup of tea: Initial state: Pending +[2023-10-19T11:54:32] INFO The perfect cup of tea: Checking requirements +[2023-10-19T11:54:32] INFO The perfect cup of tea: Final state: Ready ``` -Since `a_cup_of_tea()` is a `@tasks` collection, its state is contingent on that of its required tasks, so its readiness check will always involve checking requirements, unlike a non-collection `@task`, which can just check its own assets. +Since `a_cup_of_tea()` is a `@tasks` _collection_, its state is contingent on that of its required tasks, so its readiness check will always involve checking requirements, unlike a non-collection `@task`, which can just check its assets. One useful feature of this kind of workflow is its ability to recover from damage to its external state. Here, we remove the sugar from the tea (don't try this at home): ``` -% rm -v teatime/cup/sugar +% rm -v teatime/cup/sugar removed 'teatime/cup/sugar' % tree teatime/ teatime/ -├── box-of-tea -└── cup - ├── tea - └── water +├── box-of-teabags +├── cup +│ ├── teabag +│ └── water +└── spoon ``` Note how the workflow detects the change to the readiness of its assets and recovers: ``` % iotaa iotaa.demo a_cup_of_tea ./teatime -[2023-09-19T22:49:03] INFO A cup of steeped tea with sugar: Checking required tasks -[2023-09-19T22:49:03] INFO Steeped tea with sugar in teatime/cup: Initial state: Pending -[2023-09-19T22:49:03] INFO Steeped tea with sugar in teatime/cup: Checking required tasks -[2023-09-19T22:49:03] INFO Steeped tea with sugar in teatime/cup: Ready -[2023-09-19T22:49:03] INFO Steeped tea with sugar in teatime/cup: Executing -[2023-09-19T22:49:03] INFO Steeped tea with sugar in teatime/cup: Final state: Ready -[2023-09-19T22:49:03] INFO A cup of steeped tea with sugar: Final state: Ready +[2023-10-19T11:55:45] INFO The perfect cup of tea: Initial state: Pending +[2023-10-19T11:55:45] INFO The perfect cup of tea: Checking requirements +[2023-10-19T11:55:45] INFO Sugar in cup: Initial state: Pending +[2023-10-19T11:55:45] INFO Sugar in cup: Checking requirements +[2023-10-19T11:55:45] INFO Sugar in cup: Requirement(s) ready +[2023-10-19T11:55:45] INFO Sugar in cup: Executing +[2023-10-19T11:55:45] INFO Adding sugar to cup +[2023-10-19T11:55:45] INFO Sugar in cup: Final state: Ready +[2023-10-19T11:55:45] INFO The perfect cup of tea: Final state: Ready ``` ``` -% tree teatime/ -teatime/ -├── box-of-tea -└── cup - ├── sugar - ├── tea - └── water +% tree teatime +teatime +├── box-of-teabags +├── cup +│ ├── sugar +│ ├── teabag +│ └── water +└── spoon ``` Another useful feature is the ability to enter the workflow's task graph at an arbitrary point to obtain only a subset of the assets. For example, if we'd like a cup of tea _without_ sugar, we can start with the `steeped_tea` task rather than the higher-level `a_cup_of_tea` task. -First, empty the cup: +First, let's empty the cup: ``` % rm -v teatime/cup/* removed 'teatime/cup/sugar' -removed 'teatime/cup/tea' +removed 'teatime/cup/teabag' removed 'teatime/cup/water' -(DEV-iotaa) ~/git/iotaa % tree teatime/ +% tree teatime/ teatime/ -├── box-of-tea -└── cup +├── box-of-teabags +├── cup +└── spoon +``` + +Now request tea without sugar: + +``` +% iotaa iotaa.demo steeped_tea ./teatime +% iotaa iotaa.demo steeped_tea ./teatime +[2023-10-19T11:57:31] INFO Boiling water in cup: Initial state: Pending +[2023-10-19T11:57:31] INFO Boiling water in cup: Checking requirements +[2023-10-19T11:57:31] INFO Teabag in cup: Initial state: Pending +[2023-10-19T11:57:31] INFO Teabag in cup: Checking requirements +[2023-10-19T11:57:31] INFO Teabag in cup: Requirement(s) ready +[2023-10-19T11:57:31] INFO Teabag in cup: Executing +[2023-10-19T11:57:31] INFO Adding teabag to cup +[2023-10-19T11:57:31] INFO Teabag in cup: Final state: Ready +[2023-10-19T11:57:31] INFO Boiling water in cup: Requirement(s) ready +[2023-10-19T11:57:31] INFO Boiling water in cup: Executing +[2023-10-19T11:57:31] INFO Adding water to cup +[2023-10-19T11:57:31] INFO Boiling water in cup: Final state: Ready +[2023-10-19T11:57:31] INFO Steeped tea: Initial state: Pending +[2023-10-19T11:57:31] INFO Steeped tea: Checking requirements +[2023-10-19T11:57:31] INFO Steeped tea: Requirement(s) ready +[2023-10-19T11:57:31] INFO Steeped tea: Executing +[2023-10-19T11:57:31] WARNING Tea needs to steep for 9s ``` -Now request tea without sugar (note that task `steeped_tea` expects a path to the cup as its argument, so `./teatime/cup` is supplied here instead of just `./teatime`: +After waiting for the tea to steep: ``` -% iotaa iotaa.demo steeped_tea ./teatime/cup -[2023-09-19T22:49:40] INFO Boiling water over the tea in ./teatime/cup: Initial state: Pending -[2023-09-19T22:49:40] INFO Boiling water over the tea in ./teatime/cup: Checking required tasks -[2023-09-19T22:49:40] INFO Tea bag in ./teatime/cup: Initial state: Pending -[2023-09-19T22:49:40] INFO Tea bag in ./teatime/cup: Checking required tasks -[2023-09-19T22:49:40] INFO Tea bag in ./teatime/cup: Ready -[2023-09-19T22:49:40] INFO Tea bag in ./teatime/cup: Executing -[2023-09-19T22:49:40] INFO Tea bag in ./teatime/cup: Final state: Ready -[2023-09-19T22:49:40] INFO Boiling water over the tea in ./teatime/cup: Ready -[2023-09-19T22:49:40] INFO Boiling water over the tea in ./teatime/cup: Executing -[2023-09-19T22:49:40] INFO Boiling water over the tea in ./teatime/cup: Final state: Ready -[2023-09-19T22:49:40] INFO Steeped tea in ./teatime/cup: Initial state: Pending -[2023-09-19T22:49:40] INFO Steeped tea in ./teatime/cup: Checking required tasks -[2023-09-19T22:49:40] INFO Tea needs to steep for 9s -[2023-09-19T22:49:40] INFO Steeped tea in ./teatime/cup: Ready -[2023-09-19T22:49:40] INFO Steeped tea in ./teatime/cup: Executing +% iotaa iotaa.demo steeped_tea ./teatime +2023-10-19T11:57:57] INFO Steeped tea: Initial state: Ready ``` -After waiting for the tea to steep: +On-disk state: ``` -% iotaa iotaa.demo steeped_tea ./teatime/cup -[2023-09-19T22:49:56] INFO Steeped tea in ./teatime/cup: Initial state: Ready +% tree teatime/ +teatime/ +├── box-of-teabags +├── cup +│ ├── teabag +│ └── water +└── spoon ``` -On-disk state: +## Graphing + +The `-g` / `--graph` switch can be used to emit to `stdout` a description of the current state of the workflow graph in [Grapviz](https://graphviz.org/) [DOT](https://graphviz.org/doc/info/lang.html) format. Here, for example, the preceding demo workflow is executed in dry-run mode with graph output requested, and the graph document rendered as an SVG image by `dot` and displayed by the Linux utility `display`: ``` -% tree teatime -teatime -├── box-of-tea -└── cup - ├── tea - └── water +% iotaa --dry-run --graph iotaa.demo a_cup_of_tea ./teatime | display <(dot -T svg) +[2023-10-19T12:13:47] INFO The perfect cup of tea: Initial state: Pending +... +[2023-10-19T12:13:47] WARNING The perfect cup of tea: Final state: Pending ``` + +The displayed image (ready assets are shown in green, pending ones in orange): + +![teatime-dry-run-image](img/teatime-0.svg) + +Removing `--dry-run` and following the first phase of the demo tutorial in the previous section, the following succession of graph images are shown: + +First run, blocked by missing (external) box of tea: + +![teatime-dry-run-image](img/teatime-1.svg) + +Second run, with box of tea available: + +![teatime-dry-run-image](img/teatime-2.svg) + +Third run, waiting for tea to steep: + +![teatime-dry-run-image](img/teatime-3.svg) + +Fourth run, with sugar added to steeped tea: + +![teatime-dry-run-image](img/teatime-4.svg) + +Fifth run, showing final sate: + +![teatime-dry-run-image](img/teatime-5.svg) diff --git a/img/teatime-0.svg b/img/teatime-0.svg new file mode 100644 index 0000000..d7a3abe --- /dev/null +++ b/img/teatime-0.svg @@ -0,0 +1,199 @@ + + + + + + +g + + + +_1bc88e99f3b24153cfb80abde9870319 + +Teabag in cup + + + +_5b1d149c29769d3624f4e34af2330bec + +A cup + + + +_1bc88e99f3b24153cfb80abde9870319->_5b1d149c29769d3624f4e34af2330bec + + + + + +_a15fff0e94cf39f74a065b108fe52bbd + +Box of teabags teatime/box-of-teabags + + + +_1bc88e99f3b24153cfb80abde9870319->_a15fff0e94cf39f74a065b108fe52bbd + + + + + +_a73e34ca7f79d6d1fe3ec0d56b93c881 + +teatime/cup/teabag + + + +_1bc88e99f3b24153cfb80abde9870319->_a73e34ca7f79d6d1fe3ec0d56b93c881 + + + + + +_f9810bce5bbcd6a4d4b834b75279ddce + +teatime/cup + + + +_5b1d149c29769d3624f4e34af2330bec->_f9810bce5bbcd6a4d4b834b75279ddce + + + + + +_9f642df358ddbee291d14a457d6c628d + +teatime/box-of-teabags + + + +_a15fff0e94cf39f74a065b108fe52bbd->_9f642df358ddbee291d14a457d6c628d + + + + + +_4dfbb28a53db4770d8c9efa105b2d3ca + +A spoon + + + +_7f2d897b3949441090ec5b405dbeebb1 + +teatime/spoon + + + +_4dfbb28a53db4770d8c9efa105b2d3ca->_7f2d897b3949441090ec5b405dbeebb1 + + + + + +_63d2ed416115be6cf86c48e10dacf755 + +Sugar in cup + + + +_63d2ed416115be6cf86c48e10dacf755->_5b1d149c29769d3624f4e34af2330bec + + + + + +_e520f89ffb54953975127f0474fc8ac6 + +teatime/cup/sugar + + + +_63d2ed416115be6cf86c48e10dacf755->_e520f89ffb54953975127f0474fc8ac6 + + + + + +_fc7725b504dfe81a0923659641500bf6 + +Steeped tea + + + +_63d2ed416115be6cf86c48e10dacf755->_fc7725b504dfe81a0923659641500bf6 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406 + +Boiling water in cup + + + +_fc7725b504dfe81a0923659641500bf6->_e2ae5a6ebee14aca44ab9445c4149406 + + + + + +_e43f78155a7c2c42d6d65e1702ba0b0f + +elapsed time + + + +_fc7725b504dfe81a0923659641500bf6->_e43f78155a7c2c42d6d65e1702ba0b0f + + + + + +_6ac80c9b9d540f4035cde8e540cb0369 + +The perfect cup of tea + + + +_6ac80c9b9d540f4035cde8e540cb0369->_4dfbb28a53db4770d8c9efa105b2d3ca + + + + + +_6ac80c9b9d540f4035cde8e540cb0369->_63d2ed416115be6cf86c48e10dacf755 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406->_1bc88e99f3b24153cfb80abde9870319 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406->_5b1d149c29769d3624f4e34af2330bec + + + + + +_f1b07b232490520bf0d7034d936250d9 + +teatime/cup/water + + + +_e2ae5a6ebee14aca44ab9445c4149406->_f1b07b232490520bf0d7034d936250d9 + + + + + diff --git a/img/teatime-1.svg b/img/teatime-1.svg new file mode 100644 index 0000000..692978f --- /dev/null +++ b/img/teatime-1.svg @@ -0,0 +1,199 @@ + + + + + + +g + + + +_1bc88e99f3b24153cfb80abde9870319 + +Teabag in cup + + + +_5b1d149c29769d3624f4e34af2330bec + +A cup + + + +_1bc88e99f3b24153cfb80abde9870319->_5b1d149c29769d3624f4e34af2330bec + + + + + +_a15fff0e94cf39f74a065b108fe52bbd + +Box of teabags teatime/box-of-teabags + + + +_1bc88e99f3b24153cfb80abde9870319->_a15fff0e94cf39f74a065b108fe52bbd + + + + + +_a73e34ca7f79d6d1fe3ec0d56b93c881 + +teatime/cup/teabag + + + +_1bc88e99f3b24153cfb80abde9870319->_a73e34ca7f79d6d1fe3ec0d56b93c881 + + + + + +_f9810bce5bbcd6a4d4b834b75279ddce + +teatime/cup + + + +_5b1d149c29769d3624f4e34af2330bec->_f9810bce5bbcd6a4d4b834b75279ddce + + + + + +_9f642df358ddbee291d14a457d6c628d + +teatime/box-of-teabags + + + +_a15fff0e94cf39f74a065b108fe52bbd->_9f642df358ddbee291d14a457d6c628d + + + + + +_4dfbb28a53db4770d8c9efa105b2d3ca + +A spoon + + + +_7f2d897b3949441090ec5b405dbeebb1 + +teatime/spoon + + + +_4dfbb28a53db4770d8c9efa105b2d3ca->_7f2d897b3949441090ec5b405dbeebb1 + + + + + +_63d2ed416115be6cf86c48e10dacf755 + +Sugar in cup + + + +_63d2ed416115be6cf86c48e10dacf755->_5b1d149c29769d3624f4e34af2330bec + + + + + +_e520f89ffb54953975127f0474fc8ac6 + +teatime/cup/sugar + + + +_63d2ed416115be6cf86c48e10dacf755->_e520f89ffb54953975127f0474fc8ac6 + + + + + +_fc7725b504dfe81a0923659641500bf6 + +Steeped tea + + + +_63d2ed416115be6cf86c48e10dacf755->_fc7725b504dfe81a0923659641500bf6 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406 + +Boiling water in cup + + + +_fc7725b504dfe81a0923659641500bf6->_e2ae5a6ebee14aca44ab9445c4149406 + + + + + +_e43f78155a7c2c42d6d65e1702ba0b0f + +elapsed time + + + +_fc7725b504dfe81a0923659641500bf6->_e43f78155a7c2c42d6d65e1702ba0b0f + + + + + +_6ac80c9b9d540f4035cde8e540cb0369 + +The perfect cup of tea + + + +_6ac80c9b9d540f4035cde8e540cb0369->_4dfbb28a53db4770d8c9efa105b2d3ca + + + + + +_6ac80c9b9d540f4035cde8e540cb0369->_63d2ed416115be6cf86c48e10dacf755 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406->_1bc88e99f3b24153cfb80abde9870319 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406->_5b1d149c29769d3624f4e34af2330bec + + + + + +_f1b07b232490520bf0d7034d936250d9 + +teatime/cup/water + + + +_e2ae5a6ebee14aca44ab9445c4149406->_f1b07b232490520bf0d7034d936250d9 + + + + + diff --git a/img/teatime-2.svg b/img/teatime-2.svg new file mode 100644 index 0000000..542b6af --- /dev/null +++ b/img/teatime-2.svg @@ -0,0 +1,199 @@ + + + + + + +g + + + +_1bc88e99f3b24153cfb80abde9870319 + +Teabag in cup + + + +_5b1d149c29769d3624f4e34af2330bec + +A cup + + + +_1bc88e99f3b24153cfb80abde9870319->_5b1d149c29769d3624f4e34af2330bec + + + + + +_a15fff0e94cf39f74a065b108fe52bbd + +Box of teabags teatime/box-of-teabags + + + +_1bc88e99f3b24153cfb80abde9870319->_a15fff0e94cf39f74a065b108fe52bbd + + + + + +_a73e34ca7f79d6d1fe3ec0d56b93c881 + +teatime/cup/teabag + + + +_1bc88e99f3b24153cfb80abde9870319->_a73e34ca7f79d6d1fe3ec0d56b93c881 + + + + + +_f9810bce5bbcd6a4d4b834b75279ddce + +teatime/cup + + + +_5b1d149c29769d3624f4e34af2330bec->_f9810bce5bbcd6a4d4b834b75279ddce + + + + + +_9f642df358ddbee291d14a457d6c628d + +teatime/box-of-teabags + + + +_a15fff0e94cf39f74a065b108fe52bbd->_9f642df358ddbee291d14a457d6c628d + + + + + +_4dfbb28a53db4770d8c9efa105b2d3ca + +A spoon + + + +_7f2d897b3949441090ec5b405dbeebb1 + +teatime/spoon + + + +_4dfbb28a53db4770d8c9efa105b2d3ca->_7f2d897b3949441090ec5b405dbeebb1 + + + + + +_63d2ed416115be6cf86c48e10dacf755 + +Sugar in cup + + + +_63d2ed416115be6cf86c48e10dacf755->_5b1d149c29769d3624f4e34af2330bec + + + + + +_e520f89ffb54953975127f0474fc8ac6 + +teatime/cup/sugar + + + +_63d2ed416115be6cf86c48e10dacf755->_e520f89ffb54953975127f0474fc8ac6 + + + + + +_fc7725b504dfe81a0923659641500bf6 + +Steeped tea + + + +_63d2ed416115be6cf86c48e10dacf755->_fc7725b504dfe81a0923659641500bf6 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406 + +Boiling water in cup + + + +_fc7725b504dfe81a0923659641500bf6->_e2ae5a6ebee14aca44ab9445c4149406 + + + + + +_e43f78155a7c2c42d6d65e1702ba0b0f + +elapsed time + + + +_fc7725b504dfe81a0923659641500bf6->_e43f78155a7c2c42d6d65e1702ba0b0f + + + + + +_6ac80c9b9d540f4035cde8e540cb0369 + +The perfect cup of tea + + + +_6ac80c9b9d540f4035cde8e540cb0369->_4dfbb28a53db4770d8c9efa105b2d3ca + + + + + +_6ac80c9b9d540f4035cde8e540cb0369->_63d2ed416115be6cf86c48e10dacf755 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406->_1bc88e99f3b24153cfb80abde9870319 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406->_5b1d149c29769d3624f4e34af2330bec + + + + + +_f1b07b232490520bf0d7034d936250d9 + +teatime/cup/water + + + +_e2ae5a6ebee14aca44ab9445c4149406->_f1b07b232490520bf0d7034d936250d9 + + + + + diff --git a/img/teatime-3.svg b/img/teatime-3.svg new file mode 100644 index 0000000..a84eed0 --- /dev/null +++ b/img/teatime-3.svg @@ -0,0 +1,139 @@ + + + + + + +g + + + +_4dfbb28a53db4770d8c9efa105b2d3ca + +A spoon + + + +_7f2d897b3949441090ec5b405dbeebb1 + +teatime/spoon + + + +_4dfbb28a53db4770d8c9efa105b2d3ca->_7f2d897b3949441090ec5b405dbeebb1 + + + + + +_5b1d149c29769d3624f4e34af2330bec + +A cup + + + +_f9810bce5bbcd6a4d4b834b75279ddce + +teatime/cup + + + +_5b1d149c29769d3624f4e34af2330bec->_f9810bce5bbcd6a4d4b834b75279ddce + + + + + +_63d2ed416115be6cf86c48e10dacf755 + +Sugar in cup + + + +_63d2ed416115be6cf86c48e10dacf755->_5b1d149c29769d3624f4e34af2330bec + + + + + +_e520f89ffb54953975127f0474fc8ac6 + +teatime/cup/sugar + + + +_63d2ed416115be6cf86c48e10dacf755->_e520f89ffb54953975127f0474fc8ac6 + + + + + +_fc7725b504dfe81a0923659641500bf6 + +Steeped tea + + + +_63d2ed416115be6cf86c48e10dacf755->_fc7725b504dfe81a0923659641500bf6 + + + + + +_e2ae5a6ebee14aca44ab9445c4149406 + +Boiling water in cup + + + +_fc7725b504dfe81a0923659641500bf6->_e2ae5a6ebee14aca44ab9445c4149406 + + + + + +_e43f78155a7c2c42d6d65e1702ba0b0f + +elapsed time + + + +_fc7725b504dfe81a0923659641500bf6->_e43f78155a7c2c42d6d65e1702ba0b0f + + + + + +_6ac80c9b9d540f4035cde8e540cb0369 + +The perfect cup of tea + + + +_6ac80c9b9d540f4035cde8e540cb0369->_4dfbb28a53db4770d8c9efa105b2d3ca + + + + + +_6ac80c9b9d540f4035cde8e540cb0369->_63d2ed416115be6cf86c48e10dacf755 + + + + + +_f1b07b232490520bf0d7034d936250d9 + +teatime/cup/water + + + +_e2ae5a6ebee14aca44ab9445c4149406->_f1b07b232490520bf0d7034d936250d9 + + + + + diff --git a/img/teatime-4.svg b/img/teatime-4.svg new file mode 100644 index 0000000..eb924f6 --- /dev/null +++ b/img/teatime-4.svg @@ -0,0 +1,115 @@ + + + + + + +g + + + +_4dfbb28a53db4770d8c9efa105b2d3ca + +A spoon + + + +_7f2d897b3949441090ec5b405dbeebb1 + +teatime/spoon + + + +_4dfbb28a53db4770d8c9efa105b2d3ca->_7f2d897b3949441090ec5b405dbeebb1 + + + + + +_5b1d149c29769d3624f4e34af2330bec + +A cup + + + +_f9810bce5bbcd6a4d4b834b75279ddce + +teatime/cup + + + +_5b1d149c29769d3624f4e34af2330bec->_f9810bce5bbcd6a4d4b834b75279ddce + + + + + +_63d2ed416115be6cf86c48e10dacf755 + +Sugar in cup + + + +_63d2ed416115be6cf86c48e10dacf755->_5b1d149c29769d3624f4e34af2330bec + + + + + +_e520f89ffb54953975127f0474fc8ac6 + +teatime/cup/sugar + + + +_63d2ed416115be6cf86c48e10dacf755->_e520f89ffb54953975127f0474fc8ac6 + + + + + +_fc7725b504dfe81a0923659641500bf6 + +Steeped tea + + + +_63d2ed416115be6cf86c48e10dacf755->_fc7725b504dfe81a0923659641500bf6 + + + + + +_e43f78155a7c2c42d6d65e1702ba0b0f + +elapsed time + + + +_fc7725b504dfe81a0923659641500bf6->_e43f78155a7c2c42d6d65e1702ba0b0f + + + + + +_6ac80c9b9d540f4035cde8e540cb0369 + +The perfect cup of tea + + + +_6ac80c9b9d540f4035cde8e540cb0369->_4dfbb28a53db4770d8c9efa105b2d3ca + + + + + +_6ac80c9b9d540f4035cde8e540cb0369->_63d2ed416115be6cf86c48e10dacf755 + + + + + diff --git a/img/teatime-5.svg b/img/teatime-5.svg new file mode 100644 index 0000000..750e869 --- /dev/null +++ b/img/teatime-5.svg @@ -0,0 +1,67 @@ + + + + + + +g + + + +_4dfbb28a53db4770d8c9efa105b2d3ca + +A spoon + + + +_7f2d897b3949441090ec5b405dbeebb1 + +teatime/spoon + + + +_4dfbb28a53db4770d8c9efa105b2d3ca->_7f2d897b3949441090ec5b405dbeebb1 + + + + + +_63d2ed416115be6cf86c48e10dacf755 + +Sugar in cup + + + +_e520f89ffb54953975127f0474fc8ac6 + +teatime/cup/sugar + + + +_63d2ed416115be6cf86c48e10dacf755->_e520f89ffb54953975127f0474fc8ac6 + + + + + +_6ac80c9b9d540f4035cde8e540cb0369 + +The perfect cup of tea + + + +_6ac80c9b9d540f4035cde8e540cb0369->_4dfbb28a53db4770d8c9efa105b2d3ca + + + + + +_6ac80c9b9d540f4035cde8e540cb0369->_63d2ed416115be6cf86c48e10dacf755 + + + + + diff --git a/recipe/meta.json b/recipe/meta.json index a7ebdca..0a6788c 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -1,5 +1,5 @@ { - "build": "py_0", + "build": "0", "buildnum": "0", "name": "iotaa", "packages": { @@ -16,5 +16,5 @@ ], "run": [] }, - "version": "0.3.0" + "version": "0.4.0" } diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 8a605eb..37c52e9 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: iotaa - version: 0.3.0 + version: 0.4.0 source: path: ../src build: diff --git a/src/iotaa/__init__.py b/src/iotaa/__init__.py index 316bafc..b002fd9 100644 --- a/src/iotaa/__init__.py +++ b/src/iotaa/__init__.py @@ -5,16 +5,21 @@ import logging import sys from argparse import ArgumentParser, HelpFormatter, Namespace +from collections import defaultdict from dataclasses import dataclass from functools import cache +from hashlib import md5 from importlib import import_module from itertools import chain from json import JSONDecodeError, loads from pathlib import Path from subprocess import STDOUT, CalledProcessError, check_output from types import SimpleNamespace as ns -from typing import Any, Callable, Dict, Generator, List, Optional, Union +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union +_graph = ns(assets={}, edges=set(), tasks=set()) +_graph_color: Dict[Any, str] = defaultdict(lambda: "grey", [(True, "palegreen"), (False, "orange")]) +_graph_shape = ns(asset="box", task="ellipse") _state = ns(dry_run=False, initialized=False) @@ -22,7 +27,7 @@ @dataclass -class asset: +class Asset: """ A workflow asset (observable external state). @@ -31,11 +36,11 @@ class asset: """ ref: Any - ready: Callable + ready: Callable[..., bool] @dataclass -class result: +class Result: """ The result of running an external command. @@ -47,15 +52,25 @@ class result: success: bool -_Assets = Union[Dict[str, asset], List[asset]] -_AssetT = Optional[Union[_Assets, asset]] +_Assets = Union[Dict[str, Asset], List[Asset]] +_AssetT = Optional[Union[_Assets, Asset]] + + +def asset(ref: Any, ready: Callable[..., bool]) -> Asset: + """ + Factory function for Asset objects. + + :param ref: An object uniquely identifying the asset (e.g. a filesystem path). + :param ready: A function that, when called, indicates whether the asset is ready to use. + :return: An Asset object. + """ + return Asset(ref, ready) def dryrun() -> None: """ Enable dry-run mode. """ - _state.dry_run = True @@ -65,7 +80,6 @@ def logcfg(verbose: bool = False) -> None: :param bool: Log at the debug level? """ - logging.basicConfig( datefmt="%Y-%m-%dT%H:%M:%S", format="[%(asctime)s] %(levelname)-7s %(message)s", @@ -95,9 +109,11 @@ def main() -> None: args.module = m.stem reified = [_reify(arg) for arg in args.args] getattr(import_module(args.module), args.function)(*reified) + if args.graph: + _graph_emit() -def ref(assets: _AssetT) -> Any: +def refs(assets: _AssetT) -> Any: """ Extract and return asset identity objects. @@ -113,7 +129,7 @@ def ref(assets: _AssetT) -> Any: return {k: v.ref for k, v in assets.items()} if isinstance(assets, list): return {i: v.ref for i, v in enumerate(assets)} - if isinstance(assets, asset): + if isinstance(assets, Asset): return assets.ref return None @@ -124,7 +140,7 @@ def run( cwd: Optional[Union[Path, str]] = None, env: Optional[Dict[str, str]] = None, log: Optional[bool] = False, -) -> result: +) -> Result: """ Run a command in a subshell. @@ -133,9 +149,8 @@ def run( :param cwd: Change to this directory before running cmd. :param env: Environment variables to set before running cmd. :param log: Log output from successful cmd? (Error output is always logged.) - :return: A result object providing stderr, stdout and success info. + :return: The stderr, stdout and success info. """ - indent = " " logging.info("%s: Running: %s", taskname, cmd) if cwd: @@ -159,7 +174,7 @@ def run( logfunc("%s: %sOutput:", taskname, indent) for line in output.split("\n"): logfunc("%s: %s%s", taskname, indent * 2, line) - return result(output=output, success=success) + return Result(output=output, success=success) def runconda( @@ -170,7 +185,7 @@ def runconda( cwd: Optional[Union[Path, str]] = None, env: Optional[Dict[str, str]] = None, log: Optional[bool] = False, -) -> result: +) -> Result: """ Run a command in the specified conda environment. @@ -181,7 +196,7 @@ def runconda( :param cwd: Change to this directory before running cmd. :param env: Environment variables to set before running cmd. :param log: Log output from successful cmd? (Error output is always logged.) - :return: A result object providing stderr, stdout and success info. + :return: The stderr, stdout and success info. """ cmd = " && ".join( [ @@ -198,18 +213,18 @@ def runconda( def external(f) -> Callable[..., _AssetT]: """ - The @external decorator for assets that cannot be produced by the workflow. + The @external decorator for assets the workflow cannot produce. """ @cache def decorated_external(*args, **kwargs) -> _AssetT: - g = f(*args, **kwargs) - taskname = next(g) + taskname, top, g = _task_initial(f, *args, **kwargs) assets = next(g) ready = _ready(assets) - if not ready or _i_am_top_task(): + if not ready or top: + _graph_update_from_task(taskname, assets) _report_readiness(ready=ready, taskname=taskname, is_external=True) - return assets + return _task_final(taskname, assets) return decorated_external @@ -221,23 +236,23 @@ def task(f) -> Callable[..., _AssetT]: @cache def decorated_task(*args, **kwargs) -> _AssetT: - g = f(*args, **kwargs) - taskname = next(g) + taskname, top, g = _task_initial(f, *args, **kwargs) assets = next(g) ready_initial = _ready(assets) - if not ready_initial or _i_am_top_task(): + if not ready_initial or top: + _graph_update_from_task(taskname, assets) _report_readiness(ready=ready_initial, taskname=taskname, initial=True) if not ready_initial: if _ready(_delegate(g, taskname)): - logging.info("%s: Ready", taskname) + logging.info("%s: Requirement(s) ready", taskname) _execute(g, taskname) else: - logging.info("%s: Pending", taskname) + logging.info("%s: Requirement(s) pending", taskname) _report_readiness(ready=False, taskname=taskname) ready_final = _ready(assets) if ready_final != ready_initial: _report_readiness(ready=ready_final, taskname=taskname) - return assets + return _task_final(taskname, assets) return decorated_task @@ -249,13 +264,14 @@ def tasks(f) -> Callable[..., _AssetT]: @cache def decorated_tasks(*args, **kwargs) -> _AssetT: - g = f(*args, **kwargs) - taskname = next(g) + taskname, top, g = _task_initial(f, *args, **kwargs) + if top: + _report_readiness(ready=False, taskname=taskname, initial=True) assets = _delegate(g, taskname) ready = _ready(assets) - if not ready or _i_am_top_task(): + if not ready or top: _report_readiness(ready=ready, taskname=taskname) - return assets + return _task_final(taskname, assets) return decorated_tasks @@ -263,7 +279,7 @@ def decorated_tasks(*args, **kwargs) -> _AssetT: # Private functions -def _delegate(g: Generator, taskname: str) -> List[asset]: +def _delegate(g: Generator, taskname: str) -> List[Asset]: """ Delegate execution to the current task's requirement(s). @@ -277,8 +293,10 @@ def _delegate(g: Generator, taskname: str) -> List[asset]: # it to a list for iteration. The value of each task-function call is a collection of assets, # one asset, or None. Convert those values to lists, flatten them, and filter None objects. - logging.info("%s: Checking required tasks", taskname) - return list(filter(None, chain(*[_listify(a) for a in _listify(next(g))]))) + logging.info("%s: Checking requirements", taskname) + alist = list(filter(None, chain(*[_listify(a) for a in _listify(next(g))]))) + _graph_udpate_from_requirements(taskname, alist) + return alist def _execute(g: Generator, taskname: str) -> None: @@ -288,9 +306,8 @@ def _execute(g: Generator, taskname: str) -> None: :param g: The current task. :param taskname: The current task's name. """ - if _state.dry_run: - logging.info("%s: SKIPPING (DRY RUN ENABLED)", taskname) + logging.info("%s: SKIPPING (DRY RUN)", taskname) return try: logging.info("%s: Executing", taskname) @@ -306,34 +323,85 @@ def _formatter(prog: str) -> HelpFormatter: :param prog: The program name. :return: An argparse help formatter. """ - return HelpFormatter(prog, max_help_position=4) +def _graph_emit() -> None: + """ + Emit a task/asset graph in Graphviz dot format. + """ + f = lambda name, shape, ready=None: '%s [fillcolor=%s, label="%s", shape=%s, style=filled]' % ( + _graph_name(name), + _graph_color[ready], + name, + shape, + ) + edges = ["%s -> %s" % (_graph_name(a), _graph_name(b)) for a, b in _graph.edges] + nodes_a = [f(ref, _graph_shape.asset, ready()) for ref, ready in _graph.assets.items()] + nodes_t = [f(x, _graph_shape.task) for x in _graph.tasks] + print("digraph g {\n %s\n}" % "\n ".join(sorted(nodes_t + nodes_a + edges))) + + +def _graph_name(name: str) -> str: + """ + Convert an iotaa asset/task name to a Graphviz-appropriate node name. + + :param name: An iotaa asset/task name. + :return: A Graphviz-appropriate node name. + """ + return "_%s" % md5(str(name).encode("utf-8")).hexdigest() + + +def _graph_udpate_from_requirements(taskname: str, alist: List[Asset]) -> None: + """ + Update graph data structures with required-task info. + + :param taskname: The current task's name. + :param alist: Flattened required-task assets. + """ + asset_taskname = lambda a: getattr(a, "taskname", None) + _graph.assets.update({a.ref: a.ready for a in alist}) + _graph.edges |= set((asset_taskname(a), a.ref) for a in alist) + _graph.edges |= set((taskname, asset_taskname(a)) for a in alist) + _graph.tasks |= set(asset_taskname(a) for a in alist) + _graph.tasks.add(taskname) + + +def _graph_update_from_task(taskname: str, assets: _AssetT) -> None: + """ + Update graph data structures with current task info. + + :param taskname: The current task's name. + :param assets: A collection of assets, one asset, or None. + """ + alist = _listify(assets) + _graph.assets.update({a.ref: a.ready for a in alist}) + _graph.edges |= set((taskname, a.ref) for a in alist) + _graph.tasks.add(taskname) + + def _i_am_top_task() -> bool: """ Is the calling task the task-tree entry point? :return: Is it? """ - if _state.initialized: return False _state.initialized = True return True -def _listify(assets: _AssetT) -> List[asset]: +def _listify(assets: _AssetT) -> List[Asset]: """ Return a list representation of the provided asset(s). :param assets: A collection of assets, one asset, or None. :return: A possibly empty list of assets. """ - if assets is None: return [] - if isinstance(assets, asset): + if isinstance(assets, Asset): return [assets] if isinstance(assets, dict): return list(assets.values()) @@ -347,7 +415,6 @@ def _parse_args(raw: List[str]) -> Namespace: :param args: Raw command-line arguments. :return: Parsed command-line arguments. """ - parser = ArgumentParser(add_help=False, formatter_class=_formatter) parser.add_argument("module", help="application module", type=str) parser.add_argument("function", help="task function", type=str) @@ -355,6 +422,7 @@ def _parse_args(raw: List[str]) -> Namespace: optional = parser.add_argument_group("optional arguments") optional.add_argument("-d", "--dry-run", action="store_true", help="run in dry-run mode") optional.add_argument("-h", "--help", action="help", help="show help and exit") + optional.add_argument("-g", "--graph", action="store_true", help="emit Graphviz dot to stdout") optional.add_argument("-v", "--verbose", action="store_true", help="verbose logging") return parser.parse_args(raw) @@ -376,7 +444,6 @@ def _reify(s: str) -> Any: :param s: The string to convert. :return: A more Pythonic represetnation of the input string. """ - try: return loads(s) except JSONDecodeError: @@ -394,13 +461,38 @@ def _report_readiness( :param is_external: Is this an @external task? :param initial: Is this a initial (i.e. pre-run) readiness report? """ - extmsg = " (EXTERNAL)" if is_external and not ready else "" logf = logging.info if initial or ready else logging.warning logf( - "%s: %s state: %s%s", + "%s: %s: %s%s", taskname, - "Initial" if initial else "Final", + "State" if is_external else "Initial state" if initial else "Final state", "Ready" if ready else "Pending", extmsg, ) + + +def _task_final(taskname: str, assets: _AssetT) -> _AssetT: + """ + Final steps common to all task types. + + :param taskname: The current task's name. + :param assets: A collection of assets, one asset, or None. + :return: The same assets that were provided as input. + """ + for a in _listify(assets): + setattr(a, "taskname", taskname) + return assets + + +def _task_initial(f: Callable, *args, **kwargs) -> Tuple[str, bool, Generator]: + """ + Inital steps common to all task types. + + :param f: A task function (receives the provided args & kwargs). + :return: The task's name, its "top" status, and the generator returned by the task. + """ + top = _i_am_top_task() # Must precede delegation to other tasks! + g = f(*args, **kwargs) + taskname = next(g) + return taskname, top, g diff --git a/src/iotaa/demo.py b/src/iotaa/demo.py index 86aca4f..c6a1909 100644 --- a/src/iotaa/demo.py +++ b/src/iotaa/demo.py @@ -8,77 +8,92 @@ import logging from pathlib import Path -from iotaa import asset, external, ref, task, tasks +from iotaa import asset, external, refs, task, tasks @tasks def a_cup_of_tea(basedir): - yield "A cup of steeped tea with sugar" - cupdir = ref(cup(basedir)) - yield [cup(basedir), steeped_tea_with_sugar(cupdir)] + # A cup of steeped tea with sugar, and a spoon. + yield "The perfect cup of tea" + yield [spoon(basedir), steeped_tea_with_sugar(basedir)] + + +@task +def spoon(basedir): + # A spoon to stir the tea. + path = Path(basedir) / "spoon" + yield "A spoon" + yield asset(path, path.exists) + yield None + path.parent.mkdir(parents=True) + path.touch() @task def cup(basedir): - # Get a cup to make the tea in. + # A cup for the tea. path = Path(basedir) / "cup" - yield f"The cup: {path}" + yield "A cup" yield asset(path, path.exists) yield None path.mkdir(parents=True) @task -def steeped_tea_with_sugar(cupdir): +def steeped_tea_with_sugar(basedir): # Add sugar to the steeped tea. Requires tea to have steeped. - for x in ingredient(cupdir, "sugar", "Steeped tea with sugar", steeped_tea): + for x in ingredient(basedir, "sugar", "Sugar", steeped_tea): yield x @task -def steeped_tea(cupdir): +def steeped_tea(basedir): # Give tea time to steep. - yield f"Steeped tea in {cupdir}" - ready = False - water = ref(steeping_tea(cupdir))["water"] + yield "Steeped tea" + water = refs(steeping_tea(basedir))["water"] + steep_time = lambda x: asset("elapsed time", lambda: x) + t = 10 # seconds if water.exists(): water_poured_time = dt.datetime.fromtimestamp(water.stat().st_mtime) - ready_time = water_poured_time + dt.timedelta(seconds=10) + ready_time = water_poured_time + dt.timedelta(seconds=t) now = dt.datetime.now() ready = now >= ready_time - yield asset(None, lambda: ready) - if not ready: - logging.info("Tea needs to steep for %ss", int((ready_time - now).total_seconds())) + remaining = int((ready_time - now).total_seconds()) + yield steep_time(ready) else: - yield asset(None, lambda: False) - yield steeping_tea(cupdir) + ready = False + remaining = t + yield steep_time(False) + yield steeping_tea(basedir) + if not ready: + logging.warning("Tea needs to steep for %ss", remaining) @task -def steeping_tea(cupdir): - # Pour boiling water over the tea. Requires tea bag in cup. - for x in ingredient(cupdir, "water", "Boiling water over the tea", tea_bag): +def steeping_tea(basedir): + # Pour boiling water over the tea. Requires teabag in cup. + for x in ingredient(basedir, "water", "Boiling water", teabag): yield x @task -def tea_bag(cupdir): - # Place tea bag in the cup. Requires box of tea bags. - for x in ingredient(cupdir, "tea", "Tea bag", box_of_tea_bags): +def teabag(basedir): + # Place tea bag in the cup. Requires box of teabags. + for x in ingredient(basedir, "teabag", "Teabag", box_of_teabags): yield x @external -def box_of_tea_bags(cupdir): - path = Path(cupdir).parent / "box-of-tea" - yield f"Tea from store: {path}" +def box_of_teabags(basedir): + path = Path(basedir) / "box-of-teabags" + yield f"Box of teabags {path}" yield asset(path, path.exists) -def ingredient(cupdir, fn, name, req=None): - path = Path(cupdir) / fn - path.parent.mkdir(parents=True, exist_ok=True) - yield f"{name} in {cupdir}" +def ingredient(basedir, fn, name, req=None): + yield f"{name} in cup" + path = refs(cup(basedir)) / fn yield {fn: asset(path, path.exists)} - yield req(cupdir) if req else None + yield [cup(basedir)] + ([req(basedir)] if req else []) + logging.info("Adding %s to cup", fn) path.touch() diff --git a/src/iotaa/tests/test_iotaa.py b/src/iotaa/tests/test_iotaa.py index b4c4017..dd7abcc 100644 --- a/src/iotaa/tests/test_iotaa.py +++ b/src/iotaa/tests/test_iotaa.py @@ -5,6 +5,7 @@ # pylint: disable=missing-function-docstring,protected-access,redefined-outer-name import re +from hashlib import md5 from unittest.mock import ANY from unittest.mock import DEFAULT as D from unittest.mock import patch @@ -34,6 +35,11 @@ def foo(path): return foo +@fixture +def empty_graph(): + return iotaa.ns(assets={}, edges=set(), tasks=set()) + + @fixture def module_for_main(tmp_path): func = """ @@ -98,17 +104,44 @@ def logged(msg: str, caplog: LogCaptureFixture) -> bool: return any(re.match(r"^%s$" % re.escape(msg), rec.message) for rec in caplog.records) +def simple_assets(): + return [ + None, + iotaa.asset("foo", lambda: True), + [iotaa.asset("foo", lambda: True), iotaa.asset("bar", lambda: True)], + {"baz": iotaa.asset("foo", lambda: True), "qux": iotaa.asset("bar", lambda: True)}, + ] + + # Public API tests @pytest.mark.parametrize( - "asset", [iotaa.asset("foo", lambda: True), iotaa.asset(ref="foo", ready=lambda: True)] + # One without kwargs, one with: + "asset", + [iotaa.asset("foo", lambda: True), iotaa.asset(ref="foo", ready=lambda: True)], ) -def test_asset(asset): +def test_Asset(asset): assert asset.ref == "foo" assert asset.ready() +@pytest.mark.parametrize( + # One without kwargs, one with: + "result", + [iotaa.Result("foo", True), iotaa.Result(output="foo", success=True)], +) +def test_Result(result): + assert result.output == "foo" + assert result.success + + +def test_asset_kwargs(): + a = iotaa.asset(ref="foo", ready=lambda: True) + assert a.ref == "foo" + assert a.ready() + + def test_dryrun(): with patch.object(iotaa, "_state", iotaa.ns(dry_run=False)): assert not iotaa._state.dry_run @@ -144,12 +177,15 @@ def test_main_mocked_up(tmp_path): m = tmp_path / "a.py" m.touch() strs = ["foo", "88", "3.14", "true"] - with patch.multiple(iotaa, _parse_args=D, dryrun=D, import_module=D, logcfg=D) as mocks: + with patch.multiple( + iotaa, _graph_emit=D, _parse_args=D, dryrun=D, import_module=D, logcfg=D + ) as mocks: parse_args = mocks["_parse_args"] parse_args.return_value = iotaa.Namespace( args=strs, dry_run=True, function="a_function", + graph=True, module=m, verbose=True, ) @@ -161,16 +197,17 @@ def test_main_mocked_up(tmp_path): getattr_().assert_called_once_with("foo", 88, 3.14, True) mocks["dryrun"].assert_called_once() mocks["logcfg"].assert_called_once_with(verbose=True) + mocks["_graph_emit"].assert_called_once_with() parse_args.assert_called_once() -def test_ref_dict(): +def test_refs_dict(): expected = "bar" asset = iotaa.asset(ref="bar", ready=lambda: True) - assert iotaa.ref(assets={"foo": asset})["foo"] == expected - assert iotaa.ref(assets=[asset])[0] == expected - assert iotaa.ref(assets=asset) == expected - assert iotaa.ref(assets=None) is None + assert iotaa.refs(assets={"foo": asset})["foo"] == expected + assert iotaa.refs(assets=[asset])[0] == expected + assert iotaa.refs(assets=asset) == expected + assert iotaa.refs(assets=None) is None def test_run_failure(caplog): @@ -219,7 +256,7 @@ def test_external_not_ready(external_foo_scalar, tmp_path): f = tmp_path / "foo" assert not f.is_file() assets = list(iotaa._listify(external_foo_scalar(tmp_path))) - assert iotaa.ref(assets)[0] == f + assert iotaa.refs(assets)[0] == f assert not assets[0].ready() @@ -228,7 +265,7 @@ def test_external_ready(external_foo_scalar, tmp_path): f.touch() assert f.is_file() asset = external_foo_scalar(tmp_path) - assert iotaa.ref(asset) == f + assert iotaa.refs(asset) == f assert asset.ready() @@ -237,10 +274,10 @@ def test_task_not_ready(caplog, task_bar_dict, tmp_path): f_foo, f_bar = (tmp_path / x for x in ["foo", "bar"]) assert not any(x.is_file() for x in [f_foo, f_bar]) assets = list(iotaa._listify(task_bar_dict(tmp_path))) - assert iotaa.ref(assets)[0] == f_bar + assert iotaa.refs(assets)[0] == f_bar assert not assets[0].ready() assert not any(x.is_file() for x in [f_foo, f_bar]) - assert logged(f"task bar {f_bar}: Pending", caplog) + assert logged(f"task bar {f_bar}: Requirement(s) pending", caplog) def test_task_ready(caplog, task_bar_list, tmp_path): @@ -250,18 +287,20 @@ def test_task_ready(caplog, task_bar_list, tmp_path): assert f_foo.is_file() assert not f_bar.is_file() assets = list(iotaa._listify(task_bar_list(tmp_path))) - assert iotaa.ref(assets)[0] == f_bar + assert iotaa.refs(assets)[0] == f_bar assert assets[0].ready() assert all(x.is_file for x in [f_foo, f_bar]) - assert logged(f"task bar {f_bar}: Ready", caplog) + assert logged(f"task bar {f_bar}: Requirement(s) ready", caplog) def test_tasks_not_ready(tasks_baz, tmp_path): f_foo, f_bar = (tmp_path / x for x in ["foo", "bar"]) assert not any(x.is_file() for x in [f_foo, f_bar]) - assets = list(iotaa._listify(tasks_baz(tmp_path))) - assert iotaa.ref(assets)[0] == f_foo - assert iotaa.ref(assets)[1] == f_bar + with patch.object(iotaa, "_state") as _state: + _state.initialized = False + assets = list(iotaa._listify(tasks_baz(tmp_path))) + assert iotaa.refs(assets)[0] == f_foo + assert iotaa.refs(assets)[1] == f_bar assert not any(x.ready() for x in assets) assert not any(x.is_file() for x in [f_foo, f_bar]) @@ -272,8 +311,8 @@ def test_tasks_ready(tasks_baz, tmp_path): assert f_foo.is_file() assert not f_bar.is_file() assets = list(iotaa._listify(tasks_baz(tmp_path))) - assert iotaa.ref(assets)[0] == f_foo - assert iotaa.ref(assets)[1] == f_bar + assert iotaa.refs(assets)[0] == f_foo + assert iotaa.refs(assets)[1] == f_bar assert all(x.ready() for x in assets) assert all(x.is_file() for x in [f_foo, f_bar]) @@ -288,7 +327,7 @@ def f(): yield None assert not iotaa._delegate(f(), "task") - assert logged("task: Checking required tasks", caplog) + assert logged("task: Checking requirements", caplog) def test__delegate_scalar(caplog, delegate_assets): @@ -298,8 +337,10 @@ def test__delegate_scalar(caplog, delegate_assets): def f(): yield a1 - assert iotaa._delegate(f(), "task") == [a1] - assert logged("task: Checking required tasks", caplog) + with patch.object(iotaa, "_graph_udpate_from_requirements") as gufr: + assert iotaa._delegate(f(), "task") == [a1] + gufr.assert_called_once_with("task", [a1]) + assert logged("task: Checking requirements", caplog) def test__delegate_empty_dict_and_empty_list(caplog): @@ -308,8 +349,10 @@ def test__delegate_empty_dict_and_empty_list(caplog): def f(): yield [{}, []] - assert not iotaa._delegate(f(), "task") - assert logged("task: Checking required tasks", caplog) + with patch.object(iotaa, "_graph_udpate_from_requirements") as gufr: + assert not iotaa._delegate(f(), "task") + gufr.assert_called_once_with("task", []) + assert logged("task: Checking requirements", caplog) def test__delegate_dict_and_list_of_assets(caplog, delegate_assets): @@ -319,8 +362,10 @@ def test__delegate_dict_and_list_of_assets(caplog, delegate_assets): def f(): yield [{"foo": a1, "bar": a2}, [a3, a4]] - assert iotaa._delegate(f(), "task") == [a1, a2, a3, a4] - assert logged("task: Checking required tasks", caplog) + with patch.object(iotaa, "_graph_udpate_from_requirements") as gufr: + assert iotaa._delegate(f(), "task") == [a1, a2, a3, a4] + gufr.assert_called_once_with("task", [a1, a2, a3, a4]) + assert logged("task: Checking requirements", caplog) def test__delegate_none_and_scalar(caplog, delegate_assets): @@ -330,14 +375,16 @@ def test__delegate_none_and_scalar(caplog, delegate_assets): def f(): yield [None, a1] - assert iotaa._delegate(f(), "task") == [a1] - assert logged("task: Checking required tasks", caplog) + with patch.object(iotaa, "_graph_udpate_from_requirements") as gufr: + assert iotaa._delegate(f(), "task") == [a1] + gufr.assert_called_once_with("task", [a1]) + assert logged("task: Checking requirements", caplog) def test__execute_dry_run(caplog, rungen): with patch.object(iotaa, "_state", new=iotaa.ns(dry_run=True)): iotaa._execute(g=rungen, taskname="task") - assert logged("task: SKIPPING (DRY RUN ENABLED)", caplog) + assert logged("task: SKIPPING (DRY RUN)", caplog) def test__execute_live(caplog, rungen): @@ -351,6 +398,59 @@ def test__formatter(): assert formatter._prog == "foo" +def test__graph_emit(capsys): + assets = {"foo": lambda: True, "bar": lambda: False} # foo ready, bar pending + edges = {("qux", "baz"), ("baz", "foo"), ("baz", "bar")} + tasks = {"qux", "baz"} + with patch.object(iotaa, "_graph", iotaa.ns(assets=assets, edges=edges, tasks=tasks)): + iotaa._graph_emit() + out = capsys.readouterr().out.strip().split("\n") + # How many asset nodes were graphed? + assert 2 == len([x for x in out if "shape=%s," % iotaa._graph_shape.asset in x]) + # How many task nodes were graphed? + assert 2 == len([x for x in out if "shape=%s," % iotaa._graph_shape.task in x]) + # How many edges were graphed? + assert 3 == len([x for x in out if " -> " in x]) + # How many assets were ready? + assert 1 == len([x for x in out if "fillcolor=%s," % iotaa._graph_color[True] in x]) + # How many assets were pending? + assert 1 == len([x for x in out if "fillcolor=%s," % iotaa._graph_color[False] in x]) + + +def test__graph_name(): + name = "foo" + assert iotaa._graph_name(name) == "_%s" % md5(name.encode("utf-8")).hexdigest() + + +@pytest.mark.parametrize("assets", simple_assets()) +def test__graph_update_from_requirements(assets, empty_graph): + taskname_req = "req" + taskname_this = "task" + alist = iotaa._listify(assets) + edges = { + 0: set(), + 1: {(taskname_this, taskname_req), (taskname_req, "foo")}, + 2: {(taskname_this, taskname_req), (taskname_req, "foo"), (taskname_req, "bar")}, + }[len(alist)] + for a in alist: + setattr(a, "taskname", taskname_req) + with patch.object(iotaa, "_graph", empty_graph): + iotaa._graph_udpate_from_requirements(taskname_this, alist) + assert all(a() for a in iotaa._graph.assets.values()) + assert iotaa._graph.tasks == ({taskname_req, taskname_this} if assets else {taskname_this}) + assert iotaa._graph.edges == edges + + +@pytest.mark.parametrize("assets", simple_assets()) +def test__graph_update_from_task(assets, empty_graph): + taskname = "task" + with patch.object(iotaa, "_graph", empty_graph): + iotaa._graph_update_from_task(taskname, assets) + assert all(a() for a in iotaa._graph.assets.values()) + assert iotaa._graph.tasks == {taskname} + assert iotaa._graph.edges == {(taskname, x.ref) for x in iotaa._listify(assets)} + + @pytest.mark.parametrize("val", [True, False]) def test__i_am_top_task(val): with patch.object(iotaa, "_state", new=iotaa.ns(initialized=not val)): @@ -366,22 +466,25 @@ def test__listify(): def test__parse_args(): - # Specifying module, function, and two args (standard logging): + # Specifying module, function, and two args (standard logging, no graph): a0 = iotaa._parse_args(raw="a_module a_function arg1 arg2".split(" ")) assert a0.module == "a_module" assert a0.function == "a_function" + assert a0.graph is False assert a0.args == ["arg1", "arg2"] assert a0.verbose is False - # Specifying module, function, two args (verbose logging): + # Specifying module, function, two args (verbose logging, no graph): a1 = iotaa._parse_args(raw="a_module a_function arg1 arg2 --verbose".split(" ")) assert a1.module == "a_module" assert a1.function == "a_function" + assert a1.graph is False assert a1.args == ["arg1", "arg2"] assert a1.verbose is True # Specifying module, function, but no args (standard logging): - a2 = iotaa._parse_args(raw="a_module a_function".split(" ")) + a2 = iotaa._parse_args(raw="a_module a_function --graph".split(" ")) assert a2.module == "a_module" assert a2.function == "a_function" + assert a2.graph is True assert a2.args == [] assert a2.verbose is False # It is an error to specify just a module with no function: @@ -411,7 +514,7 @@ def test__reify(): "vals", [ (True, False, True, "Initial state: Ready"), - (False, True, False, "Final state: Pending (EXTERNAL)"), + (False, True, False, "State: Pending (EXTERNAL)"), ], ) def test__report_readiness(caplog, vals): @@ -419,3 +522,25 @@ def test__report_readiness(caplog, vals): iotaa.logging.getLogger().setLevel(iotaa.logging.INFO) iotaa._report_readiness(ready=ready, taskname="task", is_external=ext, initial=init) assert logged(f"task: {msg}", caplog) + + +@pytest.mark.parametrize("assets", simple_assets()) +def test__task_final(assets): + for a in iotaa._listify(assets): + assert getattr(a, "taskname", None) is None + assets = iotaa._task_final("task", assets) + for a in iotaa._listify(assets): + assert getattr(a, "taskname") == "task" + + +def test__task_inital(): + def f(taskname, n): + yield taskname + yield n + + with patch.object(iotaa, "_state", iotaa.ns(initialized=False)): + tn = "task" + taskname, top, g = iotaa._task_initial(f, tn, n=88) + assert taskname == tn + assert top is True + assert next(g) == 88