diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml index 66f08ce61ac..5776185557e 100644 --- a/.github/workflows/test_fast.yml +++ b/.github/workflows/test_fast.yml @@ -140,7 +140,7 @@ jobs: uses: actions/download-artifact@v4 - name: Codecov upload - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: name: ${{ github.workflow }} flags: fast-tests diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml index ed411c2eae8..f54f3b3d710 100644 --- a/.github/workflows/test_functional.yml +++ b/.github/workflows/test_functional.yml @@ -316,7 +316,7 @@ jobs: uses: actions/download-artifact@v4 - name: Codecov upload - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: name: ${{ github.workflow }} flags: functional-tests diff --git a/README.md b/README.md index bc28920d48f..e97a7731ab4 100644 --- a/README.md +++ b/README.md @@ -29,28 +29,14 @@ domains. # install cylc conda install cylc-flow -# write your first workflow -mkdir -p ~/cylc-src/example -cat > ~/cylc-src/example/flow.cylc <<__CONFIG__ -[scheduling] - initial cycle point = 1 - cycling mode = integer - [[graph]] - P1 = """ - a => b => c & d - b[-P1] => b - """ -[runtime] - [[a, b, c, d]] - script = echo "Hello $CYLC_TASK_NAME" -__CONFIG__ +# extract an example to run +cylc get-resources examples/integer-cycling # install and run it -cylc install example -cylc play example +cylc vip integer-cycling # vip = validate, install and play # watch it run -cylc tui example +cylc tui integer-cycling ``` ### The Cylc Ecosystem diff --git a/changes.d/5864.feat.md b/changes.d/5864.feat.md new file mode 100644 index 00000000000..905b6b9dadd --- /dev/null +++ b/changes.d/5864.feat.md @@ -0,0 +1 @@ +Reimplemented the `suite-state` xtrigger for interoperability with Cylc 7. diff --git a/changes.d/5873.feat.md b/changes.d/5873.feat.md new file mode 100644 index 00000000000..0b718922607 --- /dev/null +++ b/changes.d/5873.feat.md @@ -0,0 +1,3 @@ +- Allow use of `#noqa: S001` comments to skip Cylc lint + checks for a single line. +- Stop Cylc lint objecting to `%include ` syntax. \ No newline at end of file diff --git a/changes.d/5933.fix.md b/changes.d/5933.fix.md new file mode 100644 index 00000000000..8ba0546c760 --- /dev/null +++ b/changes.d/5933.fix.md @@ -0,0 +1 @@ +Fixed bug in `cylc broadcast` (and the GUI Edit Runtime command) where everything after a `#` character in a setting would be stripped out. diff --git a/changes.d/5943.feat.md b/changes.d/5943.feat.md new file mode 100644 index 00000000000..6db31d952be --- /dev/null +++ b/changes.d/5943.feat.md @@ -0,0 +1 @@ +The `stop after cycle point` can now be specified as an offset from the inital cycle point. diff --git a/changes.d/5956.break.md b/changes.d/5956.break.md new file mode 100644 index 00000000000..642c805814d --- /dev/null +++ b/changes.d/5956.break.md @@ -0,0 +1 @@ +`cylc lint`: deprecated `[cylc-lint]` section in favour of `[tool.cylc.lint]` in `pyproject.toml` diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 2d39ad74829..713db726660 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -165,9 +165,17 @@ Template variables can be used to configure handlers. For a full list of supported variables see :ref:`workflow_event_template_variables`. + + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` ''', 'handler events': ''' Specify the events for which workflow event handlers should be invoked. + + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` ''', 'mail events': ''' Specify the workflow events for which notification emails should @@ -176,6 +184,10 @@ 'startup handlers': f''' Handlers to run at scheduler startup. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``startup handler``. @@ -183,6 +195,10 @@ 'shutdown handlers': f''' Handlers to run at scheduler shutdown. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``shutdown handler``. @@ -191,6 +207,10 @@ Handlers to run if the scheduler shuts down with error status due to a configured timeout or a fatal error condition. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``aborted handler``. @@ -199,22 +219,38 @@ Workflow timeout interval. The timer starts counting down at scheduler startup. It resets on workflow restart. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionadded:: 8.0.0 ''', 'workflow timeout handlers': ''' Handlers to run if the workflow timer times out. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionadded:: 8.0.0 ''', 'abort on workflow timeout': ''' Whether the scheduler should shut down immediately with error status if the workflow timer times out. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionadded:: 8.0.0 ''', 'stall handlers': f''' Handlers to run if the scheduler stalls. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``stalled handler``. @@ -222,6 +258,10 @@ 'stall timeout': f''' The length of a timer which starts if the scheduler stalls. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``timeout``. @@ -229,6 +269,10 @@ 'stall timeout handlers': f''' Handlers to run if the stall timer times out. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``timeout handler``. @@ -237,6 +281,10 @@ Whether the scheduler should shut down immediately with error status if the stall timer times out. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``abort on timeout``. @@ -245,6 +293,10 @@ Scheduler inactivity timeout interval. The timer resets when any workflow activity occurs. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES} ``inactivity``. @@ -252,6 +304,10 @@ 'inactivity timeout handlers': f''' Handlers to run if the inactivity timer times out. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``inactivity handler``. @@ -260,6 +316,10 @@ Whether the scheduler should shut down immediately with error status if the inactivity timer times out. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionchanged:: 8.0.0 {REPLACES}``abort on inactivity``. @@ -268,6 +328,10 @@ How long to wait for intervention on restarting a completed workflow. The timer stops if any task is triggered. + .. seealso:: + + :ref:`user_guide.scheduler.workflow_events` + .. versionadded:: 8.2.0 ''' diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index de919c27c0f..6a951ae02fb 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -539,7 +539,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): ''') # NOTE: final cycle point is not a V_CYCLE_POINT to allow expressions # such as '+P1Y' (relative to initial cycle point) - Conf('final cycle point', VDR.V_STRING, desc=''' + Conf('final cycle point', VDR.V_CYCLE_POINT_WITH_OFFSETS, desc=''' The (optional) last cycle point at which tasks are run. Once all tasks have reached this cycle point, the @@ -547,6 +547,12 @@ def get_script_common_text(this: str, example: Optional[str] = None): This item can be overridden on the command line using ``cylc play --final-cycle-point`` or ``--fcp``. + + Examples: + + - ``2000`` - Shorthand for ``2000-01-01T00:00``. + - ``+P1D`` - The initial cycle point plus one day. + - ``2000 +P1D +P1Y`` - The year ``2000`` plus one day and one year. ''') Conf('initial cycle point constraints', VDR.V_STRING_LIST, desc=''' Rules to allow only some initial datetime cycle points. @@ -599,7 +605,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): {REPLACES}``[scheduling]hold after point``. ''') - Conf('stop after cycle point', VDR.V_CYCLE_POINT, desc=''' + Conf('stop after cycle point', VDR.V_CYCLE_POINT_WITH_OFFSETS, desc=''' Shut down the workflow after all tasks pass this cycle point. The stop cycle point can be overridden on the command line using @@ -612,7 +618,18 @@ def get_script_common_text(this: str, example: Optional[str] = None): choosing not to run that part of the graph. You can play the workflow and continue. + Examples: + + - ``2000`` - Shorthand for ``2000-01-01T00:00``. + - ``+P1D`` - The initial cycle point plus one day. + - ``2000 +P1D +P1Y`` - The year ``2000`` plus one day and one year. + .. versionadded:: 8.0.0 + + .. versionchanged:: 8.3.0 + + This now supports offsets (e.g. ``+P1D``) in the same way the + :cylc:conf:`[..]final cycle point` does. ''') Conf('cycling mode', VDR.V_STRING, Calendar.MODE_GREGORIAN, options=list(Calendar.MODES) + ['integer'], desc=''' diff --git a/cylc/flow/config.py b/cylc/flow/config.py index d80456266bf..76c8b4e1980 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -851,9 +851,16 @@ def process_stop_cycle_point(self) -> None: if stopcp_str is None: stopcp_str = self.cfg['scheduling']['stop after cycle point'] - if stopcp_str is not None: - self.stop_point = get_point(stopcp_str).standardise() - if self.final_point and (self.stop_point > self.final_point): + if stopcp_str: + self.stop_point = get_point_relative( + stopcp_str, + self.initial_point, + ).standardise() + if ( + self.final_point is not None + and self.stop_point is not None + and self.stop_point > self.final_point + ): LOG.warning( f"Stop cycle point '{self.stop_point}' will have no " "effect as it is after the final cycle " diff --git a/cylc/flow/etc/examples/1-hello-world/.validate b/cylc/flow/etc/examples/1-hello-world/.validate new file mode 100755 index 00000000000..968a7b349cb --- /dev/null +++ b/cylc/flow/etc/examples/1-hello-world/.validate @@ -0,0 +1,23 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip --check-circular --no-detach --no-run-name --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/1-hello-world/flow.cylc b/cylc/flow/etc/examples/1-hello-world/flow.cylc new file mode 100644 index 00000000000..92510d1928b --- /dev/null +++ b/cylc/flow/etc/examples/1-hello-world/flow.cylc @@ -0,0 +1,13 @@ +[meta] + title = Hello World + description = """ + A simple workflow which runs a single task (hello_world) once. + """ + +[scheduling] + [[graph]] + R1 = hello_world + +[runtime] + [[hello_world]] + script = echo "Hello World!" diff --git a/cylc/flow/etc/examples/1-hello-world/index.rst b/cylc/flow/etc/examples/1-hello-world/index.rst new file mode 100644 index 00000000000..a73afdf1e44 --- /dev/null +++ b/cylc/flow/etc/examples/1-hello-world/index.rst @@ -0,0 +1,21 @@ +Hello World +----------- + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/hello-world + +In the time honoured tradition, this is the minimal Cylc workflow: + +.. literalinclude:: flow.cylc + :language: cylc + +It writes the phrase "Hello World!" to standard output (captured to the +``job.out`` log file). + +Run it with:: + + $ cylc vip hello-world diff --git a/cylc/flow/etc/examples/2-integer-cycling/.validate b/cylc/flow/etc/examples/2-integer-cycling/.validate new file mode 100755 index 00000000000..054f2662862 --- /dev/null +++ b/cylc/flow/etc/examples/2-integer-cycling/.validate @@ -0,0 +1,23 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip --check-circular --no-detach --no-run-name --final-cycle-point=1 --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/2-integer-cycling/flow.cylc b/cylc/flow/etc/examples/2-integer-cycling/flow.cylc new file mode 100644 index 00000000000..b8d2f2f05ce --- /dev/null +++ b/cylc/flow/etc/examples/2-integer-cycling/flow.cylc @@ -0,0 +1,28 @@ +[meta] + title = Integer Cycling + description = """ + A basic cycling workflow which runs the same set of tasks over + and over. Each cycle will be given an integer number. + """ + +[scheduling] + # tell Cylc to count cycles as numbers starting from the number 1 + cycling mode = integer + initial cycle point = 1 + [[graph]] + P1 = """ + # this is the workflow we want to repeat: + a => b => c & d + # this is an "inter-cycle dependency", it makes the task "b" + # wait until its previous instance has completed: + b[-P1] => b + """ + +[runtime] + [[root]] + # all tasks will "inherit" the configuration in the "root" section + script = echo "Hello, I'm task $CYLC_TASK_NAME in cycle $CYLC_TASK_CYCLE_POINT!" + [[a]] + [[b]] + [[c]] + [[d]] diff --git a/cylc/flow/etc/examples/2-integer-cycling/index.rst b/cylc/flow/etc/examples/2-integer-cycling/index.rst new file mode 100644 index 00000000000..89be2bb0871 --- /dev/null +++ b/cylc/flow/etc/examples/2-integer-cycling/index.rst @@ -0,0 +1,16 @@ +Integer Cycling +=============== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/integer-cycling + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip integer-cycling diff --git a/cylc/flow/etc/examples/3-datetime-cycling/.validate b/cylc/flow/etc/examples/3-datetime-cycling/.validate new file mode 100755 index 00000000000..32c2a51908b --- /dev/null +++ b/cylc/flow/etc/examples/3-datetime-cycling/.validate @@ -0,0 +1,28 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc vip \ + --check-circular \ + --no-detach \ + --no-run-name \ + --final-cycle-point "$(isodatetime now --format 'CCYYMMDD')T00" \ + --workflow-name "$ID" +cylc lint "$ID" +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/3-datetime-cycling/flow.cylc b/cylc/flow/etc/examples/3-datetime-cycling/flow.cylc new file mode 100644 index 00000000000..76945349987 --- /dev/null +++ b/cylc/flow/etc/examples/3-datetime-cycling/flow.cylc @@ -0,0 +1,44 @@ +[meta] + title = Datetime Cycling + description = """ + A basic cycling workflow which runs the same set of tasks over + and over. Each cycle will be given a datetime identifier. + + The task "a" will wait until the real-world (or wallclock) time passes + the cycle time. + + Try changing the "initial cycle point" to "previous(00T00) - P1D" to + see how this works. + """ + +[scheduling] + # set the start of the graph to 00:00 this morning + initial cycle point = previous(T00) + + [[graph]] + # repeat this with a "P"eriod of "1" "D"ay -> P1D + P1D = """ + # this is the workflow we want to repeat: + a => b => c & d + + # this is an "inter-cycle dependency", it makes the task "b" + # wait until its previous instance has successfully completed: + b[-P1D] => b + + # this makes the task "a" wait until its cycle point matches + # the real world time - i.e. it prevents the workflow from getting + # ahead of the clock. If the workflow is running behind (e.g. after + # a delay, or from an earlier initial cycle point) it will catch + # until the clock-trigger constrains it again. To run entirely in + # "simulated time" remove this line: + @wall_clock => a + """ + +[runtime] + [[root]] + # all tasks will "inherit" the configuration in the "root" section + script = echo "Hello, I'm task $CYLC_TASK_NAME in cycle $CYLC_TASK_CYCLE_POINT!" + [[a]] + [[b]] + [[c]] + [[d]] diff --git a/cylc/flow/etc/examples/3-datetime-cycling/index.rst b/cylc/flow/etc/examples/3-datetime-cycling/index.rst new file mode 100644 index 00000000000..803a532551f --- /dev/null +++ b/cylc/flow/etc/examples/3-datetime-cycling/index.rst @@ -0,0 +1,16 @@ +Datetime Cycling +================ + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/datetime-cycling + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip datetime-cycling diff --git a/cylc/flow/etc/examples/README.md b/cylc/flow/etc/examples/README.md new file mode 100644 index 00000000000..ae4e1391d23 --- /dev/null +++ b/cylc/flow/etc/examples/README.md @@ -0,0 +1,27 @@ +# Examples + +These examples are intended to illustrate the major patterns for implementing +Cylc workflows. The hope is that users can find a workflow which fits their +pattern, make a copy and fill in the details. Keep the examples minimal and +abstract. We aren't trying to document every Cylc feature here, just the +major design patterns. + +These examples are auto-documented in cylc-doc which looks for an `index.rst` +file in each example. + +Users can extract them using `cylc get-resources` which will put them into the +configured Cylc source directory (`~/cylc-src` by default). They can then be +run using the directory name, e.g. `cylc vip hello-world`. + +Files: + +* `index.rst` + This file is used to generate a page in the documentation for the example. + This file is excluded when the user extracts the example. +* `.validate` + This is a test file, it gets detected and run automatically. + This file is excluded when the user extracts the example. +* `README.rst` + Examples can include a README file, to save duplication, you can + `.. include::` this in the `index.rst` file (hence using ReStructuredText + rather than Markdown). diff --git a/cylc/flow/etc/examples/converging-workflow/.validate b/cylc/flow/etc/examples/converging-workflow/.validate new file mode 100755 index 00000000000..ad61b518ab2 --- /dev/null +++ b/cylc/flow/etc/examples/converging-workflow/.validate @@ -0,0 +1,33 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" + +# start the workflow +cylc vip --check-circular --no-run-name --no-detach --workflow-name "$ID" + +# it should have reached the forth cycle +test -d "${HOME}/cylc-run/${ID}/log/job/4" +test ! -d "${HOME}/cylc-run/${ID}/log/job/5" + +# lint +cylc lint "$ID" + +# clean up +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/converging-workflow/flow.cylc b/cylc/flow/etc/examples/converging-workflow/flow.cylc new file mode 100644 index 00000000000..b2069a3f479 --- /dev/null +++ b/cylc/flow/etc/examples/converging-workflow/flow.cylc @@ -0,0 +1,35 @@ +[meta] + title = Converging Workflow + description = """ + A workflow which runs a pattern of tasks over and over until a + convergence condition has been met. + """ + +[scheduling] + cycling mode = integer + initial cycle point = 1 + [[graph]] + P1 = """ + # run "increment" then check the convergence condition + check_convergence[-P1]:not_converged? => increment => check_convergence + + # if the workflow has converged, then do nothing + check_convergence:converged? + """ + +[runtime] + [[increment]] + # a task which evolves the data + [[check_convergence]] + # a task which checks whether the convergence condition has been met + script = """ + if (( CYLC_TASK_CYCLE_POINT == 4 )); then + # for the purpose of example, assume convergence at cycle point 4 + cylc message -- 'convergence condition met' + else + cylc message -- 'convergence condition not met' + fi + """ + [[[outputs]]] + converged = 'convergence condition met' + not_converged = 'convergence condition not met' diff --git a/cylc/flow/etc/examples/converging-workflow/index.rst b/cylc/flow/etc/examples/converging-workflow/index.rst new file mode 100644 index 00000000000..27da079d58f --- /dev/null +++ b/cylc/flow/etc/examples/converging-workflow/index.rst @@ -0,0 +1,39 @@ +Converging Workflow +=================== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/converging-workflow + +A workflow which runs a pattern of tasks over and over until a convergence +condition has been met. + +* The ``increment`` task runs some kind of model or process which increments + us toward the solution. +* The ``check_convergence`` task, checks if the convergence condition has been + met. + +.. literalinclude:: flow.cylc + :language: cylc + +Run it with:: + + $ cylc vip converging-workflow + +.. admonition:: Example - Genetic algorithms + :class: hint + + .. _genetic algorithm: https://en.wikipedia.org/wiki/Genetic_algorithm + + An example of a converging workflow might be a `genetic algorithm`_, where you + "breed" entities, then test their "fitness", and breed again, over and over + until you end up with an entity which is able to satisfy the requirement. + + .. digraph:: Example + + random_seed -> breed -> test_fitness + test_fitness -> breed + test_fitness -> stop diff --git a/cylc/flow/etc/examples/event-driven-cycling/.validate b/cylc/flow/etc/examples/event-driven-cycling/.validate new file mode 100755 index 00000000000..ef224cd9e2c --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/.validate @@ -0,0 +1,47 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" + +# start the workflow +cylc vip --check-circular --no-run-name --workflow-name "$ID" +sleep 1 # give it a reasonable chance to start up + +# kick off the first cycle +./bin/trigger "$ID" WORLD=earth + +# wait for it to complete +cylc workflow-state "$ID" \ + --task=run \ + --point=1 \ + --status=succeeded \ + --max-polls=60 \ + --interval=1 + +# check the job received the environment variable we provided +grep 'Hello earth' "$HOME/cylc-run/$ID/log/job/1/run/NN/job.out" + +# stop the workflow +cylc stop --kill --max-polls=10 --interval=2 "$ID" + +# lint +cylc lint "$ID" + +# clean up +cylc clean "$ID" diff --git a/cylc/flow/etc/examples/event-driven-cycling/README.rst b/cylc/flow/etc/examples/event-driven-cycling/README.rst new file mode 100644 index 00000000000..d1e112374f1 --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/README.rst @@ -0,0 +1,49 @@ +Cylc is good at orchestrating tasks to a schedule, e.g: + +* ``PT1H`` - every hour +* ``P1D`` - every day +* ``P1M`` - every month +* ``PT1H ! (T00, T12)`` - every hour, except midnight and midday. + +But sometimes the things you want to run don't have a schedule. + +This example uses ``cylc ext-trigger`` to establish a pattern where Cylc waits +for an external signal and starts a new cycle every time a signal is received. + +The signal can carry data using the ext-trigger ID, this example sets the ID +as a file path containing some data that we want to make available to the tasks +that run in the cycle it triggers. + +To use this example, first start the workflow as normal:: + + cylc vip event-driven-cycling + +Then, when you're ready, kick off a new cycle, specifying any +environment variables you want to configure this cycle with:: + + ./bin/trigger WORLD=earth + +Replacing ```` with the ID you installed this workflow as. + +.. admonition:: Example - CI/CD + :class: hint + + This pattern is good for CI/CD type workflows where you're waiting on + external events. This pattern is especially powerful when used with + sub-workflows where it provides a solution to two-dimensional cycling + problems. + +.. admonition:: Example - Polar satellite data processing + :class: hint + + Polar satellites pass overhead at irregular intervals. This makes it tricky + to schedule data processing because you don't know when the satellite will + pass over the receiver station. With the event driven cycling approach you + could start a new cycle every time data arrives. + +.. note:: + + * The number of parallel cycles can be adjusted by changing the + :cylc:conf:`[scheduling]runahead limit`. + * To avoid hitting the runahead limit, ensure that failures are handled in + the graph. diff --git a/cylc/flow/etc/examples/event-driven-cycling/bin/trigger b/cylc/flow/etc/examples/event-driven-cycling/bin/trigger new file mode 100755 index 00000000000..9f917f35f5c --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/bin/trigger @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eu + +if [[ $# -lt 1 ]]; then + echo 'Usage ./trigger WORKFLOW_ID [KEY=VALUE ...]' >&2 + echo + echo 'Trigger a new cycle in the target workflow.' + echo 'Any environment variable KEY=VALUE pairs will be broadcasted to' + echo 'all tasks in the cycle.' + exit 1 +fi + +# determine the workflow +WORKFLOW_ID="$1" +shift +WORKFLOW_RUN_DIR="${HOME}/cylc-run/${WORKFLOW_ID}" +EXT_TRIGGER_DIR="${WORKFLOW_RUN_DIR}/triggers" +mkdir -p "$EXT_TRIGGER_DIR" + +# pick a trigger-id +TRIGGER_ID="$(isodatetime --print-format CCYYMMDDThhmmss)" + +# write environment variables to a broadcast file +TRIGGER_FILE="${EXT_TRIGGER_DIR}/${TRIGGER_ID}.cylc" +echo '[environment]' >"$TRIGGER_FILE" +for env in "$@"; do + echo " $env" >> "$TRIGGER_FILE" +done + +# issue the xtrigger +cylc ext-trigger "$WORKFLOW_ID" 'trigger' "$TRIGGER_ID" diff --git a/cylc/flow/etc/examples/event-driven-cycling/flow.cylc b/cylc/flow/etc/examples/event-driven-cycling/flow.cylc new file mode 100644 index 00000000000..7711c2ba258 --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/flow.cylc @@ -0,0 +1,32 @@ +[scheduling] + cycling mode = integer + initial cycle point = 1 + runahead limit = P5 # max number of cycles which can run in parallel + [[special tasks]] + # register the external trigger, it must be given a name, + # here, 'trigger' is used as a placeholder, the bash script will + # need to be updated if this is changed + external-trigger = configure("trigger") + [[graph]] + P1 = """ + # use a "?" to prevent failures causing runahead stalls + configure? => run + """ + +[runtime] + [[configure]] + # this task reads in the broadcast file the trigger wrote + # and broadcasts any variables set to all tasks in this cycle + script = """ + echo "received new ext-trigger ID=$CYLC_EXT_TRIGGER_ID" + TRIGGER_FILE="${CYLC_WORKFLOW_RUN_DIR}/triggers/${CYLC_EXT_TRIGGER_ID}.cylc" + cylc broadcast "${CYLC_WORKFLOW_ID}" \ + -p "${CYLC_TASK_CYCLE_POINT}" \ + -F "${TRIGGER_FILE}" + """ + + [[run]] + # this task could be a sub-workflow + script = """ + echo "Hello $WORLD!" + """ diff --git a/cylc/flow/etc/examples/event-driven-cycling/index.rst b/cylc/flow/etc/examples/event-driven-cycling/index.rst new file mode 100644 index 00000000000..5980591f0fd --- /dev/null +++ b/cylc/flow/etc/examples/event-driven-cycling/index.rst @@ -0,0 +1,14 @@ +Event Driven Cycling +==================== + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/event-driven-cycling + +.. include:: README.rst + +.. literalinclude:: flow.cylc + :language: cylc diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/.validate b/cylc/flow/etc/examples/inter-workflow-triggers/.validate new file mode 100755 index 00000000000..bdd414e275d --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/.validate @@ -0,0 +1,57 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +UPID=inter-workflow-triggers/upstream +DOID=inter-workflow-triggers/downstream + +ICP="$(isodatetime now --format=CCYYMMDDThh --offset=-PT2H)" + +# run the workflows +cylc vip \ + --check-circular \ + --no-run-name \ + --final-cycle-point="$ICP" \ + --workflow-name "$UPID" \ + ./upstream +cylc vip \ + --check-circular \ + --no-run-name \ + --final-cycle-point="$ICP" \ + --workflow-name "$DOID" \ + ./downstream + +# wait for the first task in the downstream to succeed +cylc workflow-state "$DOID" \ + --task=process \ + --point="$ICP" \ + --status=succeeded \ + --max-polls=60 \ + --interval=1 + +# stop the workflows +cylc stop --kill --max-polls=10 --interval=2 "$UPID" +cylc stop --kill --max-polls=10 --interval=2 "$DOID" + +# lint'em +cylc lint "$UPID" +cylc lint "$DOID" + +# clean up +cylc clean "$UPID" +cylc clean "$DOID" diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/README.rst b/cylc/flow/etc/examples/inter-workflow-triggers/README.rst new file mode 100644 index 00000000000..8aa8d7d7b6f --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/README.rst @@ -0,0 +1,14 @@ +This example shows how one workflow can "trigger off" of tasks in another +workflow. + +In this example, there are two workflows: + +* "upstream" writes a file. +* "downstream" reads this file. + +Run both workflows simultaneously to see this in action: + +.. code-block:: console + + $ cylc vip inter-workflow-triggers/upstream + $ cylc vip inter-workflow-triggers/downstream diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc b/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc new file mode 100644 index 00000000000..115ecd94755 --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/downstream/flow.cylc @@ -0,0 +1,25 @@ +[meta] + title = Downstream Workflow + description = """ + This workflow uses the data provided by the upstream workflow. + """ + +[scheduling] + # start two hours before the current hour + initial cycle point = previous(T-00) - PT2H + [[xtriggers]] + # this is an "xtrigger" - it will wait for the task "b" in the same + # cycle from the workflow "upstream" + upstream = workflow_state(workflow="inter-workflow-triggers/upstream", task="b", point="%(point)s") + [[graph]] + PT1H = """ + @upstream => process + """ + +[runtime] + [[process]] + script = echo "The random number is: $(cat "$file")" + [[[environment]]] + # this is where the data should be written to in the upstream workflow + # Note: "runN" will point to the most recent run of a workflow + file = $HOME/cylc-run/upstream/runN/share/$CYLC_TASK_CYCLE_POINT diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/index.rst b/cylc/flow/etc/examples/inter-workflow-triggers/index.rst new file mode 100644 index 00000000000..0fd70b17d21 --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/index.rst @@ -0,0 +1,28 @@ +Inter-Workflow Triggering +========================= + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/inter-workflow-triggers + +.. include:: README.rst + +.. literalinclude:: upstream/flow.cylc + :language: cylc + :caption: Upstream Workflow + +.. literalinclude:: downstream/flow.cylc + :language: cylc + :caption: Downstream Workflow + +.. admonition:: Example - Decoupled workflows + :class: hint + + This pattern is useful where you have workflows that you want to keep decoupled + from one another, but still need to exchange data. E.G. in operational + meteorology we might have a global model (covering the whole Earth) and a + regional model (just covering a little bit of it) where the regional model + obtains its boundary condition from the global model. diff --git a/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc b/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc new file mode 100644 index 00000000000..145c3b323db --- /dev/null +++ b/cylc/flow/etc/examples/inter-workflow-triggers/upstream/flow.cylc @@ -0,0 +1,27 @@ +[meta] + title = Upstream Workflow + description = """ + This is the workflow which is providing the data that the downstream + workflow wants to use. + """ + +[scheduling] + # start two hours before the current time on the whole hour + initial cycle point = previous(T-00) - PT2H + [[graph]] + PT1H = """ + # wait for the "real world" time before running "a": + @wall_clock => a + + # then run task "b" + a => b + """ + +[runtime] + [[a]] + [[b]] + # write a random number to ~/cylc-run//share/ + # for the downstream workflow to use + script = echo "$RANDOM" > "$file" + [[[environment]]] + file = ${CYLC_WORKFLOW_SHARE_DIR}/${CYLC_TASK_CYCLE_POINT} diff --git a/cylc/flow/etc/tutorial/consolidation-tutorial/.validate b/cylc/flow/etc/tutorial/consolidation-tutorial/.validate index 4ecb762f179..2c73648700e 100755 --- a/cylc/flow/etc/tutorial/consolidation-tutorial/.validate +++ b/cylc/flow/etc/tutorial/consolidation-tutorial/.validate @@ -18,4 +18,5 @@ set -eux +cylc lint . cylc validate . --icp=2000 diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate index 2292afd6aad..38d306164de 100755 --- a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate +++ b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate @@ -18,6 +18,7 @@ set -eux APIKEY="$(head --lines 1 ../api-keys)" FLOW_NAME="$(< /dev/urandom tr -dc A-Za-z | head -c6)" +cylc lint . cylc install --workflow-name "$FLOW_NAME" --no-run-name sed -i "s/DATAPOINT_API_KEY/$APIKEY/" "$HOME/cylc-run/$FLOW_NAME/flow.cylc" cylc validate --check-circular --icp=2000 "$FLOW_NAME" diff --git a/cylc/flow/etc/tutorial/retries-tutorial/.validate b/cylc/flow/etc/tutorial/retries-tutorial/.validate index 1c532cd1120..1eb85986072 100755 --- a/cylc/flow/etc/tutorial/retries-tutorial/.validate +++ b/cylc/flow/etc/tutorial/retries-tutorial/.validate @@ -18,4 +18,5 @@ set -eux +cylc lint . cylc validate --check-circular . --icp=2000 diff --git a/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc b/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc index d0e920fba0e..04b97c274c1 100644 --- a/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc +++ b/cylc/flow/etc/tutorial/retries-tutorial/flow.cylc @@ -16,8 +16,8 @@ DIE_2=$((RANDOM%6 + 1)) echo "Rolled $DIE_1 and $DIE_2..." if (($DIE_1 == $DIE_2)); then - echo "doubles!" + echo "doubles!" else - exit 1 + exit 1 fi """ diff --git a/cylc/flow/etc/tutorial/runtime-introduction/.validate b/cylc/flow/etc/tutorial/runtime-introduction/.validate index 94dace9fbbc..ec6980f05da 100755 --- a/cylc/flow/etc/tutorial/runtime-introduction/.validate +++ b/cylc/flow/etc/tutorial/runtime-introduction/.validate @@ -18,6 +18,7 @@ set -eux FLOW_NAME="$(< /dev/urandom tr -dc A-Za-z | head -c6)" SRC=$(cylc get-resources tutorial 2>&1 | head -n1 | awk '{print $NF}') +cylc lint . cylc install "${SRC}/runtime-introduction" --workflow-name "$FLOW_NAME" --no-run-name cylc validate --check-circular --icp=2000 "$FLOW_NAME" cylc play --no-detach --abort-if-any-task-fails "$FLOW_NAME" diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py index d1a459f996b..43591e7be9c 100644 --- a/cylc/flow/exceptions.py +++ b/cylc/flow/exceptions.py @@ -15,16 +15,12 @@ # along with this program. If not, see . """Exceptions for "expected" errors.""" -import errno + from textwrap import wrap from typing import ( - Callable, Dict, Iterable, - NoReturn, Optional, - Tuple, - Type, Union, TYPE_CHECKING, ) @@ -42,7 +38,6 @@ class CylcError(Exception): message to the user is more appropriate than traceback. CLI commands will catch this exception and exit with str(exception). - """ @@ -50,10 +45,14 @@ class PluginError(CylcError): """Represents an error arising from a Cylc plugin. Args: - entry_point: The plugin entry point as defined in setup.cfg + entry_point: + The plugin entry point as defined in setup.cfg (e.g. 'cylc.main_loop') - plugin_name: Name of the plugin - exc: Original exception caught when trying to run the plugin + plugin_name: + Name of the plugin + exc: + Original exception caught when trying to run the plugin + """ def __init__(self, entry_point: str, plugin_name: str, exc: Exception): @@ -78,12 +77,8 @@ class InputError(CylcError): class CylcConfigError(CylcError): - """Generic exception to handle an error in a Cylc configuration file. - - TODO: - * reference the configuration element causing the problem - - """ + """Generic exception to handle an error in a Cylc configuration file.""" + # TODO: reference the configuration element causing the problem class WorkflowConfigError(CylcConfigError): @@ -142,19 +137,6 @@ class ContactFileExists(CylcError): """Workflow contact file exists.""" -def handle_rmtree_err( - function: Callable, - path: str, - excinfo: Tuple[Type[Exception], Exception, object] -) -> NoReturn: - """Error handler for shutil.rmtree.""" - exc = excinfo[1] - if isinstance(exc, OSError) and exc.errno == errno.ENOTEMPTY: - # "Directory not empty", likely due to filesystem lag - raise FileRemovalError(exc) - raise exc - - class FileRemovalError(CylcError): """Exception for errors during deletion of files/directories, which are probably the filesystem's fault, not Cylc's.""" @@ -269,6 +251,18 @@ def __str__(self): class ClientError(CylcError): + """Base class for errors raised by Cylc client instances. + + For example, the workflow you are trying to connect to is stopped. + + Attributes: + message: + The exception message. + traceback: + The underlying exception instance if available. + workflow: + The workflow ID if available. + """ def __init__( self, @@ -291,7 +285,7 @@ def __str__(self) -> str: class WorkflowStopped(ClientError): - """Special case of ClientError for a stopped workflow.""" + """The Cylc scheduler you attempted to connect to is stopped.""" def __init__(self, workflow): self.workflow = workflow @@ -301,6 +295,16 @@ def __str__(self): class ClientTimeout(CylcError): + """The scheduler did not respond within the timeout. + + This could be due to: + * Network issues. + * Scheduler issues. + * Insufficient timeout. + + To increase the timeout, use the ``--comms-timeout`` option. + + """ def __init__(self, message: str, workflow: Optional[str] = None): self.message = message @@ -311,7 +315,7 @@ def __str__(self) -> str: class CyclingError(CylcError): - pass + """Base class for errors in cycling configuration.""" class CyclerTypeError(CyclingError): @@ -446,10 +450,14 @@ class NoPlatformsError(PlatformLookupError): """None of the platforms of a given set were reachable. Args: - identity: The name of the platform group or install target - set_type: Whether the set of platforms is a platform group or an - install target - place: Where the attempt to get the platform failed. + identity: + The name of the platform group or install target. + set_type: + Whether the set of platforms is a platform group or an install + target. + place: + Where the attempt to get the platform failed. + """ def __init__( self, identity: str, set_type: str = 'group', place: str = '' diff --git a/cylc/flow/jinja/filters/duration_as.py b/cylc/flow/jinja/filters/duration_as.py index bc692a68017..7f2775facb9 100644 --- a/cylc/flow/jinja/filters/duration_as.py +++ b/cylc/flow/jinja/filters/duration_as.py @@ -15,6 +15,8 @@ # along with this program. If not, see . """Filter for formatting ISO8601 duration strings.""" +from typing import Callable, Dict, Tuple + from metomi.isodatetime.parsers import DurationParser SECONDS_PER_MINUTE = 60.0 @@ -26,7 +28,7 @@ SECONDS_PER_DAY = SECONDS_PER_HOUR * HOURS_PER_DAY SECONDS_PER_WEEK = SECONDS_PER_DAY * DAYS_PER_WEEK -CONVERSIONS = { +CONVERSIONS: Dict[Tuple[str, str], Callable] = { ('s', 'seconds'): float, ('m', 'minutes'): lambda s: float(s) / SECONDS_PER_MINUTE, ('h', 'hours'): lambda s: float(s) / SECONDS_PER_HOUR, @@ -35,7 +37,7 @@ } -def duration_as(iso8601_duration, units): +def duration_as(iso8601_duration: str, units: str) -> float: """Format an :term:`ISO8601 duration` string as the specified units. Units for the conversion can be specified in a case-insensitive short or @@ -57,8 +59,8 @@ def duration_as(iso8601_duration, units): - ``{{CYCLE_SUBINTERVAL | duration_as('s') | int}}`` - 1800 Args: - iso8601_duration (str): Any valid ISO8601 duration as a string. - units (str): Destination unit for the duration conversion + iso8601_duration: Any valid ISO8601 duration as a string. + units: Destination unit for the duration conversion Return: The total number of the specified unit contained in the specified diff --git a/cylc/flow/jinja/filters/pad.py b/cylc/flow/jinja/filters/pad.py index 75c5e117dd3..1c013b22ce4 100644 --- a/cylc/flow/jinja/filters/pad.py +++ b/cylc/flow/jinja/filters/pad.py @@ -15,18 +15,20 @@ # along with this program. If not, see . """Filter for padding strings to a set number of chars.""" +from typing import Union -def pad(value, length, fillchar=' '): + +def pad(value: str, length: Union[int, str], fillchar: str = ' '): """Pads a string to some length with a fill character Useful for generating task names and related values in ensemble workflows. Args: - value (str): + value: The string to pad. - length (int/str): + length: The length for the returned string. - fillchar (str - optional): + fillchar: The character to fill in surplus space (space by default). Returns: diff --git a/cylc/flow/jinja/filters/strftime.py b/cylc/flow/jinja/filters/strftime.py index d1fcb5d3eef..1af97d75abc 100644 --- a/cylc/flow/jinja/filters/strftime.py +++ b/cylc/flow/jinja/filters/strftime.py @@ -15,10 +15,16 @@ # along with this program. If not, see . """Filter for formatting ISO8601 datetime strings.""" +from typing import Optional + from metomi.isodatetime.parsers import TimePointParser -def strftime(iso8601_datetime, strftime_str, strptime_str=None): +def strftime( + iso8601_datetime: str, + strftime_str: str, + strptime_str: Optional[str] = None, +): """Format an :term:`ISO8601 datetime` string using an strftime string. .. code-block:: cylc @@ -29,11 +35,11 @@ def strftime(iso8601_datetime, strftime_str, strptime_str=None): strptime string as the second argument. Args: - iso8601_datetime (str): + iso8601_datetime: Any valid ISO8601 datetime as a string. - strftime_str (str): + strftime_str: A valid strftime string to format the output datetime. - strptime_str (str - optional): + strptime_str: A valid strptime string defining the format of the provided iso8601_datetime. diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py index b3955d3b058..e7e26954d56 100644 --- a/cylc/flow/network/client.py +++ b/cylc/flow/network/client.py @@ -195,15 +195,15 @@ class WorkflowRuntimeClient( # type: ignore[misc] the contact file. Attributes: - host (str): + host: Workflow host name. - port (int): + port: Workflow host port. - timeout_handler (function): + timeout_handler: Optional function which runs before ClientTimeout is raised. This provides an interface for raising more specific exceptions in the event of a communication timeout. - header (dict): + header: Request "header" data to attach to each request. Usage: diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index c2202f3f31e..6de28ada517 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -48,7 +48,16 @@ import asyncio from pathlib import Path import re -from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + AsyncGenerator, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from packaging.version import parse as parse_version from packaging.specifiers import SpecifierSet @@ -422,7 +431,7 @@ def format_query(fields, filters=None): @pipe(preproc=format_query) -async def graphql_query(flow, fields, filters=None): +async def graphql_query(flow: dict, fields: Iterable, filters=None): """Obtain information from a GraphQL request to the flow. Requires: @@ -430,9 +439,9 @@ async def graphql_query(flow, fields, filters=None): * contact_info Args: - flow (dict): + flow: Flow information dictionary, provided by scan through the pipe. - fields (iterable): + fields: Iterable containing the fields to request e.g:: ['id', 'name'] @@ -464,12 +473,15 @@ async def graphql_query(flow, fields, filters=None): LOG.warning(f'Workflow not running: {flow["name"]}') return False try: - ret = await client.async_request( - 'graphql', - { - 'request_string': query, - 'variables': {} - } + ret = cast( + 'dict', + await client.async_request( + 'graphql', + { + 'request_string': query, + 'variables': {} + } + ) ) except WorkflowStopped: LOG.warning(f'Workflow not running: {flow["name"]}') diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py index be2f0e35707..77ed8114758 100644 --- a/cylc/flow/network/schema.py +++ b/cylc/flow/network/schema.py @@ -348,8 +348,6 @@ async def get_nodes_all(root, info, **args): _, field_ids = process_resolver_info(root, info, args) - if hasattr(args, 'id'): - args['ids'] = [args.get('id')] if field_ids: if isinstance(field_ids, str): field_ids = [field_ids] @@ -376,10 +374,8 @@ async def get_nodes_all(root, info, **args): else: # live objects can be represented by a universal ID args[arg] = [Tokens(n_id, relative=True) for n_id in args[arg]] - args['workflows'] = [ - Tokens(w_id) for w_id in args['workflows']] - args['exworkflows'] = [ - Tokens(w_id) for w_id in args['exworkflows']] + for arg in ('workflows', 'exworkflows'): + args[arg] = [Tokens(w_id) for w_id in args[arg]] resolvers = info.context.get('resolvers') return await resolvers.get_nodes_all(node_type, args) diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py index 14191962c22..2c1e4f22b60 100644 --- a/cylc/flow/option_parsers.py +++ b/cylc/flow/option_parsers.py @@ -333,12 +333,13 @@ class CylcOptionParser(OptionParser): ['-z', '--set-list', '--template-list'], metavar='NAME=VALUE1,VALUE2,...', help=( - 'Set the value of a Jinja2 template variable in the' - ' workflow definition as a comma separated' - ' list of Python strings.' - ' Values containing commas must be quoted.' - " e.g. '+s STR=a,b,c' => ['a', 'b', 'c']" - " or '+ s STR=a,\"b,c\"' => ['a', 'b,c']" + 'A more convenient alternative to --set for defining a list' + ' of strings. E.G.' + ' "-z FOO=a,b,c" is shorthand for' + ' "-s FOO=[\'a\',\'b\',\'c\']".' + ' Commas can be present in values if quoted, e.g.' + ' "-z FOO=a,\'b,c\'" is shorthand for' + ' "-s FOO=[\'a\',\'b,c\']".' + CAN_BE_USED_MULTIPLE + NOTE_PERSIST_ACROSS_RESTARTS ), diff --git a/cylc/flow/parsec/exceptions.py b/cylc/flow/parsec/exceptions.py index 3acfe31c820..690d583b885 100644 --- a/cylc/flow/parsec/exceptions.py +++ b/cylc/flow/parsec/exceptions.py @@ -78,8 +78,6 @@ class FileParseError(ParsecError): Classification of error (e.g. Jinja2Error). help_lines: Additional info to include in the exception. - - Args (ways of providing exception context - TODO rationalise this!): lines: (preferred) Dictionary in the format {filename: [context_line, ..., error_line]} @@ -140,7 +138,7 @@ def __str__(self) -> str: class TemplateVarLanguageClash(FileParseError): - ... + """Multiple workflow configuration templating engines configured.""" class EmPyError(FileParseError): diff --git a/cylc/flow/parsec/validate.py b/cylc/flow/parsec/validate.py index 4ef66e19862..d728fd79d84 100644 --- a/cylc/flow/parsec/validate.py +++ b/cylc/flow/parsec/validate.py @@ -26,7 +26,7 @@ import shlex from collections import deque from textwrap import dedent -from typing import List, Dict, Any, Tuple +from typing import List, Dict, Any, Optional, Tuple from metomi.isodatetime.data import Duration, TimePoint from metomi.isodatetime.dumpers import TimePointDumper @@ -373,7 +373,7 @@ def coerce_range(cls, value, keys): return Range((int(min_), int(max_))) @classmethod - def coerce_str(cls, value, keys): + def coerce_str(cls, value, keys) -> str: """Coerce value to a string. Examples: @@ -385,7 +385,7 @@ def coerce_str(cls, value, keys): """ if isinstance(value, list): # handle graph string merging - vraw = [] + vraw: List[str] = [] vals = [value] while vals: val = vals.pop() @@ -512,16 +512,32 @@ def parse_int_range(cls, value): return None @classmethod - def strip_and_unquote(cls, keys, value): + def _unquote(cls, keys: List[str], value: str) -> Optional[str]: + """Unquote value.""" + for substr, rec in ( + ("'''", cls._REC_MULTI_LINE_SINGLE), + ('"""', cls._REC_MULTI_LINE_DOUBLE), + ('"', cls._REC_DQ_VALUE), + ("'", cls._REC_SQ_VALUE) + ): + if value.startswith(substr): + match = rec.match(value) + if not match: + raise IllegalValueError("string", keys, value) + return match[1] + return None + + @classmethod + def strip_and_unquote(cls, keys: List[str], value: str) -> str: """Remove leading and trailing spaces and unquote value. Args: - keys (list): + keys: Keys in nested dict that represents the raw configuration. - value (str): + value: String value in raw configuration. - Return (str): + Return: Processed value. Examples: @@ -529,24 +545,13 @@ def strip_and_unquote(cls, keys, value): 'foo' """ - for substr, rec in [ - ["'''", cls._REC_MULTI_LINE_SINGLE], - ['"""', cls._REC_MULTI_LINE_DOUBLE], - ['"', cls._REC_DQ_VALUE], - ["'", cls._REC_SQ_VALUE]]: - if value.startswith(substr): - match = rec.match(value) - if not match: - raise IllegalValueError("string", keys, value) - value = match.groups()[0] - break - else: - # unquoted - value = value.split(r'#', 1)[0] + val = cls._unquote(keys, value) + if val is None: + val = value.split(r'#', 1)[0] # Note strip() removes leading and trailing whitespace, including # initial newlines on a multiline string: - return dedent(value).strip() + return dedent(val).strip() @classmethod def strip_and_unquote_list(cls, keys, value): @@ -657,6 +662,7 @@ class CylcConfigValidator(ParsecValidator): V_CYCLE_POINT = 'V_CYCLE_POINT' V_CYCLE_POINT_FORMAT = 'V_CYCLE_POINT_FORMAT' V_CYCLE_POINT_TIME_ZONE = 'V_CYCLE_POINT_TIME_ZONE' + V_CYCLE_POINT_WITH_OFFSETS = 'V_CYCLE_POINT_WITH_OFFSETS' V_INTERVAL = 'V_INTERVAL' V_INTERVAL_LIST = 'V_INTERVAL_LIST' V_PARAMETER_LIST = 'V_PARAMETER_LIST' @@ -699,6 +705,26 @@ class CylcConfigValidator(ParsecValidator): '-0830': 'UTC minus 8 hours and 30 minutes.' } ), + V_CYCLE_POINT_WITH_OFFSETS: ( + 'cycle point with support for offsets', + 'An integer or date-time cycle point, with optional offset(s).', + { + '1': 'An integer cycle point.', + '1 +P5': ( + 'An integer cycle point with an offset' + ' (this evaluates as ``6``).' + ), + '+P5': ( + 'An integer cycle point offset.' + ' This offset is added to the initial cycle point' + ), + '2000-01-01T00:00Z': 'A date-time cycle point.', + '2000-02-29T00:00Z +P1D +P1M': ( + 'A date-time cycle point with offsets' + ' (this evaluates as ``2000-04-01T00:00Z``).' + ), + } + ), V_INTERVAL: ( 'time interval', 'An ISO8601 duration.', @@ -751,6 +777,9 @@ def __init__(self): self.V_CYCLE_POINT: self.coerce_cycle_point, self.V_CYCLE_POINT_FORMAT: self.coerce_cycle_point_format, self.V_CYCLE_POINT_TIME_ZONE: self.coerce_cycle_point_time_zone, + # NOTE: This type exists for documentation reasons + # it doesn't actually process offsets, that happens later + self.V_CYCLE_POINT_WITH_OFFSETS: self.coerce_str, self.V_INTERVAL: self.coerce_interval, self.V_INTERVAL_LIST: self.coerce_interval_list, self.V_PARAMETER_LIST: self.coerce_parameter_list, @@ -1136,23 +1165,25 @@ def _coerce_type(cls, value): return val -# BACK COMPAT: BroadcastConfigValidator -# The DB at 8.0.x stores Interval values as neither ISO8601 duration -# string or DurationFloat. This has been fixed at 8.1.0, and -# the following class acts as a bridge between fixed and broken. -# url: -# https://github.com/cylc/cylc-flow/pull/5138 -# from: -# 8.0.x -# to: -# 8.1.x -# remove at: -# 8.x class BroadcastConfigValidator(CylcConfigValidator): """Validate and Coerce DB loaded broadcast config to internal objects.""" def __init__(self): CylcConfigValidator.__init__(self) + @classmethod + def coerce_str(cls, value, keys) -> str: + """Coerce value to a string. Unquotes & strips lead/trail whitespace. + + Prevents ParsecValidator from assuming '#' means comments; + '#' has valid uses in shell script such as parameter substitution. + + Examples: + >>> BroadcastConfigValidator.coerce_str('echo "${FOO#*bar}"', None) + 'echo "${FOO#*bar}"' + """ + val = ParsecValidator._unquote(keys, value) or value + return dedent(val).strip() + @classmethod def strip_and_unquote_list(cls, keys, value): """Remove leading and trailing spaces and unquote list value. @@ -1177,6 +1208,18 @@ def strip_and_unquote_list(cls, keys, value): value = value.lstrip('[').rstrip(']') return ParsecValidator.strip_and_unquote_list(keys, value) + # BACK COMPAT: BroadcastConfigValidator.coerce_interval + # The DB at 8.0.x stores Interval values as neither ISO8601 duration + # string or DurationFloat. This has been fixed at 8.1.0, and + # the following method acts as a bridge between fixed and broken. + # url: + # https://github.com/cylc/cylc-flow/pull/5138 + # from: + # 8.0.x + # to: + # 8.1.x + # remove at: + # 8.x @classmethod def coerce_interval(cls, value, keys): """Coerce an ISO 8601 interval into seconds. diff --git a/cylc/flow/resources.py b/cylc/flow/resources.py index 79884e27476..d81b7f7211b 100644 --- a/cylc/flow/resources.py +++ b/cylc/flow/resources.py @@ -16,6 +16,7 @@ """Extract named resources from the cylc.flow package.""" +from contextlib import suppress from pathlib import Path from random import shuffle import shutil @@ -31,6 +32,7 @@ RESOURCE_DIR = Path(cylc.flow.__file__).parent / 'etc' TUTORIAL_DIR = RESOURCE_DIR / 'tutorial' +EXAMPLE_DIR = RESOURCE_DIR / 'examples' # {resource: brief description} @@ -52,6 +54,11 @@ def list_resources(write=print, headers=True): for path in TUTORIAL_DIR.iterdir() if path.is_dir() ] + examples = [ + path.relative_to(RESOURCE_DIR) + for path in EXAMPLE_DIR.iterdir() + if path.is_dir() + ] if headers: write('Resources:') max_len = max(len(res) for res in RESOURCE_NAMES) @@ -62,15 +69,21 @@ def list_resources(write=print, headers=True): for tutorial in tutorials: write(f' {tutorial}') write(f' {API_KEY}') + if headers: + write('\nExamples:') + for example in examples: + write(f' {example}') -def path_is_tutorial(src: Path) -> bool: - """Returns True if the src path is in the tutorial directory.""" - try: +def path_is_source_workflow(src: Path) -> bool: + """Returns True if the src path is a Cylc workflow.""" + with suppress(ValueError): src.relative_to(TUTORIAL_DIR) - except ValueError: - return False - return True + return True + with suppress(ValueError): + src.relative_to(EXAMPLE_DIR) + return True + return False def get_resources(resource: str, tgt_dir: Optional[str]): @@ -95,11 +108,11 @@ def get_resources(resource: str, tgt_dir: Optional[str]): '\nRun `cylc get-resources --list` for resource names.' ) - is_tutorial = path_is_tutorial(src) + is_source_workflow = path_is_source_workflow(src) # get the target path if not tgt_dir: - if is_tutorial: + if is_source_workflow: # this is a tutorial => use the primary source dir _tgt_dir = Path(glbl_cfg().get(['install', 'source dirs'])[0]) else: @@ -113,8 +126,8 @@ def get_resources(resource: str, tgt_dir: Optional[str]): tgt = tgt.resolve() # extract resources - extract_resource(src, tgt, is_tutorial) - if is_tutorial: + extract_resource(src, tgt, is_source_workflow) + if is_source_workflow: set_api_key(tgt) @@ -131,16 +144,32 @@ def _backup(tgt: Path) -> None: shutil.move(str(tgt), str(backup)) -def extract_resource(src: Path, tgt: Path, is_tutorial: bool = False) -> None: +def extract_resource( + src: Path, + tgt: Path, + is_source_workflow: bool = False, +) -> None: """Extract src into tgt. NOTE: src can be a dir or a file. """ LOG.info(f"Extracting {src.relative_to(RESOURCE_DIR)} to {tgt}") - if is_tutorial and tgt.exists(): + if is_source_workflow and tgt.exists(): # target exists, back up the old copy _backup(tgt) + # files to exclude + if is_source_workflow: + excludes = [ + # test files + '.validate', + 'reftest', + # documentation files + 'index.rst', + ] + else: + excludes = [] + # create the target directory try: tgt.parent.mkdir(parents=True, exist_ok=True) @@ -151,6 +180,10 @@ def extract_resource(src: Path, tgt: Path, is_tutorial: bool = False) -> None: shutil.copytree(str(src), str(tgt)) else: shutil.copyfile(str(src), str(tgt)) + for exclude in excludes: + path = tgt / exclude + if path.exists(): + path.unlink() except IsADirectoryError as exc: LOG.error( f'Cannot extract file {exc.filename} as there is an ' diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py index c45d8f692c9..9ab91a0a569 100644 --- a/cylc/flow/scheduler.py +++ b/cylc/flow/scheduler.py @@ -1027,6 +1027,7 @@ def command_stop( point = TaskID.get_standardised_point(cycle_point) if point is not None and self.pool.set_stop_point(point): self.options.stopcp = str(point) + self.config.stop_point = point self.workflow_db_mgr.put_workflow_stop_cycle_point( self.options.stopcp) elif clock_time is not None: diff --git a/cylc/flow/scripts/broadcast.py b/cylc/flow/scripts/broadcast.py index d1a62d708da..28bd3bae4e3 100755 --- a/cylc/flow/scripts/broadcast.py +++ b/cylc/flow/scripts/broadcast.py @@ -82,6 +82,7 @@ from ansimarkup import parse as cparse from copy import deepcopy from functools import partial +import os.path import re import sys from tempfile import NamedTemporaryFile @@ -203,7 +204,7 @@ def files_to_settings(settings, setting_files, cancel_mode=False): handle.seek(0, 0) cfg.loadcfg(handle.name) else: - cfg.loadcfg(setting_file) + cfg.loadcfg(os.path.abspath(setting_file)) stack = [([], cfg.get(sparse=True))] while stack: keys, item = stack.pop() diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py index 636d611194f..0626a407f32 100755 --- a/cylc/flow/scripts/lint.py +++ b/cylc/flow/scripts/lint.py @@ -19,11 +19,9 @@ Checks code style, deprecated syntax and other issues. """ -# NOTE: docstring needed for `cylc help all` output +# NOTE: docstring needed for `cylc help all` output and docs # (if editing check this still comes out as expected) -LINT_SECTIONS = ['cylc-lint', 'cylclint', 'cylc_lint'] - COP_DOC = """cylc lint [OPTIONS] ARGS Check .cylc and .rc files for code style, deprecated syntax and other issues. @@ -37,13 +35,19 @@ This can be overridden by providing the "--exit-zero" flag. """ +NOQA = """ +Individual errors can be ignored using the ``# noqa`` line comment. +It is good practice to specify specific errors you wish to ignore using +``# noqa: S002 S007 U999`` +""" + TOMLDOC = """ -pyproject.toml configuration:{} - [cylc-lint] # any of {} - ignore = ['S001', 'S002'] # List of rules to ignore - exclude = ['etc/foo.cylc'] # List of files to ignore - rulesets = ['style', '728'] # Sets default rulesets to check - max-line-length = 130 # Max line length for linting +pyproject.toml configuration: + [tool.cylc.lint] + ignore = ['S001', 'S002'] # List of rules to ignore + exclude = ['etc/foo.cylc'] # List of files to ignore + rulesets = ['style', '728'] # Sets default rulesets to check + max-line-length = 130 # Max line length for linting """ from colorama import Fore import functools @@ -71,6 +75,8 @@ from cylc.flow import LOG from cylc.flow.exceptions import CylcError +import cylc.flow.flags +from cylc.flow.loggingutil import set_timestamps from cylc.flow.option_parsers import ( CylcOptionParser as COP, WORKFLOW_ID_OR_PATH_ARG_DOC @@ -82,8 +88,31 @@ from cylc.flow.terminal import cli_function if TYPE_CHECKING: + # BACK COMPAT: typing_extensions.Literal + # FROM: Python 3.7 + # TO: Python 3.8 + from typing_extensions import Literal from optparse import Values +LINT_TABLE = ['tool', 'cylc', 'lint'] +LINT_SECTION = '.'.join(LINT_TABLE) + +# BACK COMPAT: DEPR_LINT_SECTION +# url: +# https://github.com/cylc/cylc-flow/issues/5811 +# from: +# 8.1.0 +# to: +# 8.3.0 +# remove at: +# 8.4.0 ? +DEPR_LINT_SECTION = 'cylc-lint' + +IGNORE = 'ignore' +EXCLUDE = 'exclude' +RULESETS = 'rulesets' +MAX_LINE_LENGTH = 'max-line-length' + DEPRECATED_ENV_VARS = { 'CYLC_SUITE_HOST': 'CYLC_WORKFLOW_HOST', 'CYLC_SUITE_OWNER': 'CYLC_WORKFLOW_OWNER', @@ -399,7 +428,7 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: 'short': 'Item not indented.', # Non-indented items should be sections: 'url': STYLE_GUIDE + 'indentation', - FUNCTION: re.compile(r'^[^\{\[|\s]').findall + FUNCTION: re.compile(r'^[^%\{\[|\s]').findall }, "S003": { 'short': 'Top level sections should not be indented.', @@ -582,7 +611,9 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: }, 'U008': { 'short': 'Suicide triggers are not required at Cylc 8.', - 'url': '', + 'url': ( + 'https://cylc.github.io/cylc-doc/stable/html/7-to-8' + '/major-changes/suicide-triggers.html'), 'kwargs': True, FUNCTION: functools.partial( check_for_suicide_triggers, @@ -596,9 +627,7 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: }, 'U010': { 'short': 'rose suite-hook is deprecated at Rose 2,', - 'url': ( - 'https://cylc.github.io/cylc-doc/stable/html/7-to-8' - '/major-changes/suicide-triggers.html'), + 'url': '', FUNCTION: lambda line: 'rose suite-hook' in line, }, 'U011': { @@ -691,28 +720,34 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: list_wrapper, check=CHECK_FOR_OLD_VARS.findall), }, } -RULESETS = ['728', 'style', 'all'] +ALL_RULESETS = ['728', 'style', 'all'] EXTRA_TOML_VALIDATION = { - 'ignore': { + IGNORE: { lambda x: re.match(r'[A-Z]\d\d\d', x): '{item} not valid: Ignore codes should be in the form X001', lambda x: x in parse_checks(['728', 'style']): '{item} is a not a known linter code.' }, - 'rulesets': { - lambda item: item in RULESETS: + RULESETS: { + lambda item: item in ALL_RULESETS: '{item} not valid: Rulesets can be ' '\'728\', \'style\' or \'all\'.' }, - 'max-line-length': { + MAX_LINE_LENGTH: { lambda x: isinstance(x, int): 'max-line-length must be an integer.' }, # consider checking that item is file? - 'exclude': {} + EXCLUDE: {} } +def parse_ruleset_option(ruleset: str) -> List[str]: + if ruleset in {'all', ''}: + return ['728', 'style'] + return [ruleset] + + def get_url(check_meta: Dict) -> str: """Get URL from check data. @@ -752,9 +787,9 @@ def validate_toml_items(tomldata): if key not in EXTRA_TOML_VALIDATION.keys(): raise CylcError( f'Only {[*EXTRA_TOML_VALIDATION.keys()]} ' - f'allowed as toml sections but you used {key}' + f'allowed as toml sections but you used "{key}"' ) - if key != 'max-line-length': + if key != MAX_LINE_LENGTH: # Item should be a list... if not isinstance(items, list): raise CylcError( @@ -774,26 +809,35 @@ def validate_toml_items(tomldata): return True -def get_pyproject_toml(dir_): +def get_pyproject_toml(dir_: Path) -> Dict[str, Any]: """if a pyproject.toml file is present open it and return settings. """ - keys = ['rulesets', 'ignore', 'exclude', 'max-line-length'] - tomlfile = Path(dir_ / 'pyproject.toml') - tomldata = {} + tomlfile = dir_ / 'pyproject.toml' + tomldata: Dict[str, Union[List[str], int, None]] = { + RULESETS: [], + IGNORE: [], + EXCLUDE: [], + MAX_LINE_LENGTH: None, + } if tomlfile.is_file(): try: loadeddata = toml_loads(tomlfile.read_text()) except TOMLDecodeError as exc: raise CylcError(f'pyproject.toml did not load: {exc}') - if any( - i in loadeddata for i in LINT_SECTIONS - ): - for key in keys: - tomldata[key] = loadeddata.get('cylc-lint').get(key, []) - validate_toml_items(tomldata) - if not tomldata: - tomldata = {key: [] for key in keys} + _tool, _cylc, _lint = LINT_TABLE + try: + data = loadeddata[_tool][_cylc][_lint] + except KeyError: + if DEPR_LINT_SECTION in loadeddata: + LOG.warning( + f"The [{DEPR_LINT_SECTION}] section in pyproject.toml is " + f"deprecated. Use [{LINT_SECTION}] instead." + ) + data = loadeddata.get(DEPR_LINT_SECTION, {}) + tomldata.update(data) + validate_toml_items(tomldata) + return tomldata @@ -809,20 +853,14 @@ def merge_cli_with_tomldata(target: Path, options: 'Values') -> Dict[str, Any]: _merge_cli_with_tomldata to keep the testing of file-system touching and pure logic separate. """ - ruleset_default = False - if options.linter == 'all': - options.linter = ['728', 'style'] - elif options.linter == '': - options.linter = ['728', 'style'] - ruleset_default = True - else: - options.linter = [options.linter] + ruleset_default = (options.ruleset == '') + options.ruleset = parse_ruleset_option(options.ruleset) tomlopts = get_pyproject_toml(target) return _merge_cli_with_tomldata( { - 'exclude': [], - 'ignore': options.ignores, - 'rulesets': options.linter + EXCLUDE: [], + IGNORE: options.ignores, + RULESETS: options.ruleset }, tomlopts, ruleset_default @@ -857,30 +895,30 @@ def _merge_cli_with_tomldata( >>> result['exclude'] ['*.bk'] """ - if isinstance(clidata['rulesets'][0], list): - clidata['rulesets'] = clidata['rulesets'][0] + if isinstance(clidata[RULESETS][0], list): + clidata[RULESETS] = clidata[RULESETS][0] output = {} # Combine 'ignore' sections: - output['ignore'] = sorted(set(clidata['ignore'] + tomldata['ignore'])) + output[IGNORE] = sorted(set(clidata[IGNORE] + tomldata[IGNORE])) - # Replace 'rulesets from toml with those from CLI if they exist: + # Replace 'rulesets' from toml with those from CLI if they exist: if override_cli_default_rules: - output['rulesets'] = ( - tomldata['rulesets'] if tomldata['rulesets'] - else clidata['rulesets'] + output[RULESETS] = ( + tomldata[RULESETS] if tomldata[RULESETS] + else clidata[RULESETS] ) else: - output['rulesets'] = ( - clidata['rulesets'] if clidata['rulesets'] - else tomldata['rulesets'] + output[RULESETS] = ( + clidata[RULESETS] if clidata[RULESETS] + else tomldata[RULESETS] ) # Return 'exclude' and 'max-line-length' for the tomldata: - output['exclude'] = tomldata['exclude'] - output['max-line-length'] = tomldata.get('max-line-length', None) + output[EXCLUDE] = tomldata[EXCLUDE] + output[MAX_LINE_LENGTH] = tomldata.get(MAX_LINE_LENGTH, None) return output @@ -1066,6 +1104,33 @@ def check_cylc_file( pass +def no_qa(line: str, index: str): + """This line has a no-qa comment. + + Examples: + # No comment, no exception: + >>> no_qa('foo = bar', 'S001') + False + + # Comment, no error codes, no checking: + >>> no_qa('foo = bar # noqa', 'S001') + True + + # Comment, no relevent error codes, no checking: + >>> no_qa('foo = bar # noqa: S999, 997', 'S001') + False + + # Comment, relevent error codes, checking: + >>> no_qa('foo = bar # noqa: S001 S003', 'S001') + True + """ + NOQA = re.compile(r'.*#\s*[Nn][Oo][Qq][Aa]:?(.*)') + noqa = NOQA.findall(line) + if noqa and (noqa[0] == '' or index in noqa[0]): + return True + return False + + def lint( file_rel: Path, lines: Iterator[str], @@ -1105,9 +1170,13 @@ def lint( # run lint checks against the current line for index, check_meta in checks.items(): # Skip commented line unless check says not to. + index_str = get_index_str(check_meta, index) if ( - line.strip().startswith('#') - and not check_meta.get('evaluate commented lines', False) + ( + line.strip().startswith('#') + and not check_meta.get('evaluate commented lines', False) + ) + or no_qa(line, index_str) ): continue @@ -1139,7 +1208,7 @@ def lint( url = get_url(check_meta) yield ( - f'# [{get_index_str(check_meta, index)}]: ' + f'# [{index_str}]: ' f'{msg}\n' f'# - see {url}\n' ) @@ -1147,7 +1216,7 @@ def lint( # write a message to inform the user write( Fore.YELLOW + - f'[{get_index_str(check_meta, index)}]' + f'[{index_str}]' f' {file_rel}:{line_no}: {msg}' ) if modify: @@ -1179,29 +1248,18 @@ def get_cylc_files( yield path -REFERENCE_TEMPLATES = { - 'section heading': '\n{title}\n{underline}\n', - 'issue heading': { - 'text': '\n{check}:\n {summary}\n {url}\n\n', - 'rst': '\n{url}_\n{underline}\n{summary}\n\n', - }, - 'auto gen message': ( - 'U998 and U999 represent automatically generated' - ' sets of deprecations and upgrades.' - ), -} - - -def get_reference(linter, output_type): +def get_reference(ruleset: str, output_type: 'Literal["text", "rst"]') -> str: """Fill out a template with all the issues Cylc Lint looks for. """ - if linter in {'all', ''}: - rulesets = ['728', 'style'] - else: - rulesets = [linter] - checks = parse_checks(rulesets, reference=True) + checks = parse_checks( + parse_ruleset_option(ruleset), + reference=True + ) - issue_heading_template = REFERENCE_TEMPLATES['issue heading'][output_type] + issue_heading_template = ( + '\n{url}_\n{underline}\n{summary}\n\n' if output_type == 'rst' else + '\n{check}:\n {summary}\n {url}\n\n' + ) output = '' current_checkset = '' for index, meta in checks.items(): @@ -1210,11 +1268,15 @@ def get_reference(linter, output_type): if meta['purpose'] != current_checkset: current_checkset = meta['purpose'] title = CHECKS_DESC[meta["purpose"]] - output += REFERENCE_TEMPLATES['section heading'].format( - title=title, underline="-" * len(title)) + output += '\n{title}\n{underline}\n'.format( + title=title, underline="-" * len(title) + ) if current_checkset == 'A': - output += REFERENCE_TEMPLATES['auto gen message'] + output += ( + 'U998 and U999 represent automatically generated' + ' sets of deprecations and upgrades.' + ) # Fill a template with info about the issue. if output_type == 'rst': @@ -1261,24 +1323,28 @@ def target_version_check( disabled thatr with --exit-zero. """ cylc8 = (target / 'flow.cylc').exists() - if not cylc8 and mergedopts['rulesets'] == ['728']: + if not cylc8 and mergedopts[RULESETS] == ['728']: LOG.error( f'{target} not a Cylc 8 workflow: ' 'Lint after renaming ' '"suite.rc" to "flow.cylc"' ) sys.exit(not quiet) - elif not cylc8 and '728' in mergedopts['rulesets']: - check_names = mergedopts['rulesets'] + elif not cylc8 and '728' in mergedopts[RULESETS]: + check_names = mergedopts[RULESETS] check_names.remove('728') else: - check_names = mergedopts['rulesets'] + check_names = mergedopts[RULESETS] return check_names def get_option_parser() -> COP: parser = COP( - COP_DOC + TOMLDOC.format('', str(LINT_SECTIONS)), + ( + COP_DOC + + NOQA.replace('``', '"') + + TOMLDOC + ), argdoc=[ COP.optional(WORKFLOW_ID_OR_PATH_ARG_DOC) ], @@ -1300,7 +1366,7 @@ def get_option_parser() -> COP: ), default='', choices=["728", "style", "all", ''], - dest='linter' + dest='ruleset' ) parser.add_option( '--list-codes', @@ -1335,8 +1401,11 @@ def get_option_parser() -> COP: @cli_function(get_option_parser) def main(parser: COP, options: 'Values', target=None) -> None: + if cylc.flow.flags.verbosity < 2: + set_timestamps(LOG, False) + if options.ref_mode: - print(get_reference(options.linter, 'text')) + print(get_reference(options.ruleset, 'text')) sys.exit(0) # If target not given assume we are looking at PWD: @@ -1362,13 +1431,13 @@ def main(parser: COP, options: 'Values', target=None) -> None: # Get the checks object. checks = parse_checks( check_names, - ignores=mergedopts['ignore'], - max_line_len=mergedopts['max-line-length'] + ignores=mergedopts[IGNORE], + max_line_len=mergedopts[MAX_LINE_LENGTH] ) # Check each file matching a pattern: counter: Dict[str, int] = {} - for file in get_cylc_files(target, mergedopts['exclude']): + for file in get_cylc_files(target, mergedopts[EXCLUDE]): LOG.debug(f'Checking {file}') check_cylc_file( file, @@ -1403,4 +1472,5 @@ def main(parser: COP, options: 'Values', target=None) -> None: # NOTE: use += so that this works with __import__ # (docstring needed for `cylc help all` output) +__doc__ += NOQA __doc__ += get_reference('all', 'rst') diff --git a/cylc/flow/scripts/tui.py b/cylc/flow/scripts/tui.py index f8f0879a1e6..2d48e649ef2 100644 --- a/cylc/flow/scripts/tui.py +++ b/cylc/flow/scripts/tui.py @@ -61,6 +61,21 @@ def get_option_parser() -> COP: color=False ) + parser.add_option( + '--comms-timeout', + metavar='SEC', + help=( + "Set a timeout for network connections" + " to the running workflow. The default is no timeout." + " For task messaging connections see" + " site/user config file documentation." + ), + action='store', + default=3, + dest='comms_timeout', + type=int, + ) + return parser @@ -76,5 +91,9 @@ def main(_, options: 'Values', workflow_id: Optional[str] = None) -> None: workflow_id = tokens.duplicate(user=getuser()).id # start Tui - with suppress_logging(), TuiApp().main(workflow_id): + with suppress_logging(), TuiApp().main( + workflow_id, + client_timeout=options.comms_timeout, + ): + # tui stops according to user input pass diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py index 190393a1311..b6223fd2f2a 100644 --- a/cylc/flow/task_pool.py +++ b/cylc/flow/task_pool.py @@ -992,7 +992,7 @@ def set_stop_point(self, stop_point: 'PointBase') -> bool: LOG.info(f"Stop point unchanged: {stop_point}") return False - LOG.info("Setting stop point: {stop_point}") + LOG.info(f"Setting stop point: {stop_point}") self.stop_point = stop_point if ( diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py index 9ef4d465bf2..3743d0545f0 100644 --- a/cylc/flow/task_remote_mgr.py +++ b/cylc/flow/task_remote_mgr.py @@ -253,7 +253,7 @@ def remote_init( dirs_to_symlink = get_dirs_to_symlink(install_target, self.workflow) for key, value in dirs_to_symlink.items(): if value is not None: - cmd.append(f"{key}={quote(value)} ") + cmd.append(f"{key}={quote(value)}") # Create the ssh command try: host = get_host_from_platform( diff --git a/cylc/flow/tui/app.py b/cylc/flow/tui/app.py index fff634ea894..d802a2d5738 100644 --- a/cylc/flow/tui/app.py +++ b/cylc/flow/tui/app.py @@ -200,7 +200,7 @@ def load_child_node(self, key): @contextmanager -def updater_subproc(filters): +def updater_subproc(filters, client_timeout): """Runs the Updater in its own process. The updater provides the data for Tui to render. Running the updater @@ -209,7 +209,7 @@ def updater_subproc(filters): decoupling the application update logic from the data update logic. """ # start the updater - updater = Updater() + updater = Updater(client_timeout=client_timeout) p = Process(target=updater.start, args=(filters,)) try: p.start() @@ -285,7 +285,13 @@ def __init__(self, screen=None): self.filters = get_default_filters() @contextmanager - def main(self, w_id=None, id_filter=None, interactive=True): + def main( + self, + w_id=None, + id_filter=None, + interactive=True, + client_timeout=3, + ): """Start the Tui app. With interactive=False, this does not start the urwid event loop to @@ -299,7 +305,7 @@ def main(self, w_id=None, id_filter=None, interactive=True): """ self.set_initial_filters(w_id, id_filter) - with updater_subproc(self.filters) as updater: + with updater_subproc(self.filters, client_timeout) as updater: self.updater = updater # pre-subscribe to the provided workflow if requested diff --git a/cylc/flow/tui/updater.py b/cylc/flow/tui/updater.py index f9bd0cf2c00..26a8614b1a3 100644 --- a/cylc/flow/tui/updater.py +++ b/cylc/flow/tui/updater.py @@ -93,9 +93,6 @@ class Updater(): """ - # the maximum time to wait for a workflow update - CLIENT_TIMEOUT = 2 - # the interval between workflow listing scans BASE_SCAN_INTERVAL = 20 @@ -105,7 +102,7 @@ class Updater(): # the command signal used to tell the updater to shut down SIGNAL_TERMINATE = 'terminate' - def __init__(self): + def __init__(self, client_timeout=3): # Cylc comms clients for each workflow we're connected to self._clients = {} @@ -124,6 +121,9 @@ def __init__(self): # queue for commands to the updater self._command_queue = Queue() + # the maximum time to wait for a workflow update + self.client_timeout = client_timeout + def subscribe(self, w_id): """Subscribe to updates from a workflow.""" self._command_queue.put((self._subscribe.__name__, w_id)) @@ -269,10 +269,17 @@ async def _update_workflow(self, w_id, client, data): 'id': w_id, 'status': 'stopped', }) - except (CylcError, ZMQError): + except (CylcError, ZMQError) as exc: # something went wrong :( # remove the client on any error, we'll reconnect next time self._clients[w_id] = None + for workflow in data['workflows']: + if workflow['id'] == w_id: + workflow['_tui_data'] = ( + f'Error - {str(exc).splitlines()[0]}' + ) + break + else: # the data arrived, add it to the update workflow_data = workflow_update['workflows'][0] @@ -288,7 +295,7 @@ def _connect(self, data): try: self._clients[w_id] = get_client( Tokens(w_id)['workflow'], - timeout=self.CLIENT_TIMEOUT + timeout=self.client_timeout, ) except WorkflowStopped: for workflow in data['workflows']: @@ -297,7 +304,9 @@ def _connect(self, data): except (ZMQError, ClientError, ClientTimeout) as exc: for workflow in data['workflows']: if workflow['id'] == w_id: - workflow['_tui_data'] = f'Error: {exc}' + workflow['_tui_data'] = ( + f'Error - {str(exc).splitlines()[0]}' + ) break async def _scan(self): diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py index d4d83b02689..d980aecdc93 100644 --- a/cylc/flow/workflow_db_mgr.py +++ b/cylc/flow/workflow_db_mgr.py @@ -408,9 +408,10 @@ def put_task_event_timers(self, task_events_mgr) -> None: def put_xtriggers(self, sat_xtrig): """Put statements to update external triggers table.""" for sig, res in sat_xtrig.items(): - self.db_inserts_map[self.TABLE_XTRIGGERS].append({ - "signature": sig, - "results": json.dumps(res)}) + if not sig.startswith('wall_clock('): + self.db_inserts_map[self.TABLE_XTRIGGERS].append({ + "signature": sig, + "results": json.dumps(res)}) def put_update_task_state(self, itask): """Update task_states table for current state of itask. diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py index de30bc1e7d2..2f03c8d4394 100644 --- a/cylc/flow/workflow_files.py +++ b/cylc/flow/workflow_files.py @@ -22,21 +22,25 @@ """ +from enum import Enum +import errno import os +from pathlib import Path import re import shutil -from enum import Enum -from pathlib import Path from subprocess import ( PIPE, Popen, TimeoutExpired, ) from typing import ( + Callable, Dict, Optional, Tuple, Union, + NoReturn, + Type, ) import cylc.flow.flags @@ -48,7 +52,7 @@ InputError, ServiceFileError, WorkflowFilesError, - handle_rmtree_err, + FileRemovalError, ) from cylc.flow.hostuserutil import ( get_user, @@ -64,6 +68,19 @@ from cylc.flow.util import cli_format +def handle_rmtree_err( + function: Callable, + path: str, + excinfo: Tuple[Type[Exception], Exception, object] +) -> NoReturn: + """Error handler for shutil.rmtree.""" + exc = excinfo[1] + if isinstance(exc, OSError) and exc.errno == errno.ENOTEMPTY: + # "Directory not empty", likely due to filesystem lag + raise FileRemovalError(exc) + raise exc + + class KeyType(Enum): """Used for authentication keys - public or private""" diff --git a/cylc/flow/xtriggers/suite_state.py b/cylc/flow/xtriggers/suite_state.py new file mode 100644 index 00000000000..45a8418a832 --- /dev/null +++ b/cylc/flow/xtriggers/suite_state.py @@ -0,0 +1,83 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cylc.flow import LOG +import cylc.flow.flags +from cylc.flow.xtriggers.workflow_state import workflow_state + +if not cylc.flow.flags.cylc7_back_compat: + LOG.warning( + "The suite_state xtrigger is deprecated. " + "Please use the workflow_state xtrigger instead." + ) + + +def suite_state(suite, task, point, offset=None, status='succeeded', + message=None, cylc_run_dir=None, debug=False): + """Suite state xtrigger, required for interoperability with Cylc 7. + + * The suite_state xtrigger was renamed to workflow_state - + this breaks Cylc 7-8 interoperability. + * This suite_state xtrigger replicates workflow_state - + ensuring back-support. + + Arguments: + suite: + The workflow to interrogate. + task: + The name of the task to query. + point: + The cycle point. + offset: + The offset between the cycle this xtrigger is used in and the one + it is querying for as an ISO8601 time duration. + e.g. PT1H (one hour). + status: + The task status required for this xtrigger to be satisfied. + message: + The custom task output required for this xtrigger to be satisfied. + .. note:: + + This cannot be specified in conjunction with ``status``. + + cylc_run_dir: + The directory in which the workflow to interrogate. + + .. note:: + + This only needs to be supplied if the workflow is running in a + different location to what is specified in the global + configuration (usually ``~/cylc-run``). + + Returns: + tuple: (satisfied, results) + + satisfied: + True if ``satisfied`` else ``False``. + results: + Dictionary containing the args / kwargs which were provided + to this xtrigger. + + """ + return workflow_state( + workflow=suite, + task=task, + point=point, + offset=offset, + status=status, + message=message, + cylc_run_dir=cylc_run_dir + ) diff --git a/etc/bin/run-validate-tutorials b/etc/bin/run-validate-tutorials index e9b7be1ecde..72bf164fe2a 100755 --- a/etc/bin/run-validate-tutorials +++ b/etc/bin/run-validate-tutorials @@ -17,13 +17,15 @@ set -eu -cd "$(dirname "$0")" +cd "$(dirname "$0")/../../" -for FILE in $(echo ../../cylc/flow/etc/tutorial/*/.validate) ; do - echo "running $FILE" - ( - cd "$(dirname "$FILE")" - ./.validate - ) +for DIR in tutorial examples; do + echo "# Running tests for: $DIR" + for FILE in $(echo "cylc/flow/etc/$DIR/"*/.validate); do + echo "## Running test: $FILE" + ( + cd "$(dirname "$FILE")" + ./.validate + ) + done done - diff --git a/setup.cfg b/setup.cfg index a8ee30cf79d..83a4dbedbad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -220,6 +220,7 @@ cylc.xtriggers = echo = cylc.flow.xtriggers.echo:echo wall_clock = cylc.flow.xtriggers.wall_clock:wall_clock workflow_state = cylc.flow.xtriggers.workflow_state:workflow_state + suite_state = cylc.flow.xtriggers.suite_state:suite_state xrandom = cylc.flow.xtriggers.xrandom:xrandom [bdist_rpm] diff --git a/tests/functional/broadcast/10-file-1/broadcast.cylc b/tests/functional/broadcast/10-file-1/broadcast.cylc index f429f8a2740..243502ed547 100644 --- a/tests/functional/broadcast/10-file-1/broadcast.cylc +++ b/tests/functional/broadcast/10-file-1/broadcast.cylc @@ -1,6 +1,9 @@ script=""" printenv CYLC_FOOBAR +# This hash char should not cause the rest of the script to be stripped out +# - https://github.com/cylc/cylc-flow/pull/5933 + if (($CYLC_TASK_TRY_NUMBER < 2 )); then false fi diff --git a/tests/functional/cylc-lint/01.lint-toml.t b/tests/functional/cylc-lint/01.lint-toml.t index 183b69beaed..fbdd243e859 100644 --- a/tests/functional/cylc-lint/01.lint-toml.t +++ b/tests/functional/cylc-lint/01.lint-toml.t @@ -58,21 +58,20 @@ named_grep_ok "it returns a 728 upgrade code" "^\[U" "${TESTOUT}" # Add a pyproject.toml file cat > pyproject.toml <<__HERE__ -[cylc-lint] - # Check against these rules - rulesets = [ - "style" - ] - # do not check for these errors - ignore = [ - "S004" - ] - # do not lint files matching - # these globs: - exclude = [ - "sites/*.cylc", - ] - +[tool.cylc.lint] +# Check against these rules +rulesets = [ + "style" +] +# do not check for these errors +ignore = [ + "S004" +] +# do not lint files matching +# these globs: +exclude = [ + "sites/*.cylc", +] __HERE__ # Test that results are different: @@ -102,19 +101,19 @@ named_grep_ok "${TEST_NAME}-line-too-long-message" \ TEST_NAME="it_does_not_fail_if_max-line-length_set_but_ignored" cat > pyproject.toml <<__HERE__ -[cylc-lint] - # Check against these rules - rulesets = [ - "style" - ] - # do not check for these errors - ignore = [ - "${LINE_LEN_NO}" - ] - exclude = [ - "sites/*.cylc", - ] - max-line-length = 1 +[tool.cylc.lint] +# Check against these rules +rulesets = [ + "style" +] +# do not check for these errors +ignore = [ + "${LINE_LEN_NO}" +] +exclude = [ + "sites/*.cylc", +] +max-line-length = 1 __HERE__ run_ok "${TEST_NAME}" cylc lint grep_ok "rules and found no issues" "${TEST_NAME}.stdout" diff --git a/tests/functional/cylc-play/09-invalid-cp-opt.t b/tests/functional/cylc-play/09-invalid-cp-opt.t index 05fa6d2a2d6..ae633d5e75c 100644 --- a/tests/functional/cylc-play/09-invalid-cp-opt.t +++ b/tests/functional/cylc-play/09-invalid-cp-opt.t @@ -37,7 +37,7 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" run_fail "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --no-detach --stopcp='potato' -grep_ok "ERROR - Workflow shutting down .* potato" "${TEST_NAME_BASE}-run.stderr" +grep_ok "ERROR - Workflow shutting down .*potato" "${TEST_NAME_BASE}-run.stderr" # Check that we haven't got a database exists_ok "${WORKFLOW_RUN_DIR}/.service" diff --git a/tests/functional/xtriggers/04-suite_state.t b/tests/functional/xtriggers/04-suite_state.t new file mode 100644 index 00000000000..ece9cbe715e --- /dev/null +++ b/tests/functional/xtriggers/04-suite_state.t @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Test that deprecation warnings are printed appropriately for the suite_state +# xtrigger. + +. "$(dirname "$0")/test_header" + +set_test_number 4 + +init_workflow "$TEST_NAME_BASE" << __FLOW_CONFIG__ +[scheduling] + initial cycle point = 2000 + [[dependencies]] + [[[R1]]] + graph = @upstream => foo + [[xtriggers]] + upstream = suite_state(suite=thorin/oin/gloin, task=mithril, point=1) +[runtime] + [[foo]] +__FLOW_CONFIG__ + +msg='WARNING - The suite_state xtrigger is deprecated' + +TEST_NAME="${TEST_NAME_BASE}-val" +run_ok "$TEST_NAME" cylc validate "$WORKFLOW_NAME" + +grep_ok "$msg" "${TEST_NAME}.stderr" + +# Rename flow.cylc to suite.rc: +mv "${WORKFLOW_RUN_DIR}/flow.cylc" "${WORKFLOW_RUN_DIR}/suite.rc" + +TEST_NAME="${TEST_NAME_BASE}-val-2" +run_ok "$TEST_NAME" cylc validate "$WORKFLOW_NAME" + +grep_fail "$msg" "${TEST_NAME}.stderr" diff --git a/tests/integration/test_stop_after_cycle_point.py b/tests/integration/test_stop_after_cycle_point.py new file mode 100644 index 00000000000..663e03649f8 --- /dev/null +++ b/tests/integration/test_stop_after_cycle_point.py @@ -0,0 +1,132 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test logic pertaining to the stop after cycle points. + +This may be defined in different ways: +* In the workflow configuration. +* On the command line. +* Or loaded from the database. + +When the workflow hits the "stop after" point, it should be wiped (i.e. set +to None). +""" + +from typing import Optional + +from cylc.flow.cycling.integer import IntegerPoint +from cylc.flow.id import Tokens +from cylc.flow.workflow_status import StopMode + + +async def test_stop_after_cycle_point( + flow, + scheduler, + run, + reflog, + complete, +): + """Test the stop after cycle point. + + This ensures: + * The stop after point gets loaded from the config. + * The workflow stops when it hits this point. + * The point gets wiped when the workflow hits this point. + * The point is stored/retrieved from the DB as appropriate. + + """ + async def stops_after_cycle(schd) -> Optional[str]: + """Run the workflow until it stops and return the cycle point.""" + triggers = reflog(schd) + await complete(schd, timeout=2) + assert len(triggers) == 1 # only one task (i.e. cycle) should be run + return Tokens(list(triggers)[0][0], relative=True)['cycle'] + + def get_db_value(schd) -> Optional[str]: + """Return the cycle point value stored in the DB.""" + with schd.workflow_db_mgr.get_pri_dao() as pri_dao: + return dict(pri_dao.select_workflow_params())['stopcp'] + + config = { + 'scheduling': { + 'cycling mode': 'integer', + 'initial cycle point': '1', + 'stop after cycle point': '1', + 'graph': { + 'P1': 'a[-P1] => a', + }, + }, + } + id_ = flow(config) + schd = scheduler(id_, paused_start=False) + async with run(schd): + # the cycle point should be loaded from the workflow configuration + assert schd.config.stop_point == IntegerPoint('1') + + # this value should *not* be written to the database + assert get_db_value(schd) is None + + # the workflow should stop after cycle 1 + assert await stops_after_cycle(schd) == '1' + + # change the configured cycle point to "2" + config['scheduling']['stop after cycle point'] = '2' + id_ = flow(config, id_=id_) + schd = scheduler(id_, paused_start=False) + async with run(schd): + # the cycle point should be reloaded from the workflow configuration + assert schd.config.stop_point == IntegerPoint('2') + + # this value should not be written to the database + assert get_db_value(schd) is None + + # the workflow should stop after cycle 2 + assert await stops_after_cycle(schd) == '2' + + # override the configured value via the CLI option + schd = scheduler(id_, paused_start=False, **{'stopcp': '3'}) + async with run(schd): + # the CLI should take precedence over the config + assert schd.config.stop_point == IntegerPoint('3') + + # this value *should* be written to the database + assert get_db_value(schd) == '3' + + # the workflow should stop after cycle 3 + assert await stops_after_cycle(schd) == '3' + + # once the workflow hits this point, it should get cleared + assert get_db_value(schd) is None + + schd = scheduler(id_, paused_start=False) + async with run(schd): + # the workflow should fall back to the configured value + assert schd.config.stop_point == IntegerPoint('2') + + # override this value whilst the workflow is running + schd.command_stop( + cycle_point=IntegerPoint('4'), + mode=StopMode.REQUEST_CLEAN, + ) + assert schd.config.stop_point == IntegerPoint('4') + + # the new *should* be written to the database + assert get_db_value(schd) == '4' + + schd = scheduler(id_, paused_start=False) + async with run(schd): + # the workflow should stop after cycle 4 + assert await stops_after_cycle(schd) == '4' diff --git a/tests/integration/test_workflow_db_mgr.py b/tests/integration/test_workflow_db_mgr.py index fa15c9165b0..cf4fca7e064 100644 --- a/tests/integration/test_workflow_db_mgr.py +++ b/tests/integration/test_workflow_db_mgr.py @@ -14,10 +14,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import asyncio import pytest import sqlite3 +from typing import TYPE_CHECKING -from cylc.flow.scheduler import Scheduler +from cylc.flow.cycling.iso8601 import ISO8601Point + +if TYPE_CHECKING: + from cylc.flow.scheduler import Scheduler async def test_restart_number( @@ -29,7 +34,7 @@ async def test_restart_number( async def test(expected_restart_num: int, do_reload: bool = False): """(Re)start the workflow and check the restart number is as expected. """ - schd: Scheduler = scheduler(id_, paused_start=True) + schd: 'Scheduler' = scheduler(id_, paused_start=True) async with start(schd) as log: if do_reload: schd.command_reload_workflow() @@ -52,7 +57,7 @@ async def test(expected_restart_num: int, do_reload: bool = False): await test(expected_restart_num=3) -def db_remove_column(schd: Scheduler, table: str, column: str) -> None: +def db_remove_column(schd: 'Scheduler', table: str, column: str) -> None: """Remove a column from a scheduler DB table. ALTER TABLE DROP COLUMN is not supported by sqlite yet, so we have to copy @@ -82,7 +87,7 @@ async def test_db_upgrade_pre_803( id_ = flow(one_conf) # Run a scheduler to create a DB. - schd: Scheduler = scheduler(id_, paused_start=True) + schd: 'Scheduler' = scheduler(id_, paused_start=True) async with start(schd): assert ('n_restart', '0') in db_select(schd, False, 'workflow_params') @@ -90,7 +95,7 @@ async def test_db_upgrade_pre_803( db_remove_column(schd, "task_states", "is_manual_submit") db_remove_column(schd, "task_jobs", "flow_nums") - schd: Scheduler = scheduler(id_, paused_start=True) + schd: 'Scheduler' = scheduler(id_, paused_start=True) # Restart should fail due to the missing column. with pytest.raises(sqlite3.OperationalError): @@ -98,7 +103,7 @@ async def test_db_upgrade_pre_803( pass assert ('n_restart', '1') in db_select(schd, False, 'workflow_params') - schd: Scheduler = scheduler(id_, paused_start=True) + schd: 'Scheduler' = scheduler(id_, paused_start=True) # Run the DB upgrader for version 8.0.2 # (8.0.2 requires upgrade) @@ -117,7 +122,7 @@ async def test_workflow_param_rapid_toggle( https://github.com/cylc/cylc-flow/issues/5593 """ - schd: Scheduler = scheduler(flow(one_conf), paused_start=False) + schd: 'Scheduler' = scheduler(flow(one_conf), paused_start=False) async with run(schd): assert schd.is_paused is False schd.pause_workflow() @@ -127,3 +132,50 @@ async def test_workflow_param_rapid_toggle( w_params = dict(schd.workflow_db_mgr.pri_dao.select_workflow_params()) assert w_params['is_paused'] == '0' + + +async def test_record_only_non_clock_triggers( + flow, run, scheduler, complete, db_select +): + """Database does not record wall_clock xtriggers. + + https://github.com/cylc/cylc-flow/issues/5911 + + Includes: + - Not in DB: A normal wall clock xtrigger (wall_clock). + - In DB: An xrandom mis-labelled as wall_clock trigger DB). + - Not in DB: An execution retry xtrigger. + + @TODO: Refactor to use simulation mode to speedup after Simulation + mode upgrade bugfixes: This should speed this test up considerably. + """ + rawpoint = '1348' + id_ = flow({ + "scheduler": { + 'cycle point format': '%Y', + 'allow implicit tasks': True + }, + "scheduling": { + "initial cycle point": rawpoint, + "xtriggers": { + "another": "xrandom(100)", + "wall_clock": "xrandom(100, _=Not a real wall clock trigger)", + "real_wall_clock": "wall_clock()" + }, + "graph": { + "R1": """ + @another & @wall_clock & @real_wall_clock => foo + @real_wall_clock => bar + """ + } + }, + }) + schd = scheduler(id_, paused_start=False, run_mode='simulation') + + async with run(schd): + await complete(schd, timeout=20) + + # Assert that (only) the real clock trigger is not in the db: + assert db_select(schd, False, 'xtriggers', 'signature') == [ + ('xrandom(100)',), + ('xrandom(100, _=Not a real wall clock trigger)',)] diff --git a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html index 2b7137f8b3d..4443b972810 100644 --- a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html +++ b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html @@ -1,9 +1,9 @@ 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused 1■                                                            
-      - ̿● 1                                                                     
-         + ̿● ■ b                                                                
+   - one - paused                                                               
+      - ̿○ 1                                                                     
+           ̿○ b                                                                  
       - ̿○ 2                                                                     
          - ̿○ A                                                                  
               ̿○ a                                                               
@@ -18,4 +18,4 @@
                                                                                 
 quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
 filter tasks: T f s r R  filter workflows: W E p                                
-
\ No newline at end of file + diff --git a/tests/integration/tui/screenshots/test_auto_expansion.on-load.html b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html index df3c9f5c41b..d8f3ca712b8 100644 --- a/tests/integration/tui/screenshots/test_auto_expansion.on-load.html +++ b/tests/integration/tui/screenshots/test_auto_expansion.on-load.html @@ -1,7 +1,7 @@ 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
          - ̿○ A                                                                  
               ̿○ a                                                               
diff --git a/tests/integration/tui/screenshots/test_errors.open-error.html b/tests/integration/tui/screenshots/test_errors.open-error.html
index 142d0d88c72..31d842ca75b 100644
--- a/tests/integration/tui/screenshots/test_errors.open-error.html
+++ b/tests/integration/tui/screenshots/test_errors.open-error.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Error: Something went wrong :(                                              │
 │                                                                              │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │                                                                              │
 │                                                                              │
diff --git a/tests/integration/tui/screenshots/test_job_logs.01-job.out.html b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
index c1c767b98cf..6f6f87fdce8 100644
--- a/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
+++ b/tests/integration/tui/screenshots/test_job_logs.01-job.out.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  job: 1/a/01                                                                 │
 │  this is a job log                                                           │
diff --git a/tests/integration/tui/screenshots/test_job_logs.02-job.out.html b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
index 0eb94051201..e14a77b99e5 100644
--- a/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
+++ b/tests/integration/tui/screenshots/test_job_logs.02-job.out.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  job: 1/a/02                                                                 │
 │  this is a job log                                                           │
diff --git a/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
index bf5e3812008..c25e28ebd11 100644
--- a/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
+++ b/tests/integration/tui/screenshots/test_navigation.cursor-at-bottom-of-screen.html
@@ -10,7 +10,7 @@
                  ̿○ b12                                                          
             - ̿○ B2                                                              
                  ̿○ b21                                                          
-                 ̿○ b22                                                          
+                 ̿○ b22                                                          
                                                                                 
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
index cefab5264f4..4c988d41792 100644
--- a/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
+++ b/tests/integration/tui/screenshots/test_navigation.family-A-collapsed.html
@@ -3,7 +3,7 @@
 - ~cylc                                                                         
    - one - paused                                                               
       - ̿○ 1                                                                     
-         + ̿○ A                                                                  
+         + ̿○ A                                                                  
          - ̿○ B                                                                  
             - ̿○ B1                                                              
                  ̿○ b11                                                          
diff --git a/tests/integration/tui/screenshots/test_navigation.on-load.html b/tests/integration/tui/screenshots/test_navigation.on-load.html
index a0bd107742b..ea7d6577c9b 100644
--- a/tests/integration/tui/screenshots/test_navigation.on-load.html
+++ b/tests/integration/tui/screenshots/test_navigation.on-load.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + one - paused                                                               
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
index 6b26ced563e..e91225fef23 100644
--- a/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
+++ b/tests/integration/tui/screenshots/test_navigation.workflow-expanded.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
          - ̿○ A                                                                  
               ̿○ a1                                                              
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
index f28cced0714..5590254a28d 100644
--- a/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_offline_mutation.clean-mutation-selected.html
@@ -4,7 +4,7 @@
    + one - stop│  Action                                        │               
                │  < (cancel)                                 >  │               
                │                                                │               
-               │  < clean                                    >  │               
+               │  < clean                                    >  │               
                │  < log                                      >  │               
                │  < play                                     >  │               
                │  < reinstall-reload                         >  │               
diff --git a/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
index c2355597f78..eb04278317b 100644
--- a/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_offline_mutation.stop-all-mutation-selected.html
@@ -4,7 +4,7 @@
    + one - stop│  Action                                        │               
                │  < (cancel)                                 >  │               
                │                                                │               
-               │  < stop-all                                 >  │               
+               │  < stop-all                                 >  │               
                │                                                │               
                │                                                │               
                │                                                │               
diff --git a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
index af0a063a14f..8bfe41ea904 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
@@ -4,7 +4,7 @@
    - one - paus│  Action                                        │               
       - ̿○ 1    │  < (cancel)                                 >  │               
            ̿○ on│                                                │               
-               │  < hold                                     >  │               
+               │  < hold                                     >  │               
                │  < kill                                     >  │               
                │  < log                                      >  │               
                │  < poll                                     >  │               
diff --git a/tests/integration/tui/screenshots/test_online_mutation.task-selected.html b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
index 7d94d5e43dd..df7a917ff60 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.task-selected.html
@@ -3,7 +3,7 @@
 - ~cylc                                                                         
    - one - paused                                                               
       - ̿○ 1                                                                     
-           ̿○ one                                                                
+           ̿○ one                                                                
                                                                                 
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
index 74c02508239..aabddcc9cf6 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.1-workflow-running.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
            ̿○ one                                                                
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
index 09c3bbd7fb0..ee75b87558d 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.2-workflow-stopped.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - stopped                                                              
+   - one - stopped                                                              
         Workflow is not running                                                 
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
index 74c02508239..aabddcc9cf6 100644
--- a/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
+++ b/tests/integration/tui/screenshots/test_restart_reconnect.3-workflow-restarted.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
            ̿○ one                                                                
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
index f88e1b0124d..14be08c1066 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.log-file-selection.html
@@ -14,7 +14,7 @@
 │                     ┌──────────────────────────────────────┐                 │
 │                     │  Select File                         │                 │
 │                     │                                      │                 │
-│                     │  < config/01-start-01.cylc        >  │                 │
+│                     │  < config/01-start-01.cylc        >  │                 │
 │                     │  < config/flow-processed.cylc     >  │                 │
 │                     │  < scheduler/01-start-01.log      >  │                 │
 │                     │                                      │                 │
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
index 68dbcc10f9c..57d6e4066b1 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.scheduler-log-file.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  this is the                                                                 │
 │  scheduler log file                                                          │
diff --git a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
index 011734bc410..c7ab1e925ec 100644
--- a/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
+++ b/tests/integration/tui/screenshots/test_scheduler_logs.workflow-configuration-file.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  [scheduling]                                                                │
 │      [[graph]]                                                               │
diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
index 019184ec897..7d190cb6f4d 100644
--- a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
+++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.subscribed.html
@@ -1,7 +1,7 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
+   - one - paused                                                               
       - ̿○ 1                                                                     
            ̿○ one                                                                
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
index 8fa0f4329a1..39c2f247c97 100644
--- a/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
+++ b/tests/integration/tui/screenshots/test_subscribe_unsubscribe.unsubscribed.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + one - paused                                                               
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
index 4814892df7a..481d427be6f 100644
--- a/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
+++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.err.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  job: 1/a/02                                                                 │
 │  this is a job error                                                         │
diff --git a/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
index 0eb94051201..e14a77b99e5 100644
--- a/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
+++ b/tests/integration/tui/screenshots/test_task_logs.latest-job.out.html
@@ -1,7 +1,7 @@
 
┌──────────────────────────────────────────────────────────────────────────────┐
 │  Host: myhost                                                                │
 │  Path: mypath                                                                │
-│  < Select File                                                            >  │
+│  < Select File                                                            >  │
 │                                                                              │
 │  job: 1/a/02                                                                 │
 │  this is a job log                                                           │
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
index d54d9538d26..34601749413 100644
--- a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura-enter.html
@@ -12,7 +12,7 @@
                │  id: ~cylc/root                                │               
                │                                                │               
                │  Action                                        │               
-               │  < (cancel)                                 >  │               
+               │  < (cancel)                                 >  │               
                │                                                │               
                │  < stop-all                                 >  │               
                │                                                │               
diff --git a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
index 7f80031804b..d507b83d799 100644
--- a/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
+++ b/tests/integration/tui/screenshots/test_tui_basics.test-rakiura.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
                                                                                 
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-active.html b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
index 282f76735ed..994d00e911e 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-active.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + one - stopping                                                             
    + two - paused                                                               
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
index 8c26ce6ccc9..65b11fe4411 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-starts-with-t.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + tre - stopped                                                              
    + two - paused                                                               
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
index 8c26ce6ccc9..65b11fe4411 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped-or-paused.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + tre - stopped                                                              
    + two - paused                                                               
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
index 1ff602df101..3913180f312 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.filter-stopped.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + tre - stopped                                                              
                                                                                 
                                                                                 
diff --git a/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
index 0651eedec30..d8cf9ff39fd 100644
--- a/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
+++ b/tests/integration/tui/screenshots/test_workflow_states.unfiltered.html
@@ -1,6 +1,6 @@
 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
-- ~cylc                                                                         
+- ~cylc                                                                         
    + one - stopping                                                             
    + tre - stopped                                                              
    + two - paused                                                               
diff --git a/tests/unit/parsec/test_validate.py b/tests/unit/parsec/test_validate.py
index 8c73b840eb3..3e24bcf6365 100644
--- a/tests/unit/parsec/test_validate.py
+++ b/tests/unit/parsec/test_validate.py
@@ -18,12 +18,13 @@
 from typing import List
 
 import pytest
-from pytest import approx
+from pytest import approx, param
 
 from cylc.flow.parsec.config import ConfigNode as Conf
 from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
 from cylc.flow.parsec.exceptions import IllegalValueError
 from cylc.flow.parsec.validate import (
+    BroadcastConfigValidator,
     CylcConfigValidator as VDR,
     DurationFloat,
     ListValueError,
@@ -456,11 +457,9 @@ def test_coerce_int_list():
             validator.coerce_int_list(value, ['whatever'])
 
 
-def test_coerce_str():
-    """Test coerce_str."""
-    validator = ParsecValidator()
-    # The good
-    for value, result in [
+@pytest.mark.parametrize(
+    'value, expected',
+    [
         ('', ''),
         ('Hello World!', 'Hello World!'),
         ('"Hello World!"', 'Hello World!'),
@@ -474,9 +473,15 @@ def test_coerce_str():
          'Hello:\n    foo\nGreet\n    baz'),
         ('False', 'False'),
         ('None', 'None'),
-        (['a', 'b'], 'a\nb')
-    ]:
-        assert validator.coerce_str(value, ['whatever']) == result
+        (['a', 'b'], 'a\nb'),
+        ('abc#def', 'abc'),
+    ]
+)
+def test_coerce_str(value: str, expected: str):
+    """Test coerce_str."""
+    validator = ParsecValidator()
+    # The good
+    assert validator.coerce_str(value, ['whatever']) == expected
 
 
 def test_coerce_str_list():
@@ -498,9 +503,51 @@ def test_coerce_str_list():
         assert validator.coerce_str_list(value, ['whatever']) == results
 
 
-def test_strip_and_unquote():
+@pytest.mark.parametrize('value, expected', [
+    param(
+        "'a'",
+        'a',
+        id="single quotes"
+    ),
+    param(
+        '"\'a\'"',
+        "'a'",
+        id="single quotes inside double quotes"
+    ),
+    param(
+        '" a b" # comment',
+        ' a b',
+        id="comment outside"
+    ),
+    param(
+        '"""bene\ngesserit"""',
+        'bene\ngesserit',
+        id="multiline double quotes"
+    ),
+    param(
+        "'''kwisatz\n  haderach'''",
+        'kwisatz\n  haderach',
+        id="multiline single quotes"
+    ),
+    param(
+        '"""a\nb"""  # comment',
+        'a\nb',
+        id="multiline with comment outside"
+    ),
+])
+def test_unquote(value: str, expected: str):
+    """Test strip_and_unquote."""
+    assert ParsecValidator._unquote(['a'], value) == expected
+
+
+@pytest.mark.parametrize('value', [
+    '"""',
+    "'''",
+    "'don't do this'",
+])
+def test_strip_and_unquote__bad(value: str):
     with pytest.raises(IllegalValueError):
-        ParsecValidator.strip_and_unquote(['a'], '"""')
+        ParsecValidator.strip_and_unquote(['a'], value)
 
 
 def test_strip_and_unquote_list_parsec():
@@ -692,3 +739,23 @@ def test_type_help_examples():
                 except Exception:
                     raise Exception(
                         f'Example "{example}" failed for type "{vdr}"')
+
+
+@pytest.mark.parametrize('value, expected', [
+    param(
+        """
+        a="don't have a cow"
+        a=${a#*have}
+        echo "$a" # let's see what happens
+        """,
+        "a=\"don't have a cow\"\na=${a#*have}\necho \"$a\" # let's see what happens",
+        id="multiline"
+    ),
+    param(
+        '"sleep 30 # ja!"  ',
+        'sleep 30 # ja!',
+        id="quoted"
+    ),
+])
+def test_broadcast_coerce_str(value: str, expected: str):
+    assert BroadcastConfigValidator.coerce_str(value, ['whatever']) == expected
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index db3ae052ca3..f5f4bb8cc98 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -16,15 +16,18 @@
 # along with this program.  If not, see .
 """Tests `cylc lint` CLI Utility."""
 
+import logging
 from pathlib import Path
 from pprint import pformat
 import re
+from textwrap import dedent
 from types import SimpleNamespace
 
 import pytest
 from pytest import param
 
 from cylc.flow.scripts.lint import (
+    LINT_SECTION,
     MANUAL_DEPRECATIONS,
     get_cylc_files,
     get_pyproject_toml,
@@ -148,6 +151,8 @@
     [[and_another_thing]]
         [[[remote]]]
             host = `rose host-select thingy`
+
+%include foo.cylc
 """
 
 
@@ -228,7 +233,6 @@ def assert_contains(items, contains, instances=None):
     16: 3,
 }
 
-
 @pytest.mark.parametrize(
     # 11 won't be tested because there is no jinja2 shebang
     'number', set(range(1, len(MANUAL_DEPRECATIONS) + 1)) - {11}
@@ -471,52 +475,85 @@ def test_get_upg_info(fixture_get_deprecations, findme):
 
 
 @pytest.mark.parametrize(
-    'expect',
+    'settings, expected',
     [
-        param({
-            'rulesets': ['style'],
-            'ignore': ['S004'],
-            'exclude': ['sites/*.cylc']},
-            id="it returns what we want"
+        param(
+            """
+            rulesets = ['style']
+            ignore = ['S004']
+            exclude = ['sites/*.cylc']
+            """,
+            {
+                'rulesets': ['style'],
+                'ignore': ['S004'],
+                'exclude': ['sites/*.cylc'],
+                'max-line-length': None,
+            },
+            id="returns what we want"
         ),
-        param({
-            'northgate': ['sites/*.cylc'],
-            'mons-meg': 42},
-            id="it only returns requested sections"
+        param(
+            """
+            northgate = ['sites/*.cylc']
+            mons-meg = 42
+            """,
+            (CylcError, ".*northgate"),
+            id="invalid settings fail validation"
         ),
-        param({
-            'max-line-length': 22},
-            id='it sets max line length'
+        param(
+            "max-line-length = 22",
+            {
+                'exclude': [],
+                'ignore': [],
+                'rulesets': [],
+                'max-line-length': 22,
+            },
+            id='sets max line length'
         )
     ]
 )
-def test_get_pyproject_toml(tmp_path, expect):
+def test_get_pyproject_toml(tmp_path, settings, expected):
     """It returns only the lists we want from the toml file."""
-    tomlcontent = "[cylc-lint]"
-    permitted_keys = ['rulesets', 'ignore', 'exclude', 'max-line-length']
-
-    for section, value in expect.items():
-        tomlcontent += f'\n{section} = {value}'
+    tomlcontent = "[tool.cylc.lint]\n" + dedent(settings)
     (tmp_path / 'pyproject.toml').write_text(tomlcontent)
-    tomldata = get_pyproject_toml(tmp_path)
 
-    control = {}
-    for key in permitted_keys:
-        control[key] = expect.get(key, [])
-    assert tomldata == control
+    if isinstance(expected, tuple):
+        exc, match = expected
+        with pytest.raises(exc, match=match):
+            get_pyproject_toml(tmp_path)
+    else:
+        assert get_pyproject_toml(tmp_path) == expected
 
 
-@pytest.mark.parametrize('tomlfile', [None, '', '[cylc-lint]'])
+@pytest.mark.parametrize(
+    'tomlfile',
+    [None, '', '[tool.cylc.lint]', '[cylc-lint]']
+)
 def test_get_pyproject_toml_returns_blank(tomlfile, tmp_path):
     if tomlfile is not None:
         tfile = (tmp_path / 'pyproject.toml')
         tfile.write_text(tomlfile)
-    expect = {k: [] for k in {
-        'exclude', 'ignore', 'max-line-length', 'rulesets'
-    }}
+    expect = {
+        'exclude': [], 'ignore': [], 'max-line-length': None, 'rulesets': []
+    }
     assert get_pyproject_toml(tmp_path) == expect
 
 
+def test_get_pyproject_toml__depr(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+):
+    """It warns if the section is deprecated."""
+    file = tmp_path / 'pyproject.toml'
+    caplog.set_level(logging.WARNING)
+
+    file.write_text(f'[{LINT_SECTION}]\nmax-line-length=14')
+    assert get_pyproject_toml(tmp_path)['max-line-length'] == 14
+    assert not caplog.text
+
+    file.write_text('[cylc-lint]\nmax-line-length=17')
+    assert get_pyproject_toml(tmp_path)['max-line-length'] == 17
+    assert "[cylc-lint] section in pyproject.toml is deprecated" in caplog.text
+
+
 @pytest.mark.parametrize(
     'input_, error',
     [
@@ -649,3 +686,18 @@ def test_indents(spaces, expect):
         assert expect in result
     else:
         assert not result
+
+
+def test_noqa():
+    """Comments turn of checks.
+
+    """
+    output = lint_text(
+        'foo = bar#noqa\n'
+        'qux = baz # noqa: S002\n'
+        'buzz = food # noqa: S007\n'
+        'quixotic = foolish # noqa: S007, S992 S002\n',
+        ['style']
+    )
+    assert len(output.messages) == 1
+    assert 'flow.cylc:3' in output.messages[0]
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 67f5f0f559a..494dd98e62b 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -114,7 +114,7 @@ def test_xfunction_imports(
         """
         flow_file.write_text(flow_config)
         workflow_config = WorkflowConfig(
-            workflow="name_a_tree", fpath=flow_file, options=Mock(spec=[]),
+            workflow="name_a_tree", fpath=flow_file, options=SimpleNamespace(),
             xtrigger_mgr=xtrigger_mgr
         )
         assert 'tree' in workflow_config.xtrigger_mgr.functx_map
@@ -148,7 +148,7 @@ def test_xfunction_import_error(self, mock_glbl_cfg, tmp_path):
             WorkflowConfig(
                 workflow="caiman_workflow",
                 fpath=flow_file,
-                options=Mock(spec=[])
+                options=SimpleNamespace()
             )
         assert "not found" in str(excinfo.value)
 
@@ -179,7 +179,7 @@ def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path):
         flow_file.write_text(flow_config)
         with pytest.raises(XtriggerConfigError) as excinfo:
             WorkflowConfig(workflow="capybara_workflow", fpath=flow_file,
-                           options=Mock(spec=[]))
+                           options=SimpleNamespace())
         assert "not found" in str(excinfo.value)
 
     def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path):
@@ -211,7 +211,7 @@ def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path):
             WorkflowConfig(
                 workflow="workflow_with_not_callable",
                 fpath=flow_file,
-                options=Mock(spec=[])
+                options=SimpleNamespace()
             )
         assert "callable" in str(excinfo.value)
 
@@ -378,14 +378,16 @@ def test_process_icp(
         expected_err: Exception class expected to be raised plus the message.
     """
     set_cycling_type(cycling_type, time_zone="+0530")
-    mocked_config = Mock(cycling_type=cycling_type)
-    mocked_config.cfg = {
-        'scheduling': {
-            'initial cycle point constraints': [],
-            **scheduling_cfg
-        }
-    }
-    mocked_config.options.icp = None
+    mocked_config = SimpleNamespace(
+        cycling_type=cycling_type,
+        options=SimpleNamespace(icp=None),
+        cfg={
+            'scheduling': {
+                'initial cycle point constraints': [],
+                **scheduling_cfg
+            },
+        },
+    )
     monkeypatch.setattr('cylc.flow.config.get_current_time_string',
                         lambda: '20050102T0615+0530')
 
@@ -461,9 +463,10 @@ def test_process_startcp(
         expected_err: Expected exception.
     """
     set_cycling_type(ISO8601_CYCLING_TYPE, time_zone="+0530")
-    mocked_config = Mock(initial_point='18990501T0000+0530')
-    mocked_config.options.startcp = startcp
-    mocked_config.options.starttask = starttask
+    mocked_config = SimpleNamespace(
+        initial_point='18990501T0000+0530',
+        options=SimpleNamespace(startcp=startcp, starttask=starttask),
+    )
     monkeypatch.setattr('cylc.flow.config.get_current_time_string',
                         lambda: '20050102T0615+0530')
     if expected_err is not None:
@@ -662,17 +665,20 @@ def test_process_fcp(
         expected_err: Exception class expected to be raised plus the message.
     """
     set_cycling_type(cycling_type, time_zone='+0530')
-    mocked_config = Mock(cycling_type=cycling_type)
-    mocked_config.cfg = {
-        'scheduling': {
-            'final cycle point constraints': [],
-            **scheduling_cfg
-        }
-    }
-    mocked_config.initial_point = loader.get_point(
-        scheduling_cfg['initial cycle point']).standardise()
-    mocked_config.final_point = None
-    mocked_config.options.fcp = options_fcp
+    mocked_config = SimpleNamespace(
+        cycling_type=cycling_type,
+        cfg={
+            'scheduling': {
+                'final cycle point constraints': [],
+                **scheduling_cfg,
+            },
+        },
+        initial_point=loader.get_point(
+            scheduling_cfg['initial cycle point']
+        ).standardise(),
+        final_point = None,
+        options=SimpleNamespace(fcp=options_fcp),
+    )
 
     if expected_err:
         err, msg = expected_err
@@ -692,24 +698,28 @@ def test_process_fcp(
     [
         pytest.param(
             None, None, None, None, None,
-            id="No stopcp"
+            id="no-stopcp"
         ),
         pytest.param(
             '1993', None, '1993', None, None,
-            id="From config by default"
+            id="stopcp"
         ),
         pytest.param(
             '1993', '1066', '1066', '1066', None,
-            id="From options"
+            id="stop-cp-and-cli-option"
         ),
         pytest.param(
             '1993', 'reload', '1993', None, None,
-            id="From cfg if --stopcp=reload on restart"
+            id="stop-cp-and-cli-reload-option"
         ),
         pytest.param(
             '3000', None, None, None,
             "will have no effect as it is after the final cycle point",
-            id="stopcp > fcp"
+            id="stopcp-beyond-fcp"
+        ),
+        pytest.param(
+            '+P12Y -P2Y', None, '2000', None, None,
+            id="stopcp-relative-to-icp"
         ),
     ]
 )
@@ -734,12 +744,13 @@ def test_process_stop_cycle_point(
     set_cycling_type(ISO8601_CYCLING_TYPE, dump_format='CCYY')
     caplog.set_level(logging.WARNING, CYLC_LOG)
     fcp = loader.get_point('2012').standardise()
-    mock_config = Mock(
+    mock_config = SimpleNamespace(
         cfg={
             'scheduling': {
                 'stop after cycle point': cfg_stopcp
             }
         },
+        initial_point=ISO8601Point('1990'),
         final_point=fcp,
         stop_point=None,
         options=RunOptions(stopcp=options_stopcp),
@@ -887,7 +898,7 @@ def test_prelim_process_graph(
             processing.
         expected_err: Exception class expected to be raised plus the message.
     """
-    mock_config = Mock(cfg={
+    mock_config = SimpleNamespace(cfg={
         'scheduling': scheduling_cfg
     })
 
@@ -913,13 +924,15 @@ def _test(utc_mode, expected, expected_warnings=0):
                 UTC mode = {utc_mode['glbl']}
             '''
         )
-        mock_config = Mock()
-        mock_config.cfg = {
-            'scheduler': {
-                'UTC mode': utc_mode['workflow']
-            }
-        }
-        mock_config.options.utc_mode = utc_mode['stored']
+        mock_config = SimpleNamespace(
+            cfg={
+                'scheduler': {
+                    'UTC mode': utc_mode['workflow']
+                }
+            },
+            options=SimpleNamespace(utc_mode=utc_mode['stored']),
+        )
+
         WorkflowConfig.process_utc_mode(mock_config)
         assert mock_config.cfg['scheduler']['UTC mode'] is expected
         assert get_utc_mode() is expected
@@ -963,13 +976,15 @@ def test_cycle_point_tz(caplog, monkeypatch):
 
     def _test(cp_tz, utc_mode, expected, expected_warnings=0):
         set_utc_mode(utc_mode)
-        mock_config = Mock()
-        mock_config.cfg = {
-            'scheduler': {
-                'cycle point time zone': cp_tz['workflow']
-            }
-        }
-        mock_config.options.cycle_point_tz = cp_tz['stored']
+        mock_config = SimpleNamespace(
+            cfg={
+                'scheduler': {
+                    'cycle point time zone': cp_tz['workflow'],
+                },
+            },
+            options=SimpleNamespace(cycle_point_tz=cp_tz['stored']),
+        )
+
         WorkflowConfig.process_cycle_point_tz(mock_config)
         assert mock_config.cfg['scheduler'][
             'cycle point time zone'] == expected
@@ -1148,7 +1163,7 @@ def test_process_runahead_limit(
     set_cycling_type: Callable
 ) -> None:
     set_cycling_type(cycling_type)
-    mock_config = Mock(cycling_type=cycling_type)
+    mock_config = SimpleNamespace(cycling_type=cycling_type)
     mock_config.cfg = {
         'scheduling': {
             'runahead limit': runahead_limit
@@ -1170,7 +1185,7 @@ def test_check_circular(opt, monkeypatch, caplog, tmp_flow_config):
     # ----- Setup -----
     caplog.set_level(logging.WARNING, CYLC_LOG)
 
-    options = Mock(spec=[], is_validate=True)
+    options = SimpleNamespace(is_validate=True)
     if opt:
         setattr(options, opt, True)
 
@@ -1739,7 +1754,7 @@ def test_cylc_env_at_parsing(
 
     # Parse the workflow config then check the environment.
     WorkflowConfig(
-        workflow="name", fpath=flow_file, options=Mock(spec=[]),
+        workflow="name", fpath=flow_file, options=SimpleNamespace(),
         run_dir=run_dir
     )
 
diff --git a/tests/unit/xtriggers/test_workflow_state.py b/tests/unit/xtriggers/test_workflow_state.py
index b3d25737cc2..ed02750f04a 100644
--- a/tests/unit/xtriggers/test_workflow_state.py
+++ b/tests/unit/xtriggers/test_workflow_state.py
@@ -42,7 +42,7 @@ def test_inferred_run(tmp_run_dir: Callable, monkeymock: MonkeyMock):
     assert results['workflow'] == expected_workflow_id
 
 
-def test_back_compat(tmp_run_dir):
+def test_back_compat(tmp_run_dir, caplog):
     """Test workflow_state xtrigger backwards compatibility with Cylc 7
     database."""
     id_ = 'celebrimbor'
@@ -80,7 +80,15 @@ def test_back_compat(tmp_run_dir):
     finally:
         conn.close()
 
+    # Test workflow_state function
     satisfied, _ = workflow_state(id_, task='mithril', point='2012')
     assert satisfied
     satisfied, _ = workflow_state(id_, task='arkenstone', point='2012')
     assert not satisfied
+
+    # Test back-compat (old suite_state function)
+    from cylc.flow.xtriggers.suite_state import suite_state
+    satisfied, _ = suite_state(suite=id_, task='mithril', point='2012')
+    assert satisfied
+    satisfied, _ = suite_state(suite=id_, task='arkenstone', point='2012')
+    assert not satisfied