From bcf16294b03b82e09258e145785f773e4d59e65a Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:17:15 +0000 Subject: [PATCH 1/7] Cylc will no longer support Empy from 8.4.0. This commit removes empy functionality and tests from Cylc Rose. --- .github/workflows/tests.yml | 2 +- README.md | 2 +- cylc/rose/__init__.py | 8 +- cylc/rose/utilities.py | 25 +----- tests/functional/01_empy/flow.cylc | 36 -------- .../functional/01_empy/processed.conf.control | 30 ------- tests/functional/01_empy/rose-suite.conf | 8 -- .../08_template_engine_conflict/flow.cylc | 10 --- .../rose-suite.conf | 2 - tests/functional/test_pre_configure.py | 12 --- tests/functional/test_rose_stem.py | 2 +- tests/unit/test_config_node.py | 29 +----- tests/unit/test_config_tree.py | 88 ++++++------------- tests/unit/test_functional_post_install.py | 42 --------- tests/unit/test_rose_stem_units.py | 3 +- 15 files changed, 38 insertions(+), 261 deletions(-) delete mode 100644 tests/functional/01_empy/flow.cylc delete mode 100644 tests/functional/01_empy/processed.conf.control delete mode 100644 tests/functional/01_empy/rose-suite.conf delete mode 100644 tests/functional/08_template_engine_conflict/flow.cylc delete mode 100644 tests/functional/08_template_engine_conflict/rose-suite.conf diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 41cf8918..92508bec 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,7 @@ jobs: uses: cylc/release-actions/install-cylc-components@v1 with: cylc_flow: true - cylc_flow_opts: 'empy' + cylc_flow_opts: '' metomi_rose: true metomi_rose_opts: '' diff --git a/README.md b/README.md index 60e3c215..ed2b28a1 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ environment for Rose integration. This plugin provides support for the `rose-suite.conf` file, namely: -* Jinja2/EmPy template variables. +* Jinja2 template variables. * Scheduler environment variables. * File installation. * Optional configurations. diff --git a/cylc/rose/__init__.py b/cylc/rose/__init__.py index 8c79ba7b..19489f7e 100644 --- a/cylc/rose/__init__.py +++ b/cylc/rose/__init__.py @@ -59,15 +59,15 @@ ``opts=A B C``, A space limited list of optional configs. ``[env]``, "Variables which the cylc-rose plugin will export to the environment." - ``[template variables]``, "Variables which can be used by Jinja2 or Empy + ``[template variables]``, "Variables which can be used by Jinja2 or in the workflow definition." ``[file:destination]``, A file from one or more sources to be installed. .. note:: - For compatibility with Cylc 7, sections ``[suite.rc:empy]`` and - ``[suite.rc:jinja2]`` will be processed, but are deprecated and provided - for ease of porting Cylc 7 workflows. + For compatibility with Cylc 7, section ``[suite.rc:jinja2]`` will be + processed, but is deprecated and provided for ease of porting Cylc 7 + workflows. The ``global.cylc`` file diff --git a/cylc/rose/utilities.py b/cylc/rose/utilities.py index 6ed6e93a..6e7f6b9c 100644 --- a/cylc/rose/utilities.py +++ b/cylc/rose/utilities.py @@ -49,7 +49,7 @@ ROSE_SUITE_OPT_CONF_KEYS = 'ROSE_SUITE_OPT_CONF_KEYS' -SECTIONS = {'jinja2:suite.rc', 'empy:suite.rc', 'template variables'} +SECTIONS = {'jinja2:suite.rc', 'template variables'} SET_BY_CYLC = 'set by Cylc' ROSE_ORIG_HOST_INSTALLED_OVERRIDE_STRING = ( ' ROSE_ORIG_HOST set by cylc install.' @@ -234,8 +234,6 @@ def id_templating_section( templating = None if section and 'jinja2' in section: templating = 'jinja2:suite.rc' - elif section and 'empy' in section: - templating = 'empy:suite.rc' if not templating: templating = 'template variables' @@ -340,7 +338,7 @@ def merge_rose_cylc_suite_install_conf(old, new): >>> merge_rose_cylc_suite_install_conf(old, new)['opts'] {'value': 'a b c d e', 'state': '', 'comments': []} """ - # remove jinja2/empy:suite.rc from old if template variables in new + # remove jinja2:suite.rc from old if template variables in new for before, after in itertools.permutations(SECTIONS, 2): if new.value.get(after, '') and old.value.get(before, ''): # Choosing not to warn if user downgrades here because @@ -766,7 +764,6 @@ def deprecation_warnings(config_tree): Logs a warning for deprecated items: - "root-dir" - "jinja2:suite.rc" - - "empy:suite.rc" - root-dir If ALL_MODES is True this deprecation will ignore whether there is a @@ -774,24 +771,12 @@ def deprecation_warnings(config_tree): """ deprecations = { - 'empy:suite.rc': { - MESSAGE: ( - "'rose-suite.conf[empy:suite.rc]' is deprecated." - " Use [template variables] instead."), - ALL_MODES: False, - }, 'jinja2:suite.rc': { MESSAGE: ( "'rose-suite.conf[jinja2:suite.rc]' is deprecated." " Use [template variables] instead."), ALL_MODES: False, }, - 'empy:flow.cylc': { - MESSAGE: ( - "'rose-suite.conf[empy:flow.cylc]' is not used by Cylc." - " Use [template variables] instead."), - ALL_MODES: False, - }, 'jinja2:flow.cylc': { MESSAGE: ( "'rose-suite.conf[jinja2:flow.cylc]' is not used by Cylc." @@ -951,9 +936,7 @@ def record_cylc_install_options( # Get Values for standard ROSE variable ROSE_ORIG_HOST. rose_orig_host = get_host() - for section in [ - 'env', 'jinja2:suite.rc', 'empy:suite.rc', 'template variables' - ]: + for section in list(SECTIONS) + ['env']: if section in cli_config: cli_config[section].set(['ROSE_ORIG_HOST'], rose_orig_host) cli_config[section]['ROSE_ORIG_HOST'].comments = [ @@ -1045,7 +1028,7 @@ def retrieve_installed_cli_opts(srcdir, opts): # Get --suite-defines/-S # Work out whether user has used "template variables", "jinja2:suite.rc" - # or "empy:suite.rc" (There is an assumption that they aren't mixing + # (There is an assumption that they aren't mixing # them that is not guarded against): for section in SECTIONS: if cli_config.value.get(section, False): diff --git a/tests/functional/01_empy/flow.cylc b/tests/functional/01_empy/flow.cylc deleted file mode 100644 index 73f24035..00000000 --- a/tests/functional/01_empy/flow.cylc +++ /dev/null @@ -1,36 +0,0 @@ -#!empy -[meta] - title = "Add empy vars from a rose-suite.conf" - description = """ - Natively, in Cylc! - """ - -[scheduling] - initial cycle point = @(ICP) - final cycle point = @(FCP) - cycling mode = integer - [[graph]] -@[ for member in MEMBERS ] - P1 = @(TASK1) => @(TASK2)_@(member) => @(TASK3) -@[ end for ] -@[ for key, value in SAMUELJOHNSON.items() ] - P1 = @(TASK3) => @(value)_auf_deutsch_ist_@(key) => fin -@[ end for ] - -[runtime] - [[root]] - script = echo "This task is ${CYLC_TASK_ID}" - - [[@(TASK1)]] - -@[ for member in MEMBERS ] - [[@(TASK2)_@(member)]] -@[ end for ] - - [[@(TASK3)]] - -@[ for key, value in SAMUELJOHNSON.items() ] - [[@(value)_auf_deutsch_ist_@(key)]] -@[ end for ] - - [[fin]] diff --git a/tests/functional/01_empy/processed.conf.control b/tests/functional/01_empy/processed.conf.control deleted file mode 100644 index 94376282..00000000 --- a/tests/functional/01_empy/processed.conf.control +++ /dev/null @@ -1,30 +0,0 @@ -[meta] - title = "Add empy vars from a rose-suite.conf" - description = """ - Natively, in Cylc! - """ -[scheduling] - initial cycle point = 1 - final cycle point = 1 - cycling mode = integer - [[graph]] - P1 = respond => plunge_control => allow - P1 = respond => plunge_yan => allow - P1 = respond => plunge_tan => allow - P1 = respond => plunge_tethera => allow - P1 = allow => 1_auf_deutsch_ist_ein => fin - P1 = allow => 2_auf_deutsch_ist_zwei => fin - P1 = allow => 3_auf_deutsch_ist_drei => fin -[runtime] - [[root]] - script = echo "This task is ${CYLC_TASK_ID}" - [[respond]] - [[plunge_control]] - [[plunge_yan]] - [[plunge_tan]] - [[plunge_tethera]] - [[allow]] - [[1_auf_deutsch_ist_ein]] - [[2_auf_deutsch_ist_zwei]] - [[3_auf_deutsch_ist_drei]] - [[fin]] diff --git a/tests/functional/01_empy/rose-suite.conf b/tests/functional/01_empy/rose-suite.conf deleted file mode 100644 index 5500e6db..00000000 --- a/tests/functional/01_empy/rose-suite.conf +++ /dev/null @@ -1,8 +0,0 @@ -[empy:suite.rc] -ICP=1 -FCP=1 -TASK1="respond" -TASK2="plunge" -TASK3="allow" -MEMBERS=["control", "yan", "tan", "tethera"] -SAMUELJOHNSON={"ein": 1, "zwei": 2, "drei": 3} diff --git a/tests/functional/08_template_engine_conflict/flow.cylc b/tests/functional/08_template_engine_conflict/flow.cylc deleted file mode 100644 index f2495fd3..00000000 --- a/tests/functional/08_template_engine_conflict/flow.cylc +++ /dev/null @@ -1,10 +0,0 @@ -#!jinja2 -[scheduling] - initial cycle point = now - [[dependencies]] - R1 = """ - x - """ - -[runtime] - [[x]] diff --git a/tests/functional/08_template_engine_conflict/rose-suite.conf b/tests/functional/08_template_engine_conflict/rose-suite.conf deleted file mode 100644 index b05f4c50..00000000 --- a/tests/functional/08_template_engine_conflict/rose-suite.conf +++ /dev/null @@ -1,2 +0,0 @@ -[empy:suite.rc] -BOOL=True diff --git a/tests/functional/test_pre_configure.py b/tests/functional/test_pre_configure.py index b8cda69d..ed3685be 100644 --- a/tests/functional/test_pre_configure.py +++ b/tests/functional/test_pre_configure.py @@ -37,14 +37,6 @@ '07_cli_override', 'failed 1.1\n(add --verbose for more context)', id='template variable not set' - ), - param( - '08_template_engine_conflict', - ( - 'A plugin set the templating engine to empy which does' - ' not match #!jinja2 set in flow.cylc.' - ), - id='template engine conflict' ) ] ) @@ -60,7 +52,6 @@ async def test_validate_fail(srcdir, expect, cylc_validate_cli): 'srcdir, envvars, args', [ ('00_jinja2_basic', None, None), - ('01_empy', None, None), ('02_env', None, None), ( '04_opts_set_from_env', @@ -92,7 +83,6 @@ async def test_validate(monkeypatch, srcdir, envvars, args, cylc_validate_cli): 'srcdir, envvars', [ ('00_jinja2_basic', None), - ('01_empy', None), ( '04_opts_set_from_env', {'ROSE_SUITE_OPT_CONF_KEYS': 'Gaelige'}, @@ -138,9 +128,7 @@ def test_warn_if_root_dir_set(root_dir_config, tmp_path, caplog): ) @pytest.mark.parametrize( 'rose_config', [ - 'empy:suite.rc', 'jinja2:suite.rc', - 'empy:flow.cylc', 'jinja2:flow.cylc', 'JinjA2:flOw.cylC', ] diff --git a/tests/functional/test_rose_stem.py b/tests/functional/test_rose_stem.py index 878ff1a7..d2b76063 100644 --- a/tests/functional/test_rose_stem.py +++ b/tests/functional/test_rose_stem.py @@ -188,7 +188,7 @@ async def test_manual_project_override(rose_stem, rose_stem_project): check_template_variables( { "RUN_NAMES": - "['earl_grey', 'milk', 'sugar', " "'spoon', 'cup', 'milk']", + "['earl_grey', 'milk', 'sugar', 'spoon', 'cup', 'milk']", "SOURCE_FOO": '"fcm:foo.x_tr@head"', "HOST_SOURCE_FOO": '"fcm:foo.x_tr@head"', "SOURCE_BAR": f'"{rose_stem_project}"', diff --git a/tests/unit/test_config_node.py b/tests/unit/test_config_node.py index 6bb908b6..c49fb342 100644 --- a/tests/unit/test_config_node.py +++ b/tests/unit/test_config_node.py @@ -28,7 +28,6 @@ from cylc.rose.utilities import ( ROSE_ORIG_HOST_INSTALLED_OVERRIDE_STRING, - MultipleTemplatingEnginesError, add_cylc_install_to_rose_conf_node_opts, deprecation_warnings, dump_rose_log, @@ -200,14 +199,6 @@ def test_dump_rose_log(monkeypatch, tmp_path): None, id="OK - jinja2:suite.rc", ), - pytest.param( - ( - (['empy:suite.rc', 'foo'], 'Hello World'), - ), - 'empy:suite.rc', - None, - id="OK - empy:suite.rc", - ), pytest.param( ( ('opt', 'a b'), @@ -216,24 +207,6 @@ def test_dump_rose_log(monkeypatch, tmp_path): None, id="OK - no template variables section set", ), - pytest.param( - ( - (['empy:suite.rc', 'foo'], 'Hello World'), - (['jinja2:suite.rc', 'foo'], 'Hello World'), - ), - None, - MultipleTemplatingEnginesError, - id="FAILS - empy and jinja2 sections set", - ), - pytest.param( - ( - (['empy:suite.rc', 'foo'], 'Hello World'), - (['template variables', 'foo'], 'Hello World'), - ), - None, - MultipleTemplatingEnginesError, - id="FAILS - empy and template variables sections set", - ), pytest.param( ( (['file:stuffin', 'turkey'], 'gobble'), @@ -351,7 +324,7 @@ def test_deprecation_warnings( @pytest.mark.parametrize( 'tv_string', - (('template variables'), ('jinja2:suite.rc'), ('empy:suite.rc')), + (('template variables'), ('jinja2:suite.rc')), ) def test_retrieve_installed_cli_opts(tmp_path, tv_string): """It merges src, dest and cli. diff --git a/tests/unit/test_config_tree.py b/tests/unit/test_config_tree.py index a622aa5d..3a40e57e 100644 --- a/tests/unit/test_config_tree.py +++ b/tests/unit/test_config_tree.py @@ -30,7 +30,6 @@ load_rose_config, ) from cylc.rose.utilities import ( - MultipleTemplatingEnginesError, get_cli_opts_node, merge_opts, merge_rose_cylc_suite_install_conf, @@ -85,7 +84,7 @@ def rose_config_template(tmp_path, scope='module'): def wrapped_function(section): """Fixture which returns a tmp_path containing a rose config tree. - uses ``wrapped_function`` to allow passing either "empy" or "jinja2" + uses ``wrapped_function`` to allow passing "jinja2" section types. Creates: @@ -127,31 +126,26 @@ def wrapped_function(section): @pytest.mark.parametrize( - 'override, section, exp_ANOTHER_JINJA2_ENV, exp_JINJA2_VAR', + 'override, exp_ANOTHER_JINJA2_ENV, exp_JINJA2_VAR', [ - (None, 'jinja2', 'Defined in config', 64), - (None, 'empy', 'Defined in config', 64), - ('environment', 'jinja2', 'Optional config picked from env var', 42), - ('CLI', 'jinja2', 'Optional config picked from CLI', 99), - ('environment', 'empy', 'Optional config picked from env var', 42), - ('CLI', 'empy', 'Optional config picked from CLI', 99), - ('override', 'jinja2', 'Variable overridden', 99), - ('override', 'empy', 'Variable overridden', 99) + (None, 'Defined in config', 64), + ('environment', 'Optional config picked from env var', 42), + ('CLI', 'Optional config picked from CLI', 99), + ('override', 'Variable overridden', 99), ] ) def test_get_rose_vars( monkeypatch, rose_config_template, override, - section, exp_ANOTHER_JINJA2_ENV, exp_JINJA2_VAR ): - """Test reading of empy or jinja2 vars + """Test reading of jinja2 vars Scenarios tested: - Read in a basic rose-suite.conf file. Ensure we don't return env, - just jinja2/empy. + just jinja2. - Get optional config name from an environment variable. - Get optional config name from command line option. - Get optional config name from an explicit over-ride string. @@ -169,9 +163,9 @@ def test_get_rose_vars( elif override == 'override': options.opt_conf_keys = ["chips"] options.defines = [ - f"[{section}:suite.rc]Another_Jinja2_var='Variable overridden'" + "[jinja2:suite.rc]Another_Jinja2_var='Variable overridden'" ] - suite_path = rose_config_template(section) + suite_path = rose_config_template('jinja2') config_tree = load_rose_config( suite_path, options ) @@ -251,64 +245,38 @@ def test_get_rose_vars_jinja2_ROSE_VARS(tmp_path): ]) -def test_get_rose_vars_fail_if_empy_AND_jinja2(tmp_path): - """It should raise an error if both empy and jinja2 sections defined.""" - (tmp_path / 'rose-suite.conf').write_text( - "[jinja2:suite.rc]\n" - "[empy:suite.rc]\n" - ) - config_tree = load_rose_config(tmp_path) - with pytest.raises(MultipleTemplatingEnginesError): - process_config(config_tree) - - @pytest.mark.parametrize( - 'override, section, exp_ANOTHER_JINJA2_ENV, exp_JINJA2_VAR, opts_format', + 'override, exp_ANOTHER_JINJA2_ENV, exp_JINJA2_VAR, opts_format', [ - (None, 'jinja2', '"Defined in config"', 64, 'list'), - (None, 'empy', '"Defined in config"', 64, 'list'), + (None, '"Defined in config"', 64, 'list'), ( - 'environment', 'jinja2', "'Optional config picked from env var'", + 'environment', "'Optional config picked from env var'", 42, 'list' ), - ('CLI', 'jinja2', "'Optional config picked from CLI'", 99, 'list'), - ( - 'environment', 'empy', - "'Optional config picked from env var'", 42, 'list' - ), - ('CLI', 'empy', "'Optional config picked from CLI'", 99, 'list'), - ('override', 'jinja2', 'Variable overridden', 99, 'list'), - ('override', 'empy', 'Variable overridden', 99, 'list'), - (None, 'jinja2', '"Defined in config"', 64, 'str'), - (None, 'empy', '"Defined in config"', 64, 'str'), + ('CLI', "'Optional config picked from CLI'", 99, 'list'), + ('override', 'Variable overridden', 99, 'list'), + (None, '"Defined in config"', 64, 'str'), ( - 'environment', 'jinja2', "'Optional config picked from env var'", + 'environment', "'Optional config picked from env var'", 42, 'str' ), - ('CLI', 'jinja2', "'Optional config picked from CLI'", 99, 'str'), - ( - 'environment', 'empy', "'Optional config picked from env var'", - 42, 'str' - ), - ('CLI', 'empy', "'Optional config picked from CLI'", 99, 'str'), - ('override', 'jinja2', 'Variable overridden', 99, 'str'), - ('override', 'empy', 'Variable overridden', 99, 'str') + ('CLI', "'Optional config picked from CLI'", 99, 'str'), + ('override', 'Variable overridden', 99, 'str'), ] ) def test_rose_config_tree_loader( monkeypatch, rose_config_template, override, - section, exp_ANOTHER_JINJA2_ENV, exp_JINJA2_VAR, opts_format ): - """Test reading of empy or jinja2 vars + """Test reading of jinja2 vars Scenarios tested: - Read in a basic rose-suite.conf file. Ensure we don't return env, - just jinja2/empy. + just jinja2. - Get optional config name from an environment variable. - Get optional config name from command line option. - Get optional config name from an explicit over-ride string. @@ -330,12 +298,12 @@ def test_rose_config_tree_loader( options = SimpleNamespace() options.opt_conf_keys = conf_keys options.defines = [ - f"[{section}:suite.rc]Another_Jinja2_var=Variable overridden" + "[jinja2:suite.rc]Another_Jinja2_var=Variable overridden" ] result = rose_config_tree_loader( - rose_config_template(section), options - ).node.value[section + ':suite.rc'].value + rose_config_template('jinja2'), options + ).node.value['jinja2:suite.rc'].value results = { 'Another_Jinja2_var': result['Another_Jinja2_var'].value, 'JINJA2_VAR': result['JINJA2_VAR'].value @@ -368,12 +336,6 @@ def test_rose_config_tree_loader( {'rose_template_vars': ['FOO="Bar"']}, id="rose_template_vars-set-by--S-(jinja2 already set in file)" ), - param( - {'empy:suite.rc': {'FOO': '"Bar"'}}, - '[empy:suite.rc]\n', - {'rose_template_vars': ['FOO="Bar"']}, - id="rose_template_vars-set-by--S-(empy already set in file)" - ), param( {'Any Old Section': {'FOO': '"Bar"'}}, '', @@ -397,7 +359,7 @@ def rose_fileinstall_config_template(tmp_path, scope='module'): def wrapped_function(section): """Fixture which returns a tmp_path containing a rose config tree. - uses ``wrapped_function`` to allow passing either "empy" or "jinja2" + uses ``wrapped_function`` to allow passing "jinja2" section types. Creates: diff --git a/tests/unit/test_functional_post_install.py b/tests/unit/test_functional_post_install.py index 4a43a80b..85680184 100644 --- a/tests/unit/test_functional_post_install.py +++ b/tests/unit/test_functional_post_install.py @@ -38,7 +38,6 @@ from cylc.rose.fileinstall import rose_fileinstall from cylc.rose.utilities import ( ROSE_ORIG_HOST_INSTALLED_OVERRIDE_STRING, - MultipleTemplatingEnginesError, ) HOST = get_host() @@ -279,47 +278,6 @@ def test_functional_rose_database_dumped_correctly(tmp_path): assert (rundir / '.rose-config_processors-file.db').is_file() -@pytest.mark.parametrize( - ( - 'opts, files, expect' - ), - [ - pytest.param( - # opts: - SimpleNamespace( - opt_conf_keys='', defines=['[jinja2:suite.rc]FOO=1'], - define_suites=[], clear_rose_install_opts=False - ), - # {file: content} - { - 'test/rose-suite.conf': - f'\n[empy:suite.rc]\nFOO=7\nROSE_ORIG_HOST={HOST}\n' - }, - ( - r"((jinja2:suite\.rc)|(empy:suite.rc)); " - r"((jinja2:suite\.rc)|(empy:suite.rc))" - ), - id='CLI contains different templating' - ), - ] -) -def test_template_section_conflict( - monkeypatch, tmp_path, opts, files, expect -): - """Cylc install fails if multiple template sections set:""" - testdir = tmp_path / 'test' - # Set up existing files, should these exist: - for fname, content in files.items(): - path = tmp_path / fname - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content) - - with pytest.raises(MultipleTemplatingEnginesError) as exc_info: - # Run the entry point top-level function: - record_cylc_install_options(testdir, testdir, opts) - assert exc_info.match(expect) - - def test_rose_fileinstall_exception(tmp_path, monkeypatch): """It throws an exception if you try to install files to a non existent destination. diff --git a/tests/unit/test_rose_stem_units.py b/tests/unit/test_rose_stem_units.py index 5128030f..c32bccec 100644 --- a/tests/unit/test_rose_stem_units.py +++ b/tests/unit/test_rose_stem_units.py @@ -417,12 +417,11 @@ def test_ascertain_project_if_name_supplied( @pytest.mark.parametrize( 'language, expect', ( - ('empy', '[empy:suite.rc]'), ('jinja2', '[jinja2:suite.rc]'), ('template variables', '[template variables]'), ) ) -def test_process_template_engine_set_correctly(monkeypatch, language, expect): +def test_process_template_engine_set(monkeypatch, language, expect): """Defines are correctly assigned a [