Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Do not allow plugins to act on global.cylc #5839

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 32 additions & 26 deletions cylc/flow/parsec/fileparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
value type is known).
"""

from copy import deepcopy
import os
from pathlib import Path
import re
Expand All @@ -51,6 +52,7 @@

if t.TYPE_CHECKING:
from optparse import Values
from typing import Dict, Union


# heading/sections can contain commas (namespace name lists) and any
Expand Down Expand Up @@ -105,6 +107,12 @@
_UNCLOSED_MULTILINE = re.compile(
r'(?<![\w>])\[.*\]'
)
TEMPLATING_DETECTED = 'templating_detected'
EXTRA_VARS_TEMPLATE = {
'env': {},
'template_variables': {},
'templating_detected': None
wxtim marked this conversation as resolved.
Show resolved Hide resolved
}


def get_cylc_env_vars() -> t.Dict[str, str]:
Expand Down Expand Up @@ -242,7 +250,7 @@ def multiline(flines, value, index, maxline):
return quot + newvalue + line, index


def process_plugins(fpath, opts):
def process_plugins(fpath: 'Union[str, Path]', opts: 'Values'):
"""Run a Cylc pre-configuration plugin.

Plugins should return a dictionary containing:
Expand All @@ -263,12 +271,16 @@ def process_plugins(fpath, opts):
'templating_detected': None
}
"""
fpath = Path(fpath)

# Set out blank dictionary for return:
extra_vars = {
'env': {},
'template_variables': {},
'templating_detected': None
}
extra_vars = deepcopy(EXTRA_VARS_TEMPLATE)

# Don't run this on global configs:
if fpath.name == 'global.cylc':
return extra_vars

fpath = fpath.parent
wxtim marked this conversation as resolved.
Show resolved Hide resolved
wxtim marked this conversation as resolved.
Show resolved Hide resolved

# Run entry point pre_configure items, trying to merge values with each.:
for entry_point in iter_entry_points(
Expand Down Expand Up @@ -302,26 +314,20 @@ def process_plugins(fpath, opts):
)
extra_vars[section].update(section_update)

templating_detected = plugin_result.get(TEMPLATING_DETECTED, None)
if (
'templating_detected' in plugin_result and
plugin_result['templating_detected'] is not None and
extra_vars['templating_detected'] is not None and
extra_vars['templating_detected'] !=
plugin_result['templating_detected']
templating_detected is not None
and extra_vars[TEMPLATING_DETECTED] is not None
and extra_vars[TEMPLATING_DETECTED] != templating_detected
):
# Don't allow subsequent plugins with different templating_detected
raise ParsecError(
"Can't merge templating languages "
f"{extra_vars['templating_detected']} and "
f"{plugin_result['templating_detected']}"
"Can't merge templating languages"
wxtim marked this conversation as resolved.
Show resolved Hide resolved
f"{extra_vars[TEMPLATING_DETECTED]} and "
f"{templating_detected}"
)
elif (
'templating_detected' in plugin_result and
plugin_result['templating_detected'] is not None
):
extra_vars['templating_detected'] = plugin_result[
'templating_detected'
]
elif plugin_result.get(TEMPLATING_DETECTED, None) is not None:
wxtim marked this conversation as resolved.
Show resolved Hide resolved
extra_vars[TEMPLATING_DETECTED] = templating_detected
wxtim marked this conversation as resolved.
Show resolved Hide resolved

return extra_vars

Expand Down Expand Up @@ -352,7 +358,7 @@ def merge_template_vars(
>>> merge_template_vars(a, b)
{'FOO': 42, 'BAZ': 3.14159, 'BAR': 'Hello World'}
"""
if plugin_result['templating_detected'] is not None:
if plugin_result[TEMPLATING_DETECTED] is not None:
plugin_tvars = plugin_result['template_variables']
will_be_overwritten = (
native_tvars.keys() &
Expand Down Expand Up @@ -456,7 +462,7 @@ def read_and_proc(
do_jinja2 = True
do_contin = True

extra_vars = process_plugins(Path(fpath).parent, opts)
extra_vars = process_plugins(fpath, opts)
wxtim marked this conversation as resolved.
Show resolved Hide resolved

if not template_vars:
template_vars = {}
Expand All @@ -483,12 +489,12 @@ def read_and_proc(

# Fail if templating_detected ≠ hashbang
process_with = hashbang_and_plugin_templating_clash(
extra_vars['templating_detected'], flines
extra_vars[TEMPLATING_DETECTED], flines
)
# process with EmPy
if do_empy:
if (
extra_vars['templating_detected'] == 'empy' and
extra_vars[TEMPLATING_DETECTED] == 'empy' and
not process_with and
process_with != 'empy'
):
Expand All @@ -508,7 +514,7 @@ def read_and_proc(
# process with Jinja2
if do_jinja2:
if (
extra_vars['templating_detected'] == 'jinja2' and
extra_vars[TEMPLATING_DETECTED] == 'jinja2' and
not process_with and
process_with != 'jinja2'
):
Expand Down
39 changes: 34 additions & 5 deletions tests/unit/parsec/test_fileparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,15 @@
)
from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
from cylc.flow.parsec.fileparse import (
EXTRA_VARS_TEMPLATE,
_prepend_old_templatevars,
_get_fpath_for_source,
get_cylc_env_vars,
addict,
addsect,
multiline,
parse,
process_plugins,
read_and_proc,
merge_template_vars
)
Expand Down Expand Up @@ -741,7 +743,7 @@ def test_get_fpath_for_source(tmp_path):

def test_user_has_no_cwd(tmp_path):
"""Test we can parse a config file even if cwd does not exist."""
cwd = tmp_path/"cwd"
cwd = tmp_path / "cwd"
os.mkdir(cwd)
os.chdir(cwd)
os.rmdir(cwd)
Expand All @@ -765,15 +767,42 @@ def test_get_cylc_env_vars(monkeypatch):
{
"CYLC_VERSION": "betwixt",
"CYLC_ENV_NAME": "between",
"CYLC_QUESTION": "que?",
"CYLC_ANSWER": "42",
"CYLC_QUESTION": "que?",
"CYLC_ANSWER": "42",
"FOO": "foo"
}
)
assert (
get_cylc_env_vars() == {
"CYLC_QUESTION": "que?",
"CYLC_ANSWER": "42",
"CYLC_QUESTION": "que?",
"CYLC_ANSWER": "42",
}
)


class EntryPointWrapper:
"""Wraps a method to make it look like an entry point."""

def __init__(self, fcn):
self.name = fcn.__name__
self.fcn = fcn

def load(self):
return self.fcn


@EntryPointWrapper
def pre_configure_basic(*_, **__):
"""Simple plugin that returns one env var and one template var."""
return {'env': {'foo': 44}, 'template_variables': {}}


def test_plugins_not_called_on_global_config(monkeypatch):
monkeypatch.setattr(
'cylc.flow.parsec.fileparse.iter_entry_points',
lambda x: [pre_configure_basic]
)
result = process_plugins('/pennine/way/flow.cylc', {})
assert result != EXTRA_VARS_TEMPLATE
result = process_plugins('/appalachian/trail/global.cylc', {})
assert result == EXTRA_VARS_TEMPLATE
8 changes: 4 additions & 4 deletions tests/unit/plugins/test_pre_configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def test_pre_configure(monkeypatch):
'cylc.flow.parsec.fileparse.iter_entry_points',
lambda x: [pre_configure_basic]
)
extra_vars = process_plugins(None, None)
extra_vars = process_plugins('/', None)
assert extra_vars == {
'env': {
'ANSWER': '42'
Expand All @@ -90,7 +90,7 @@ def test_pre_configure_duplicate(monkeypatch):
]
)
with pytest.raises(ParsecError):
process_plugins(None, None)
process_plugins('/', None)


def test_pre_configure_templating_detected(monkeypatch):
Expand All @@ -103,7 +103,7 @@ def test_pre_configure_templating_detected(monkeypatch):
]
)
with pytest.raises(ParsecError):
process_plugins(None, None)
process_plugins('/', None)


def test_pre_configure_exception(monkeypatch):
Expand All @@ -113,7 +113,7 @@ def test_pre_configure_exception(monkeypatch):
lambda x: [pre_configure_error]
)
with pytest.raises(PluginError) as exc_ctx:
process_plugins(None, None)
process_plugins('/', None)
# the context of the original error should be preserved in the raised
# exception
assert exc_ctx.value.entry_point == 'cylc.pre_configure'
Expand Down