diff --git a/tests/test_common.py b/tests/test_common.py index 9d5a08d05..f5654ebb5 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,3 +1,5 @@ +from datetime import datetime, timedelta + import pytest from aiohttp import web @@ -142,9 +144,28 @@ def test_apply_parameter_to_query_do_nothing(): {'code': 'Paris_France', 'domain': 'Test'}, ), ({'code': '{{city}}_{{country}}', 'domain': 'Test'}, None, {'domain': 'Test'}), + ( + {'column': 'date', 'operator': 'eq', 'value': '{{ t + delta }}'}, + {'t': datetime(2020, 12, 31), 'delta': timedelta(days=1)}, + {'column': 'date', 'operator': 'eq', 'value': datetime(2021, 1, 1)}, + ), + ( + {'column': 'date', 'operator': 'eq', 'value': '{{ t.strftime("%d/%m/%Y") }}'}, + {'t': datetime(2020, 12, 31)}, + {'column': 'date', 'operator': 'eq', 'value': '31/12/2020'}, + ), + ( + {'column': 'date', 'operator': 'in', 'value': '{{ allowed_dates }}'}, + {'allowed_dates': [datetime(2020, 12, 31), datetime(2021, 1, 1)]}, + { + 'column': 'date', + 'operator': 'in', + 'value': [datetime(2020, 12, 31), datetime(2021, 1, 1)], + }, + ), ], ) -def test_apply_parameter_to_query(query, params, expected): +def test_nosql_apply_parameters_to_query(query, params, expected): assert nosql_apply_parameters_to_query(query, params) == expected diff --git a/toucan_connectors/common.py b/toucan_connectors/common.py index 0da19a4ce..9d22a2a47 100644 --- a/toucan_connectors/common.py +++ b/toucan_connectors/common.py @@ -8,6 +8,7 @@ import pyjq from aiohttp import ClientSession from jinja2 import Environment, StrictUndefined, Template, meta +from jinja2.nativetypes import NativeEnvironment from pydantic import Field from toucan_data_sdk.utils.helpers import slugify @@ -16,7 +17,6 @@ RE_PARAM = r'%\(([^(%\()]*)\)s' RE_JINJA = r'{{([^({{)}]*)}}' -RE_PARAM_ALONE = r'^' + RE_PARAM + '$' RE_JINJA_ALONE = r'^' + RE_JINJA + '$' # Identify jinja params with no quotes around or complex condition @@ -71,26 +71,25 @@ def _render_query(query, parameters): return {key: _render_query(value, parameters) for key, value in deepcopy(query).items()} elif isinstance(query, list): return [_render_query(elt, parameters) for elt in deepcopy(query)] - elif type(query) is str: + elif isinstance(query, str): if not _has_parameters(query): return query - clean_p = deepcopy(parameters) + + # Replace param templating with jinja templating: + query = re.sub(RE_PARAM, r'{{ \g<1> }}', query) + # Add quotes to string parameters to keep type if not complex - if re.match(RE_PARAM_ALONE, query) or re.match(RE_JINJA_ALONE, query): + clean_p = deepcopy(parameters) + if re.match(RE_JINJA_ALONE, query): clean_p = _prepare_parameters(clean_p) - # Render jinja then render parameters `%()s` - res = Template(query).render(clean_p) % clean_p - - # Remove extra quotes with literal_eval - try: - res = ast.literal_eval(res) - if isinstance(res, str): - return res - else: - return _prepare_result(res) - except (SyntaxError, ValueError): - return res + env = NativeEnvironment() + res = env.from_string(query).render(clean_p) + # NativeEnvironment's render() isn't recursive, so we need to + # apply recursively the literal_eval by hand for lists and dicts: + if isinstance(res, (list, dict)): + return _prepare_result(res) + return res else: return query