From 48b3af568066f49b06c769c20fcc59cd0efd35ae Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Sat, 18 Nov 2023 14:51:04 +0100 Subject: [PATCH 01/11] [FIX] jsonifier: Call callable for unknown field If a field name into the parser definition doesn't exist into the model but is resolved by a callable, call the method. Prior to this change, it was o more possible to define computed json value for custome keys --- jsonifier/models/models.py | 13 +++++++++---- jsonifier/tests/test_get_parser.py | 2 ++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/jsonifier/models/models.py b/jsonifier/models/models.py index 44582f2c80d..4e4473d99bc 100644 --- a/jsonifier/models/models.py +++ b/jsonifier/models/models.py @@ -71,14 +71,19 @@ def _add_json_key(self, values, json_key, value): def _jsonify_record(self, parser, rec, root): """JSONify one record (rec). Private function called by jsonify.""" strict = self.env.context.get("jsonify_record_strict", False) - for field in parser: - field_dict, subparser = rec.__parse_field(field) + for field_key in parser: + field_dict, subparser = rec.__parse_field(field_key) + function = field_dict.get("function") try: self._jsonify_record_validate_field(rec, field_dict, strict) except SwallableException: - continue + if not function: + # If we have a function we can use it to get the value + # even if the field is not available. + # If not, well there's nothing we can do. + continue json_key = field_dict.get("target", field_dict["name"]) - if field_dict.get("function"): + if function: try: value = self._jsonify_record_handle_function( rec, field_dict, strict diff --git a/jsonifier/tests/test_get_parser.py b/jsonifier/tests/test_get_parser.py index 364e9ac2105..4c119aea97d 100644 --- a/jsonifier/tests/test_get_parser.py +++ b/jsonifier/tests/test_get_parser.py @@ -252,10 +252,12 @@ def test_json_export_callable_parser(self): # callable subparser ("name", lambda rec, fname: rec[fname] + " rocks!"), ("name:custom", "jsonify_custom"), + ("unknown_field", lambda rec, fname: "yeah again!"), ] expected_json = { "name": "Akretion rocks!", "custom": "yeah!", + "unknown_field": "yeah again!", } json_partner = self.partner.jsonify(parser) self.assertDictEqual(json_partner[0], expected_json) From 36c8217b1d1aa4f9e35120b1c374f5ed77e09799 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 21 Feb 2025 12:27:12 +0100 Subject: [PATCH 02/11] oca-port: blacklist PR(s) 2762 for jsonifier --- .oca/oca-port/blacklist/jsonifier.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .oca/oca-port/blacklist/jsonifier.json diff --git a/.oca/oca-port/blacklist/jsonifier.json b/.oca/oca-port/blacklist/jsonifier.json new file mode 100644 index 00000000000..9a2622bcf1a --- /dev/null +++ b/.oca/oca-port/blacklist/jsonifier.json @@ -0,0 +1,5 @@ +{ + "pull_requests": { + "OCA/server-tools#2762": "already ported" + } +} From 64cfc9a023b19a9592f0e0941597b5eef9ee57f9 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 1 Jun 2022 10:58:48 +0200 Subject: [PATCH 03/11] jsonifier: reduce complexity of _jsonify_record --- jsonifier/models/models.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jsonifier/models/models.py b/jsonifier/models/models.py index 4e4473d99bc..7632867058a 100644 --- a/jsonifier/models/models.py +++ b/jsonifier/models/models.py @@ -169,11 +169,6 @@ def _jsonify_record_handle_subparser(self, rec, field_dict, strict, subparser): def _jsonify_record_handle_resolver(self, rec, field, resolver, json_key): value = rec._jsonify_value(field, rec[field.name]) value = resolver.resolve(field, rec)[0] if resolver else value - if isinstance(value, dict) and "_json_key" in value and "_value" in value: - # Allow override of json_key. - # In this case, - # the final value must be encapsulated into _value key - value, json_key = value["_value"], value["_json_key"] return value, json_key def jsonify(self, parser, one=False, with_fieldname=False): From 645bdcb3ef8fcc1edac4af4953e3ae20e8c3fcb1 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 1 Jun 2022 11:11:37 +0200 Subject: [PATCH 04/11] jsonifier: allow json_key override by resolver Quite handy to take full control of the final result. --- jsonifier/models/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jsonifier/models/models.py b/jsonifier/models/models.py index 7632867058a..4e4473d99bc 100644 --- a/jsonifier/models/models.py +++ b/jsonifier/models/models.py @@ -169,6 +169,11 @@ def _jsonify_record_handle_subparser(self, rec, field_dict, strict, subparser): def _jsonify_record_handle_resolver(self, rec, field, resolver, json_key): value = rec._jsonify_value(field, rec[field.name]) value = resolver.resolve(field, rec)[0] if resolver else value + if isinstance(value, dict) and "_json_key" in value and "_value" in value: + # Allow override of json_key. + # In this case, + # the final value must be encapsulated into _value key + value, json_key = value["_value"], value["_json_key"] return value, json_key def jsonify(self, parser, one=False, with_fieldname=False): From e61f7acd4631b0844daa7ed5260ad7db8930d499 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Tue, 19 Sep 2023 17:57:32 +0200 Subject: [PATCH 05/11] [ADD] jsonifier: with_fieldname option Orig commit msg: [ADD] use jsonifier features to retund postprocessed values, containing field string as well Better Version, not using function features , leaves parser syntax intact. [FIX] Review fixes [FIX] small comment and style fix --- jsonifier/models/models.py | 12 ++++---- jsonifier/readme/USAGE.rst | 28 ++++++++++++++++++ jsonifier/tests/test_get_parser.py | 47 ++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 jsonifier/readme/USAGE.rst diff --git a/jsonifier/models/models.py b/jsonifier/models/models.py index 4e4473d99bc..f6b293b39a8 100644 --- a/jsonifier/models/models.py +++ b/jsonifier/models/models.py @@ -213,11 +213,13 @@ def jsonify(self, parser, one=False, with_fieldname=False): parsers = {False: parser["fields"]} if "fields" in parser else parser["langs"] for lang in parsers: translate = lang or parser.get("language_agnostic") - records = self.with_context(lang=lang) if translate else self - records = ( - records.with_context(with_fieldname=True) if with_fieldname else records - ) - for record, json in zip(records, results, strict=True): + new_ctx = {} + if translate: + new_ctx["lang"] = lang + if with_fieldname: + new_ctx["with_fieldname"] = True + records = self.with_context(**new_ctx) if new_ctx else self + for record, json in zip(records, results, strict=False): self._jsonify_record(parsers[lang], record, json) if resolver: diff --git a/jsonifier/readme/USAGE.rst b/jsonifier/readme/USAGE.rst new file mode 100644 index 00000000000..7b5ae0cc534 --- /dev/null +++ b/jsonifier/readme/USAGE.rst @@ -0,0 +1,28 @@ +with_fieldname parameter +========================== + +The with_fieldname option of jsonify() method, when true, will inject on +the same level of the data "_fieldname_$field" keys that will +contain the field name, in the language of the current user. + + + Examples of with_fieldname usage: + +.. code-block:: python + + # example 1 + parser = [('name')] + a.jsonify(parser=parser) + [{'name': 'SO3996'}] + >>> a.jsonify(parser=parser, with_fieldname=False) + [{'name': 'SO3996'}] + >>> a.jsonify(parser=parser, with_fieldname=True) + [{'fieldname_name': 'Order Reference', 'name': 'SO3996'}}] + + + # example 2 - with a subparser- + parser=['name', 'create_date', ('order_line', ['id' , 'product_uom', 'is_expense'])] + >>> a.jsonify(parser=parser, with_fieldname=False) + [{'name': 'SO3996', 'create_date': '2015-06-02T12:18:26.279909+00:00', 'order_line': [{'id': 16649, 'product_uom': 'stuks', 'is_expense': False}, {'id': 16651, 'product_uom': 'stuks', 'is_expense': False}, {'id': 16650, 'product_uom': 'stuks', 'is_expense': False}]}] + >>> a.jsonify(parser=parser, with_fieldname=True) + [{'fieldname_name': 'Order Reference', 'name': 'SO3996', 'fieldname_create_date': 'Creation Date', 'create_date': '2015-06-02T12:18:26.279909+00:00', 'fieldname_order_line': 'Order Lines', 'order_line': [{'fieldname_id': 'ID', 'id': 16649, 'fieldname_product_uom': 'Unit of Measure', 'product_uom': 'stuks', 'fieldname_is_expense': 'Is expense', 'is_expense': False}]}] diff --git a/jsonifier/tests/test_get_parser.py b/jsonifier/tests/test_get_parser.py index 4c119aea97d..899acdc1bd1 100644 --- a/jsonifier/tests/test_get_parser.py +++ b/jsonifier/tests/test_get_parser.py @@ -214,6 +214,53 @@ def test_json_export(self): "_fieldname_partner_latitude": "Geo Latitude", "create_date": "2019-10-31T14:39:49", } + expected_json_with_fieldname = { + "_fieldname_lang": "Language", + "lang": "en_US", + "_fieldname_comment": "Notes", + "comment": None, + "_fieldname_credit_limit": "Credit Limit", + "credit_limit": 0.0, + "_fieldname_name": "Name", + "name": "Akretion", + "_fieldname_color": "Color Index", + "color": 0, + "_fieldname_children": "Contact", + "children": [ + { + "_fieldname_children": "Contact", + "children": [], + "_fieldname_email": "Email", + "email": None, + "_fieldname_country": "Country", + "country": { + "_fieldname_code": "Country Code", + "code": "FR", + "_fieldname_name": "Country Name", + "name": "France", + }, + "_fieldname_name": "Name", + "name": "Sebatien Beau", + "_fieldname_id": "ID", + "id": self.partner.child_ids.id, + } + ], + "_fieldname_country": "Country", + "country": { + "_fieldname_code": "Country Code", + "code": "FR", + "_fieldname_name": "Country Name", + "name": "France", + }, + "_fieldname_active": "Active", + "active": True, + "_fieldname_category_id": "Tags", + "category_id": [{"_fieldname_name": "Tag Name", "name": "Inovator"}], + "_fieldname_create_date": "Created on", + "create_date": "2019-10-31T15:39:49+01:00", + "_fieldname_date": "Date", + "date": "2019-10-31", + } json_partner = self.partner.jsonify(parser) self.assertDictEqual(json_partner[0], expected_json) json_partner_with_fieldname = self.partner.jsonify( From 62c98f60bff0d9001b0c63ab128403f24ccb9524 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Tue, 20 Aug 2024 08:00:17 +0200 Subject: [PATCH 06/11] jsonifier: fix field validation An invalid field must be skipped in any case if not in strict mode. Error message has been downgraded to warning since is not really broken till you use string mode. Log tests reworked. --- jsonifier/models/models.py | 16 ++++++++-------- jsonifier/tests/test_get_parser.py | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/jsonifier/models/models.py b/jsonifier/models/models.py index f6b293b39a8..72dc109d892 100644 --- a/jsonifier/models/models.py +++ b/jsonifier/models/models.py @@ -121,15 +121,15 @@ def _jsonify_record_validate_field(self, rec, field_dict, strict): if strict: # let it fail rec._fields[field_name] # pylint: disable=pointless-statement - if not tools.config["test_enable"]: - # If running live, log proper error - # so that techies can track it down - _logger.error( - "%(model)s.%(fname)s not available", - {"model": self._name, "fname": field_name}, - ) + else: + if not tools.config["test_enable"]: + # If running live, log proper error + # so that techies can track it down + _logger.warning( + "%(model)s.%(fname)s not available", + {"model": self._name, "fname": field_name}, + ) raise SwallableException() - return True def _jsonify_record_handle_function(self, rec, field_dict, strict): diff --git a/jsonifier/tests/test_get_parser.py b/jsonifier/tests/test_get_parser.py index 899acdc1bd1..7146e607a9d 100644 --- a/jsonifier/tests/test_get_parser.py +++ b/jsonifier/tests/test_get_parser.py @@ -427,24 +427,25 @@ def test_bad_parsers_strict(self): def test_bad_parsers_fail_gracefully(self): rec = self.category - logger_patch_path = "odoo.addons.jsonifier.models.models._logger.error" - - # logging is disabled when testing as it's useless and makes build fail. + # logging is disabled when testing as it makes too much noise tools.config["test_enable"] = False + logger_name = "odoo.addons.jsonifier.models.models" bad_field_name = ["Name"] - with mock.patch(logger_patch_path) as mocked_logger: + with self.assertLogs(logger=logger_name, level="WARNING") as capt: rec.jsonify(bad_field_name, one=True) - mocked_logger.assert_called() + self.assertIn("res.partner.category.Name not availabl", capt.output[0]) bad_function_name = {"fields": [{"name": "name", "function": "notafunction"}]} - with mock.patch(logger_patch_path) as mocked_logger: + with self.assertLogs(logger=logger_name, level="WARNING") as capt: rec.jsonify(bad_function_name, one=True) - mocked_logger.assert_called() + self.assertIn( + "res.partner.category.notafunction not available", capt.output[0] + ) bad_subparser = {"fields": [({"name": "name"}, [{"name": "subparser_name"}])]} - with mock.patch(logger_patch_path) as mocked_logger: + with self.assertLogs(logger=logger_name, level="WARNING") as capt: rec.jsonify(bad_subparser, one=True) - mocked_logger.assert_called() + self.assertIn("res.partner.category.name not relational", capt.output[0]) tools.config["test_enable"] = True From 04c5dd51da38e0fd3ccc0c3d8d6ce3484000d0b4 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Thu, 20 Feb 2025 18:05:37 +0100 Subject: [PATCH 07/11] jsonifier: fix get_json_parser caching The resolver was returned as a full recordset, hence its cursor could diverge from the original one and get closed while the other was still active. --- jsonifier/models/ir_exports.py | 5 +++-- jsonifier/models/models.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/jsonifier/models/ir_exports.py b/jsonifier/models/ir_exports.py index c4cca1b7ec6..6e621d651aa 100644 --- a/jsonifier/models/ir_exports.py +++ b/jsonifier/models/ir_exports.py @@ -109,7 +109,8 @@ def get_json_parser(self): if line.target: names = line.target.split("/") function = line.instance_method_name - options = {"resolver": line.resolver_id, "function": function} + # resolver must be passed as ID to avoid cache issues + options = {"resolver": line.resolver_id.id, "function": function} update_dict(dict_parser, names, options) lang_parsers[lang] = convert_dict(dict_parser) if list(lang_parsers.keys()) == [False]: @@ -117,7 +118,7 @@ def get_json_parser(self): else: parser["langs"] = lang_parsers if self.global_resolver_id: - parser["resolver"] = self.global_resolver_id + parser["resolver"] = self.global_resolver_id.id if self.language_agnostic: parser["language_agnostic"] = self.language_agnostic return parser diff --git a/jsonifier/models/models.py b/jsonifier/models/models.py index 72dc109d892..55ae11cdf7e 100644 --- a/jsonifier/models/models.py +++ b/jsonifier/models/models.py @@ -102,6 +102,9 @@ def _jsonify_record(self, parser, rec, root): value = rec._jsonify_value(field, rec[field.name]) resolver = field_dict.get("resolver") if resolver: + if isinstance(resolver, int): + # cached versions of the parser are stored as integer + resolver = self.env["ir.exports.resolver"].browse(resolver) value, json_key = self._jsonify_record_handle_resolver( rec, field, resolver, json_key ) @@ -208,7 +211,9 @@ def jsonify(self, parser, one=False, with_fieldname=False): if isinstance(parser, list): parser = convert_simple_to_full_parser(parser) resolver = parser.get("resolver") - + if isinstance(resolver, int): + # cached versions of the parser are stored as integer + resolver = self.env["ir.exports.resolver"].browse(resolver) results = [{} for record in self] parsers = {False: parser["fields"]} if "fields" in parser else parser["langs"] for lang in parsers: From 969ece530f61c2559f3113d519fb033a436341ec Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 21 Feb 2025 11:34:48 +0100 Subject: [PATCH 08/11] jsonifier: adapt tests --- jsonifier/tests/test_get_parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jsonifier/tests/test_get_parser.py b/jsonifier/tests/test_get_parser.py index 7146e607a9d..daf3018278e 100644 --- a/jsonifier/tests/test_get_parser.py +++ b/jsonifier/tests/test_get_parser.py @@ -219,8 +219,7 @@ def test_json_export(self): "lang": "en_US", "_fieldname_comment": "Notes", "comment": None, - "_fieldname_credit_limit": "Credit Limit", - "credit_limit": 0.0, + "_fieldname_partner_latitude": "Geo Latitude", "_fieldname_name": "Name", "name": "Akretion", "_fieldname_color": "Color Index", @@ -257,9 +256,10 @@ def test_json_export(self): "_fieldname_category_id": "Tags", "category_id": [{"_fieldname_name": "Tag Name", "name": "Inovator"}], "_fieldname_create_date": "Created on", - "create_date": "2019-10-31T15:39:49+01:00", + "create_date": "2019-10-31T14:39:49", "_fieldname_date": "Date", "date": "2019-10-31", + "partner_latitude": 0.0, } json_partner = self.partner.jsonify(parser) self.assertDictEqual(json_partner[0], expected_json) From c9e04890f6756d116c81a3f87a7437e30043e725 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 21 Feb 2025 10:52:00 +0100 Subject: [PATCH 09/11] jsonifier: make pre-commit happy --- jsonifier/tests/test_get_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jsonifier/tests/test_get_parser.py b/jsonifier/tests/test_get_parser.py index daf3018278e..56d35021d6a 100644 --- a/jsonifier/tests/test_get_parser.py +++ b/jsonifier/tests/test_get_parser.py @@ -3,7 +3,6 @@ # Simone Orsi # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). -from unittest import mock from odoo import tools from odoo.exceptions import UserError From 6611b50e60f7ebfaa14535fcf07c6532697b5160 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 21 Feb 2025 14:01:37 +0100 Subject: [PATCH 10/11] jsonifier: adapt tests --- jsonifier/tests/test_get_parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jsonifier/tests/test_get_parser.py b/jsonifier/tests/test_get_parser.py index 56d35021d6a..0f587ec5a48 100644 --- a/jsonifier/tests/test_get_parser.py +++ b/jsonifier/tests/test_get_parser.py @@ -253,11 +253,9 @@ def test_json_export(self): "_fieldname_active": "Active", "active": True, "_fieldname_category_id": "Tags", - "category_id": [{"_fieldname_name": "Tag Name", "name": "Inovator"}], + "category_id": [{"_fieldname_name": "Name", "name": "Inovator"}], "_fieldname_create_date": "Created on", "create_date": "2019-10-31T14:39:49", - "_fieldname_date": "Date", - "date": "2019-10-31", "partner_latitude": 0.0, } json_partner = self.partner.jsonify(parser) From 05abf85a8887e14765aa7251850dfdfa7c7453a3 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Mon, 24 Feb 2025 17:51:57 +0100 Subject: [PATCH 11/11] jsonifier: drop rst --- jsonifier/readme/USAGE.rst | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 jsonifier/readme/USAGE.rst diff --git a/jsonifier/readme/USAGE.rst b/jsonifier/readme/USAGE.rst deleted file mode 100644 index 7b5ae0cc534..00000000000 --- a/jsonifier/readme/USAGE.rst +++ /dev/null @@ -1,28 +0,0 @@ -with_fieldname parameter -========================== - -The with_fieldname option of jsonify() method, when true, will inject on -the same level of the data "_fieldname_$field" keys that will -contain the field name, in the language of the current user. - - - Examples of with_fieldname usage: - -.. code-block:: python - - # example 1 - parser = [('name')] - a.jsonify(parser=parser) - [{'name': 'SO3996'}] - >>> a.jsonify(parser=parser, with_fieldname=False) - [{'name': 'SO3996'}] - >>> a.jsonify(parser=parser, with_fieldname=True) - [{'fieldname_name': 'Order Reference', 'name': 'SO3996'}}] - - - # example 2 - with a subparser- - parser=['name', 'create_date', ('order_line', ['id' , 'product_uom', 'is_expense'])] - >>> a.jsonify(parser=parser, with_fieldname=False) - [{'name': 'SO3996', 'create_date': '2015-06-02T12:18:26.279909+00:00', 'order_line': [{'id': 16649, 'product_uom': 'stuks', 'is_expense': False}, {'id': 16651, 'product_uom': 'stuks', 'is_expense': False}, {'id': 16650, 'product_uom': 'stuks', 'is_expense': False}]}] - >>> a.jsonify(parser=parser, with_fieldname=True) - [{'fieldname_name': 'Order Reference', 'name': 'SO3996', 'fieldname_create_date': 'Creation Date', 'create_date': '2015-06-02T12:18:26.279909+00:00', 'fieldname_order_line': 'Order Lines', 'order_line': [{'fieldname_id': 'ID', 'id': 16649, 'fieldname_product_uom': 'Unit of Measure', 'product_uom': 'stuks', 'fieldname_is_expense': 'Is expense', 'is_expense': False}]}]