From 1cafa02ddabde64256b05dc0a86326bd571a08e4 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 6 Jan 2025 15:39:08 -0500 Subject: [PATCH 1/6] Add support for auto-generated key serialization --- .../core/src/clp/ffi/ir_stream/Serializer.cpp | 592 ++++++++++++------ .../core/src/clp/ffi/ir_stream/Serializer.hpp | 43 +- .../core/tests/test-ir_encoding_methods.cpp | 24 +- 3 files changed, 441 insertions(+), 218 deletions(-) diff --git a/components/core/src/clp/ffi/ir_stream/Serializer.cpp b/components/core/src/clp/ffi/ir_stream/Serializer.cpp index 295ab356c..53b2ecf62 100644 --- a/components/core/src/clp/ffi/ir_stream/Serializer.cpp +++ b/components/core/src/clp/ffi/ir_stream/Serializer.cpp @@ -1,5 +1,6 @@ #include "Serializer.hpp" +#include #include #include #include @@ -35,6 +36,56 @@ using clp::ir::four_byte_encoded_variable_t; namespace clp::ffi::ir_stream { namespace { +/** + * Concept that defines the method to serialize a schema tree node identified by the given locator. + * @param serialization_method + * @param locator + * @return Whether serialization succeeded. + */ +template +concept SchemaTreeNodeSerializationMethodReq = requires( + SerializationMethod serialization_method, + SchemaTree::NodeLocator const& locator +) { + { + serialization_method(locator) + } -> std::same_as; +}; + +/** + * Concept that defines the method to serialize a node-ID-value pair. + * @param serialization_method + * @param node_id + * @param val + * @param schema_tree_node_type The type of the schema tree node that corresponds to `val`. + * @return Whether serialization succeeded. + */ +template +concept NodeIdValuePairSerializationMethodReq = requires( + SerializationMethod serialization_method, + SchemaTree::Node::id_t id, + msgpack::object const& val, + SchemaTree::Node::Type schema_tree_node_type +) { + { + serialization_method(id, val, schema_tree_node_type) + } -> std::same_as; +}; + +/** + * Concept that defines the method to serialize a node-ID-value pair whose value is an empty map. + * @param serialization_method + * @param node_id + * @return Whether serialization succeeded. + */ +template +concept EmptyMapSerializationMethodReq + = requires(SerializationMethod serialization_method, SchemaTree::Node::id_t node_id) { + { + serialization_method(node_id) + } -> std::same_as; + }; + /** * Class for iterating the kv-pairs of a MessagePack map. */ @@ -146,6 +197,23 @@ template vector& output_buf ) -> bool; +/** + * Serializes the given MessagePack value into `output_buf`. + * @tparam encoded_variable_t + * @param val + * @param schema_tree_node_type + * @param logtype_buf + * @param output_buf + * @return Whether serialization succeeded. + */ +template +[[nodiscard]] auto serialize_value( + msgpack::object const& val, + SchemaTree::Node::Type schema_tree_node_type, + string& logtype_buf, + vector& output_buf +) -> bool; + /** * Checks whether the given msgpack array can be serialized into the key-value pair IR format. * @param array @@ -156,6 +224,30 @@ template */ [[nodiscard]] auto is_msgpack_array_serializable(msgpack::object const& array) -> bool; +/** + * Serializes the given msgpack map using a depth-first search (DFS). + * @tparam SchemaTreeNodeSerializationMethod + * @tparam NodeIdValuePairSerializationMethod + * @tparam EmptyMapSerializationMethod + * @param msgpack_map + * @param schema_tree + * @param schema_tree_node_serialization_method + * @param node_id_value_pair_serialization_method + * @param empty_map_serialization_method + * @return Whether serialization succeeded. + */ +template < + SchemaTreeNodeSerializationMethodReq SchemaTreeNodeSerializationMethod, + NodeIdValuePairSerializationMethodReq NodeIdValuePairSerializationMethod, + EmptyMapSerializationMethodReq EmptyMapSerializationMethod> +[[nodiscard]] auto serialize_msgpack_map_using_dfs( + msgpack::object_map const& msgpack_map, + SchemaTree& schema_tree, + SchemaTreeNodeSerializationMethod schema_tree_node_serialization_method, + NodeIdValuePairSerializationMethod node_id_value_pair_serialization_method, + EmptyMapSerializationMethod empty_map_serialization_method +) -> bool; + auto get_schema_tree_node_type_from_msgpack_val(msgpack::object const& val ) -> optional { optional ret_val; @@ -245,6 +337,63 @@ auto serialize_value_array( return serialize_clp_string(oss.str(), logtype_buf, output_buf); } +template +auto serialize_value( + msgpack::object const& val, + SchemaTree::Node::Type schema_tree_node_type, + string& logtype_buf, + vector& output_buf +) -> bool { + switch (schema_tree_node_type) { + case SchemaTree::Node::Type::Int: + if (msgpack::type::POSITIVE_INTEGER == val.type + && static_cast(INT64_MAX) < val.as()) + { + return false; + } + serialize_value_int(val.as(), output_buf); + break; + + case SchemaTree::Node::Type::Float: + serialize_value_float(val.as(), output_buf); + break; + + case SchemaTree::Node::Type::Bool: + serialize_value_bool(val.as(), output_buf); + break; + + case SchemaTree::Node::Type::Str: + if (false + == serialize_value_string( + val.as(), + logtype_buf, + output_buf + )) + { + return false; + } + break; + + case SchemaTree::Node::Type::Obj: + if (msgpack::type::NIL != val.type) { + return false; + } + serialize_value_null(output_buf); + break; + + case SchemaTree::Node::Type::UnstructuredArray: + if (false == serialize_value_array(val, logtype_buf, output_buf)) { + return false; + } + break; + + default: + // Unknown schema tree node type + return false; + } + return true; +} + auto is_msgpack_array_serializable(msgpack::object const& array) -> bool { vector validation_stack{&array}; while (false == validation_stack.empty()) { @@ -286,6 +435,94 @@ auto is_msgpack_array_serializable(msgpack::object const& array) -> bool { return true; } + +template < + SchemaTreeNodeSerializationMethodReq SchemaTreeNodeSerializationMethod, + NodeIdValuePairSerializationMethodReq NodeIdValuePairSerializationMethod, + EmptyMapSerializationMethodReq EmptyMapSerializationMethod> +[[nodiscard]] auto serialize_msgpack_map_using_dfs( + msgpack::object_map const& msgpack_map, + SchemaTree& schema_tree, + SchemaTreeNodeSerializationMethod schema_tree_node_serialization_method, + NodeIdValuePairSerializationMethod node_id_value_pair_serialization_method, + EmptyMapSerializationMethod empty_map_serialization_method +) -> bool { + vector dfs_stack; + dfs_stack.emplace_back( + SchemaTree::cRootId, + span{msgpack_map.ptr, msgpack_map.size} + ); + while (false == dfs_stack.empty()) { + auto& curr{dfs_stack.back()}; + if (false == curr.has_next_child()) { + // Visited all children, so pop node + dfs_stack.pop_back(); + continue; + } + + // Convert the current value's type to its corresponding schema-tree node type + auto const& [key, val]{curr.get_next_child()}; + if (msgpack::type::STR != key.type) { + // A map containing non-string keys is not serializable + return false; + } + + auto const opt_schema_tree_node_type{get_schema_tree_node_type_from_msgpack_val(val)}; + if (false == opt_schema_tree_node_type.has_value()) { + return false; + } + auto const schema_tree_node_type{opt_schema_tree_node_type.value()}; + + SchemaTree::NodeLocator const locator{ + curr.get_schema_tree_node_id(), + key.as(), + schema_tree_node_type + }; + + // Get the schema-tree node that corresponds with the current kv-pair, or add it if it + // doesn't exist. + auto opt_schema_tree_node_id{schema_tree.try_get_node_id(locator)}; + if (false == opt_schema_tree_node_id.has_value()) { + opt_schema_tree_node_id.emplace(schema_tree.insert_node(locator)); + if (false == schema_tree_node_serialization_method(locator)) { + return false; + } + } + auto const schema_tree_node_id{opt_schema_tree_node_id.value()}; + + if (msgpack::type::MAP == val.type) { + // Serialize map + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) + auto const& inner_map{val.via.map}; + auto const inner_map_size(inner_map.size); + if (0 != inner_map_size) { + // Add map for DFS iteration + dfs_stack.emplace_back( + schema_tree_node_id, + span{inner_map.ptr, inner_map_size} + ); + } else { + if (false == empty_map_serialization_method(schema_tree_node_id)) { + return false; + } + } + continue; + } + + // Serialize primitive + if (false + == node_id_value_pair_serialization_method( + schema_tree_node_id, + val, + schema_tree_node_type + )) + { + return false; + } + } + + return true; +} } // namespace template @@ -331,32 +568,180 @@ auto Serializer::change_utc_offset(UtcOffset utc_offset) -> } template -auto Serializer::serialize_msgpack_map(msgpack::object_map const& msgpack_map +auto Serializer::serialize_msgpack_map( + msgpack::object_map const& auto_gen_kv_pairs_map, + msgpack::object_map const& user_gen_kv_pairs_map ) -> bool { - if (0 == msgpack_map.size) { - serialize_value_empty_object(m_ir_buf); - return true; - } + m_auto_gen_keys_schema_tree.take_snapshot(); + m_user_gen_keys_schema_tree.take_snapshot(); + TransactionManager revert_manager{ + []() noexcept -> void {}, + [&]() noexcept -> void { + m_user_gen_keys_schema_tree.revert(); + m_auto_gen_keys_schema_tree.revert(); + } + }; m_schema_tree_node_buf.clear(); - m_key_group_buf.clear(); - m_value_group_buf.clear(); + m_sequential_serialization_buf.clear(); + m_user_gen_val_group_buf.clear(); + + // Serialize auto-generated kv pairs + auto auto_gen_schema_tree_node_serialization_method + = [this](SchemaTree::NodeLocator const& locator) -> bool { + return this->serialize_schema_tree_node(locator); + }; + + auto auto_gen_node_id_value_pairs_serialization_method + = [&](SchemaTree::Node::id_t node_id, + msgpack::object const& val, + SchemaTree::Node::Type schema_tree_node_type) -> bool { + if (false + == encode_and_serialize_schema_tree_node_id< + true, + cProtocol::Payload::EncodedSchemaTreeNodeIdByte, + cProtocol::Payload::EncodedSchemaTreeNodeIdShort, + cProtocol::Payload::EncodedSchemaTreeNodeIdInt>( + node_id, + m_sequential_serialization_buf + )) + { + return false; + } + if (false + == serialize_value( + val, + schema_tree_node_type, + m_logtype_buf, + m_sequential_serialization_buf + )) + { + return false; + } + return true; + }; + + auto auto_gen_empty_map_serialization_method = [&](SchemaTree::Node::id_t node_id) -> bool { + if (false + == encode_and_serialize_schema_tree_node_id< + true, + cProtocol::Payload::EncodedSchemaTreeNodeIdByte, + cProtocol::Payload::EncodedSchemaTreeNodeIdShort, + cProtocol::Payload::EncodedSchemaTreeNodeIdInt>( + node_id, + m_sequential_serialization_buf + )) + { + return false; + } + serialize_value_empty_object(m_sequential_serialization_buf); + return true; + }; - if (false == serialize_msgpack_map_using_dfs(msgpack_map)) { + if (0 != auto_gen_kv_pairs_map.size + && false + == serialize_msgpack_map_using_dfs( + auto_gen_kv_pairs_map, + m_auto_gen_keys_schema_tree, + auto_gen_schema_tree_node_serialization_method, + auto_gen_node_id_value_pairs_serialization_method, + auto_gen_empty_map_serialization_method + )) + { return false; } + // Serialize user-generated kv pairs + auto user_gen_schema_tree_node_serialization_method + = [this](SchemaTree::NodeLocator const& locator) -> bool { + return this->serialize_schema_tree_node(locator); + }; + + auto user_gen_node_id_value_pairs_serialization_method + = [&](SchemaTree::Node::id_t node_id, + msgpack::object const& val, + SchemaTree::Node::Type schema_tree_node_type) -> bool { + if (false + == encode_and_serialize_schema_tree_node_id< + false, + cProtocol::Payload::EncodedSchemaTreeNodeIdByte, + cProtocol::Payload::EncodedSchemaTreeNodeIdShort, + cProtocol::Payload::EncodedSchemaTreeNodeIdInt>( + node_id, + m_sequential_serialization_buf + )) + { + return false; + } + if (false + == serialize_value( + val, + schema_tree_node_type, + m_logtype_buf, + m_user_gen_val_group_buf + )) + { + return false; + } + return true; + }; + + auto user_gen_empty_map_serialization_method = [&](SchemaTree::Node::id_t node_id) -> bool { + if (false + == encode_and_serialize_schema_tree_node_id< + false, + cProtocol::Payload::EncodedSchemaTreeNodeIdByte, + cProtocol::Payload::EncodedSchemaTreeNodeIdShort, + cProtocol::Payload::EncodedSchemaTreeNodeIdInt>( + node_id, + m_sequential_serialization_buf + )) + { + return false; + } + serialize_value_empty_object(m_user_gen_val_group_buf); + return true; + }; + + if (0 == user_gen_kv_pairs_map.size) { + serialize_value_empty_object(m_sequential_serialization_buf); + } else { + if (false + == serialize_msgpack_map_using_dfs( + user_gen_kv_pairs_map, + m_user_gen_keys_schema_tree, + user_gen_schema_tree_node_serialization_method, + user_gen_node_id_value_pairs_serialization_method, + user_gen_empty_map_serialization_method + )) + { + return false; + } + } + + // Copy serialized results into `m_ir_buf` m_ir_buf.insert( m_ir_buf.cend(), m_schema_tree_node_buf.cbegin(), m_schema_tree_node_buf.cend() ); - m_ir_buf.insert(m_ir_buf.cend(), m_key_group_buf.cbegin(), m_key_group_buf.cend()); - m_ir_buf.insert(m_ir_buf.cend(), m_value_group_buf.cbegin(), m_value_group_buf.cend()); + m_ir_buf.insert( + m_ir_buf.cend(), + m_sequential_serialization_buf.cbegin(), + m_sequential_serialization_buf.cend() + ); + m_ir_buf.insert( + m_ir_buf.cend(), + m_user_gen_val_group_buf.cbegin(), + m_user_gen_val_group_buf.cend() + ); + + revert_manager.mark_success(); return true; } template +template auto Serializer::serialize_schema_tree_node( SchemaTree::NodeLocator const& locator ) -> bool { @@ -386,7 +771,7 @@ auto Serializer::serialize_schema_tree_node( if (false == encode_and_serialize_schema_tree_node_id< - false, + is_auto_generated_node, cProtocol::Payload::EncodedSchemaTreeNodeParentIdByte, cProtocol::Payload::EncodedSchemaTreeNodeParentIdShort, cProtocol::Payload::EncodedSchemaTreeNodeParentIdInt>( @@ -400,158 +785,6 @@ auto Serializer::serialize_schema_tree_node( return serialize_string(locator.get_key_name(), m_schema_tree_node_buf); } -template -auto Serializer::serialize_key(SchemaTree::Node::id_t id) -> bool { - return encode_and_serialize_schema_tree_node_id< - false, - cProtocol::Payload::EncodedSchemaTreeNodeIdByte, - cProtocol::Payload::EncodedSchemaTreeNodeIdShort, - cProtocol::Payload::EncodedSchemaTreeNodeIdInt>(id, m_key_group_buf); -} - -template -auto Serializer::serialize_val( - msgpack::object const& val, - SchemaTree::Node::Type schema_tree_node_type -) -> bool { - switch (schema_tree_node_type) { - case SchemaTree::Node::Type::Int: - if (msgpack::type::POSITIVE_INTEGER == val.type - && static_cast(INT64_MAX) < val.as()) - { - return false; - } - serialize_value_int(val.as(), m_value_group_buf); - break; - - case SchemaTree::Node::Type::Float: - serialize_value_float(val.as(), m_value_group_buf); - break; - - case SchemaTree::Node::Type::Bool: - serialize_value_bool(val.as(), m_value_group_buf); - break; - - case SchemaTree::Node::Type::Str: - if (false - == serialize_value_string( - val.as(), - m_logtype_buf, - m_value_group_buf - )) - { - return false; - } - break; - - case SchemaTree::Node::Type::Obj: - if (msgpack::type::NIL != val.type) { - return false; - } - serialize_value_null(m_value_group_buf); - break; - - case SchemaTree::Node::Type::UnstructuredArray: - if (false - == serialize_value_array(val, m_logtype_buf, m_value_group_buf)) - { - return false; - } - break; - - default: - // Unknown schema tree node type - return false; - } - return true; -} - -template -auto Serializer::serialize_msgpack_map_using_dfs( - msgpack::object_map const& msgpack_map -) -> bool { - m_schema_tree.take_snapshot(); - TransactionManager revert_manager{ - []() noexcept -> void {}, - [&]() noexcept -> void { m_schema_tree.revert(); } - }; - - vector dfs_stack; - dfs_stack.emplace_back( - SchemaTree::cRootId, - span{msgpack_map.ptr, msgpack_map.size} - ); - while (false == dfs_stack.empty()) { - auto& curr{dfs_stack.back()}; - if (false == curr.has_next_child()) { - // Visited all children, so pop node - dfs_stack.pop_back(); - continue; - } - - // Convert the current value's type to its corresponding schema-tree node type - auto const& [key, val]{curr.get_next_child()}; - if (msgpack::type::STR != key.type) { - // A map containing non-string keys is not serializable - return false; - } - - auto const opt_schema_tree_node_type{get_schema_tree_node_type_from_msgpack_val(val)}; - if (false == opt_schema_tree_node_type.has_value()) { - return false; - } - auto const schema_tree_node_type{opt_schema_tree_node_type.value()}; - - SchemaTree::NodeLocator const locator{ - curr.get_schema_tree_node_id(), - key.as(), - schema_tree_node_type - }; - - // Get the schema-tree node that corresponds with the current kv-pair, or add it if it - // doesn't exist. - auto opt_schema_tree_node_id{m_schema_tree.try_get_node_id(locator)}; - if (false == opt_schema_tree_node_id.has_value()) { - opt_schema_tree_node_id.emplace(m_schema_tree.insert_node(locator)); - if (false == serialize_schema_tree_node(locator)) { - return false; - } - } - auto const schema_tree_node_id{opt_schema_tree_node_id.value()}; - - if (msgpack::type::MAP == val.type) { - // Serialize map - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) - auto const& inner_map{val.via.map}; - auto const inner_map_size(inner_map.size); - if (0 != inner_map_size) { - // Add map for DFS iteration - dfs_stack.emplace_back( - schema_tree_node_id, - span{inner_map.ptr, inner_map_size} - ); - } else { - // Value is an empty map, so we can serialize it immediately - if (false == serialize_key(schema_tree_node_id)) { - return false; - } - serialize_value_empty_object(m_value_group_buf); - } - continue; - } - - // Serialize primitive - if (false - == (serialize_key(schema_tree_node_id) && serialize_val(val, schema_tree_node_type))) - { - return false; - } - } - - revert_manager.mark_success(); - return true; -} - // Explicitly declare template specializations so that we can define the template methods in this // file template auto Serializer::create( @@ -565,37 +798,24 @@ template auto Serializer::change_utc_offset(UtcOff ) -> void; template auto Serializer::serialize_msgpack_map( - msgpack::object_map const& msgpack_map + msgpack::object_map const& auto_gen_kv_pairs_map, + msgpack::object_map const& user_gen_kv_pairs_map ) -> bool; template auto Serializer::serialize_msgpack_map( - msgpack::object_map const& msgpack_map + msgpack::object_map const& auto_gen_kv_pairs_map, + msgpack::object_map const& user_gen_kv_pairs_map ) -> bool; -template auto Serializer::serialize_schema_tree_node( +template auto Serializer::serialize_schema_tree_node( SchemaTree::NodeLocator const& locator ) -> bool; -template auto Serializer::serialize_schema_tree_node( +template auto Serializer::serialize_schema_tree_node( SchemaTree::NodeLocator const& locator ) -> bool; - -template auto Serializer::serialize_key(SchemaTree::Node::id_t id -) -> bool; -template auto Serializer::serialize_key(SchemaTree::Node::id_t id -) -> bool; - -template auto Serializer::serialize_val( - msgpack::object const& val, - SchemaTree::Node::Type schema_tree_node_type -) -> bool; -template auto Serializer::serialize_val( - msgpack::object const& val, - SchemaTree::Node::Type schema_tree_node_type -) -> bool; - -template auto Serializer::serialize_msgpack_map_using_dfs( - msgpack::object_map const& msgpack_map +template auto Serializer::serialize_schema_tree_node( + SchemaTree::NodeLocator const& locator ) -> bool; -template auto Serializer::serialize_msgpack_map_using_dfs( - msgpack::object_map const& msgpack_map +template auto Serializer::serialize_schema_tree_node( + SchemaTree::NodeLocator const& locator ) -> bool; } // namespace clp::ffi::ir_stream diff --git a/components/core/src/clp/ffi/ir_stream/Serializer.hpp b/components/core/src/clp/ffi/ir_stream/Serializer.hpp index edc6bf0cb..906f66ff8 100644 --- a/components/core/src/clp/ffi/ir_stream/Serializer.hpp +++ b/components/core/src/clp/ffi/ir_stream/Serializer.hpp @@ -82,11 +82,15 @@ class Serializer { auto change_utc_offset(UtcOffset utc_offset) -> void; /** - * Serializes the given msgpack map as a key-value pair log event. - * @param msgpack_map + * Serializes the given msgpack maps as a key-value pair log event. + * @param auto_gen_kv_pairs_map + * @param user_gen_kv_pairs_map * @return Whether serialization succeeded. */ - [[nodiscard]] auto serialize_msgpack_map(msgpack::object_map const& msgpack_map) -> bool; + [[nodiscard]] auto serialize_msgpack_map( + msgpack::object_map const& auto_gen_kv_pairs_map, + msgpack::object_map const& user_gen_kv_pairs_map + ) -> bool; private: // Constructors @@ -95,43 +99,22 @@ class Serializer { // Methods /** * Serializes a schema tree node identified by the given locator into `m_schema_tree_node_buf`. + * @tparam is_auto_generated_node * @param locator * @return Whether serialization succeeded. */ + template [[nodiscard]] auto serialize_schema_tree_node(SchemaTree::NodeLocator const& locator) -> bool; - /** - * Serializes the given key ID into `m_key_group_buf`. - * @param id - * @return Forwards `encode_and_serialize_schema_tree_node_id`'s return values. - */ - [[nodiscard]] auto serialize_key(SchemaTree::Node::id_t id) -> bool; - - /** - * Serializes the given MessagePack value into `m_value_group_buf`. - * @param val - * @param schema_tree_node_type The type of the schema tree node that corresponds to `val`. - * @return Whether serialization succeeded. - */ - [[nodiscard]] auto - serialize_val(msgpack::object const& val, SchemaTree::Node::Type schema_tree_node_type) -> bool; - - /** - * Serializes the given msgpack map using a depth-first search (DFS). - * @param msgpack_map - * @return Whether serialization succeeded. - */ - [[nodiscard]] auto serialize_msgpack_map_using_dfs(msgpack::object_map const& msgpack_map - ) -> bool; - UtcOffset m_curr_utc_offset{0}; Buffer m_ir_buf; - SchemaTree m_schema_tree; + SchemaTree m_auto_gen_keys_schema_tree; + SchemaTree m_user_gen_keys_schema_tree; std::string m_logtype_buf; Buffer m_schema_tree_node_buf; - Buffer m_key_group_buf; - Buffer m_value_group_buf; + Buffer m_sequential_serialization_buf; + Buffer m_user_gen_val_group_buf; }; } // namespace clp::ffi::ir_stream diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index 578ae49f7..390f15f0c 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -349,7 +349,15 @@ auto unpack_and_serialize_msgpack_bytes( if (msgpack::type::MAP != msgpack_obj.type) { return false; } - return serializer.serialize_msgpack_map(msgpack_obj.via.map); + + auto const msgpack_empty_map_buf{nlohmann::json::to_msgpack(nlohmann::json::parse("{}"))}; + auto const msgpack_empty_map_obj_handle{msgpack::unpack( + size_checked_pointer_cast(msgpack_empty_map_buf.data()), + msgpack_empty_map_buf.size() + )}; + auto const msgpack_empty_map_obj{msgpack_empty_map_obj_handle.get()}; + + return serializer.serialize_msgpack_map(msgpack_empty_map_obj.via.map, msgpack_obj.via.map); } // NOLINTNEXTLINE(misc-no-recursion) @@ -388,7 +396,19 @@ auto unpack_and_assert_serialization_failure( auto const msgpack_obj_handle{msgpack::unpack(msgpack_bytes.data(), msgpack_bytes.size())}; auto const msgpack_obj{msgpack_obj_handle.get()}; REQUIRE((msgpack::type::MAP == msgpack_obj.type)); - if (serializer.serialize_msgpack_map(msgpack_obj.via.map)) { + + auto const msgpack_empty_map_buf{nlohmann::json::to_msgpack(nlohmann::json::parse("{}"))}; + auto const msgpack_empty_map_obj_handle{msgpack::unpack( + size_checked_pointer_cast(msgpack_empty_map_buf.data()), + msgpack_empty_map_buf.size() + )}; + auto const msgpack_empty_map_obj{msgpack_empty_map_obj_handle.get()}; + + if (serializer.serialize_msgpack_map(msgpack_obj.via.map, msgpack_empty_map_obj.via.map)) { + // Serialization should fail + return false; + } + if (serializer.serialize_msgpack_map(msgpack_empty_map_obj.via.map, msgpack_obj.via.map)) { // Serialization should fail return false; } From 56fd9bb3e0756ddfe9e5636a6b0928fecd4c0ddd Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 6 Jan 2025 15:54:53 -0500 Subject: [PATCH 2/6] Refactor unit test code --- .../core/tests/test-ir_encoding_methods.cpp | 67 +++++++++++++------ 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index 390f15f0c..b2f07e6f6 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -193,16 +193,23 @@ auto flush_and_clear_serializer_buffer( /** * Unpacks and serializes the given msgpack bytes using kv serializer. * @tparam encoded_variable_t - * @param msgpack_bytes + * @param auto_gen_msgpack_bytes + * @param user_gen_msgpack_bytes * @param serializer * @return Whether serialization succeeded. */ template [[nodiscard]] auto unpack_and_serialize_msgpack_bytes( - vector const& msgpack_bytes, + vector const& auto_gen_msgpack_bytes, + vector const& user_gen_msgpack_bytes, Serializer& serializer ) -> bool; +/** + * @return A msgpack object handle that holds an empty msgpack map. + */ +[[nodiscard]] auto create_msgpack_empty_map_obj_handle() -> msgpack::object_handle; + /** * Counts the number of leaves in a JSON tree. A node is considered as a leaf if it's a primitive * value, an empty map (`{}`), or an array. @@ -338,26 +345,40 @@ auto flush_and_clear_serializer_buffer( template auto unpack_and_serialize_msgpack_bytes( - vector const& msgpack_bytes, + vector const& auto_gen_msgpack_bytes, + vector const& user_gen_msgpack_bytes, Serializer& serializer ) -> bool { - auto const msgpack_obj_handle{msgpack::unpack( - clp::size_checked_pointer_cast(msgpack_bytes.data()), - msgpack_bytes.size() + auto const auto_gen_msgpack_byte_handle{msgpack::unpack( + clp::size_checked_pointer_cast(auto_gen_msgpack_bytes.data()), + auto_gen_msgpack_bytes.size() )}; - auto const msgpack_obj{msgpack_obj_handle.get()}; - if (msgpack::type::MAP != msgpack_obj.type) { + auto const auto_gen_msgpack_obj{auto_gen_msgpack_byte_handle.get()}; + if (msgpack::type::MAP != auto_gen_msgpack_obj.type) { return false; } + auto const user_gen_msgpack_byte_handle{msgpack::unpack( + clp::size_checked_pointer_cast(user_gen_msgpack_bytes.data()), + user_gen_msgpack_bytes.size() + )}; + auto const user_gen_msgpack_obj{user_gen_msgpack_byte_handle.get()}; + if (msgpack::type::MAP != user_gen_msgpack_obj.type) { + return false; + } + + return serializer.serialize_msgpack_map( + auto_gen_msgpack_obj.via.map, + user_gen_msgpack_obj.via.map + ); +} + +auto create_msgpack_empty_map_obj_handle() -> msgpack::object_handle { auto const msgpack_empty_map_buf{nlohmann::json::to_msgpack(nlohmann::json::parse("{}"))}; - auto const msgpack_empty_map_obj_handle{msgpack::unpack( + return msgpack::unpack( size_checked_pointer_cast(msgpack_empty_map_buf.data()), msgpack_empty_map_buf.size() - )}; - auto const msgpack_empty_map_obj{msgpack_empty_map_obj_handle.get()}; - - return serializer.serialize_msgpack_map(msgpack_empty_map_obj.via.map, msgpack_obj.via.map); + ); } // NOLINTNEXTLINE(misc-no-recursion) @@ -397,11 +418,7 @@ auto unpack_and_assert_serialization_failure( auto const msgpack_obj{msgpack_obj_handle.get()}; REQUIRE((msgpack::type::MAP == msgpack_obj.type)); - auto const msgpack_empty_map_buf{nlohmann::json::to_msgpack(nlohmann::json::parse("{}"))}; - auto const msgpack_empty_map_obj_handle{msgpack::unpack( - size_checked_pointer_cast(msgpack_empty_map_buf.data()), - msgpack_empty_map_buf.size() - )}; + auto const msgpack_empty_map_obj_handle{create_msgpack_empty_map_obj_handle()}; auto const msgpack_empty_map_obj{msgpack_empty_map_obj_handle.get()}; if (serializer.serialize_msgpack_map(msgpack_obj.via.map, msgpack_empty_map_obj.via.map)) { @@ -1212,7 +1229,11 @@ TEMPLATE_TEST_CASE( flush_and_clear_serializer_buffer(serializer, ir_buf); auto const empty_obj = nlohmann::json::parse("{}"); - REQUIRE(unpack_and_serialize_msgpack_bytes(nlohmann::json::to_msgpack(empty_obj), serializer)); + REQUIRE(unpack_and_serialize_msgpack_bytes( + nlohmann::json::to_msgpack(empty_obj), + nlohmann::json::to_msgpack(empty_obj), + serializer + )); serialized_json_objects.emplace_back(empty_obj); // Test encoding basic object @@ -1239,7 +1260,11 @@ TEMPLATE_TEST_CASE( {"empty_object", empty_obj}, {"empty_array", empty_array}}; - REQUIRE(unpack_and_serialize_msgpack_bytes(nlohmann::json::to_msgpack(basic_obj), serializer)); + REQUIRE(unpack_and_serialize_msgpack_bytes( + nlohmann::json::to_msgpack(empty_obj), + nlohmann::json::to_msgpack(basic_obj), + serializer + )); serialized_json_objects.emplace_back(basic_obj); auto basic_array = empty_array; @@ -1255,6 +1280,7 @@ TEMPLATE_TEST_CASE( REQUIRE( (false == unpack_and_serialize_msgpack_bytes( + nlohmann::json::to_msgpack(empty_obj), nlohmann::json::to_msgpack(element), serializer )) @@ -1271,6 +1297,7 @@ TEMPLATE_TEST_CASE( recursive_obj.emplace("obj_" + std::to_string(i), recursive_obj); recursive_obj.emplace("array_" + std::to_string(i), recursive_array); REQUIRE(unpack_and_serialize_msgpack_bytes( + nlohmann::json::to_msgpack(empty_obj), nlohmann::json::to_msgpack(recursive_obj), serializer )); From b7b250bc6bf1068658fb2144862d9ecf2ad74964 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 6 Jan 2025 16:53:15 -0500 Subject: [PATCH 3/6] Update IR unit handler interface to check whether the inserted node is auto/user-generated --- .../src/clp/ffi/ir_stream/Deserializer.hpp | 15 ++++++++++---- .../ffi/ir_stream/IrUnitHandlerInterface.hpp | 4 +++- .../ir_unit_deserialization_methods.cpp | 9 ++------- .../ir_unit_deserialization_methods.hpp | 20 ++++++++++--------- components/core/src/clp_s/JsonParser.cpp | 1 + .../tests/test-ffi_IrUnitHandlerInterface.cpp | 2 ++ .../core/tests/test-ir_encoding_methods.cpp | 1 + 7 files changed, 31 insertions(+), 21 deletions(-) diff --git a/components/core/src/clp/ffi/ir_stream/Deserializer.hpp b/components/core/src/clp/ffi/ir_stream/Deserializer.hpp index d31699cd2..993fa2223 100644 --- a/components/core/src/clp/ffi/ir_stream/Deserializer.hpp +++ b/components/core/src/clp/ffi/ir_stream/Deserializer.hpp @@ -211,18 +211,25 @@ auto Deserializer::deserialize_next_ir_unit(ReaderInterface& read return result.error(); } - auto const node_locator{result.value()}; - if (m_user_gen_keys_schema_tree->has_node(node_locator)) { + auto const& [is_auto_generated, node_locator]{result.value()}; + auto& schema_tree_to_insert{ + is_auto_generated ? m_auto_gen_keys_schema_tree : m_user_gen_keys_schema_tree + }; + + if (schema_tree_to_insert->has_node(node_locator)) { return std::errc::protocol_error; } - if (auto const err{m_ir_unit_handler.handle_schema_tree_node_insertion(node_locator)}; + if (auto const err{m_ir_unit_handler.handle_schema_tree_node_insertion( + is_auto_generated, + node_locator + )}; IRErrorCode::IRErrorCode_Success != err) { return ir_error_code_to_errc(err); } - std::ignore = m_user_gen_keys_schema_tree->insert_node(node_locator); + std::ignore = schema_tree_to_insert->insert_node(node_locator); break; } diff --git a/components/core/src/clp/ffi/ir_stream/IrUnitHandlerInterface.hpp b/components/core/src/clp/ffi/ir_stream/IrUnitHandlerInterface.hpp index cb2f0ac18..aa98401bc 100644 --- a/components/core/src/clp/ffi/ir_stream/IrUnitHandlerInterface.hpp +++ b/components/core/src/clp/ffi/ir_stream/IrUnitHandlerInterface.hpp @@ -17,6 +17,7 @@ template concept IrUnitHandlerInterface = requires( Handler handler, KeyValuePairLogEvent&& log_event, + bool is_auto_generated, UtcOffset utc_offset_old, UtcOffset utc_offset_new, SchemaTree::NodeLocator schema_tree_node_locator @@ -42,11 +43,12 @@ concept IrUnitHandlerInterface = requires( /** * Handles a schema tree node insertion IR unit. + * @param is_auto_generated Whether the node is from auto-gen keys schema tree * @param schema_tree_node_locator The locator of the node to insert. * @return IRErrorCode::Success on success, user-defined error code on failures. */ { - handler.handle_schema_tree_node_insertion(schema_tree_node_locator) + handler.handle_schema_tree_node_insertion(is_auto_generated, schema_tree_node_locator) } -> std::same_as; /** diff --git a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp index cea4a1b84..71144ab55 100644 --- a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp +++ b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp @@ -512,7 +512,7 @@ auto deserialize_ir_unit_schema_tree_node_insertion( ReaderInterface& reader, encoded_tag_t tag, std::string& key_name -) -> OUTCOME_V2_NAMESPACE::std_result { +) -> OUTCOME_V2_NAMESPACE::std_result> { auto const type{schema_tree_node_tag_to_type(tag)}; if (false == type.has_value()) { return ir_error_code_to_errc(IRErrorCode::IRErrorCode_Corrupted_IR); @@ -523,18 +523,13 @@ auto deserialize_ir_unit_schema_tree_node_insertion( return parent_node_id_result.error(); } auto const [is_auto_generated, parent_id]{parent_node_id_result.value()}; - if (is_auto_generated) { - // Currently, we don't support auto-generated keys. - return std::errc::protocol_not_supported; - } - if (auto const err{deserialize_schema_tree_node_key_name(reader, key_name)}; IRErrorCode::IRErrorCode_Success != err) { return ir_error_code_to_errc(err); } - return SchemaTree::NodeLocator{parent_id, key_name, type.value()}; + return {is_auto_generated, SchemaTree::NodeLocator{parent_id, key_name, type.value()}}; } auto deserialize_ir_unit_utc_offset_change(ReaderInterface& reader diff --git a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp index 451f627db..921df8efe 100644 --- a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp +++ b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -28,20 +29,21 @@ namespace clp::ffi::ir_stream { * @param tag * @param key_name Returns the key name of the deserialized new node. This should be the underlying * storage of the returned schema tree node locator. - * @return A result containing the locator of the inserted schema tree node or an error code - * indicating the failure: - * - std::errc::result_out_of_range if the IR stream is truncated. - * - std::errc::protocol_error if the deserialized node type isn't supported. - * - std::errc::protocol_not_supported if the IR stream contains auto-generated keys (TODO: Remove - * this once auto-generated keys are fully supported). - * - Forwards `deserialize_schema_tree_node_key_name`'s return values. - * - Forwards `deserialize_schema_tree_node_parent_id`'s return values. + * @return A result containing a pair or an error code indicating the failure: + * - The pair: + * - Whether the node is for auto-generated keys schema tree. + * - The locator of the inserted schema tree node. + * - The possible error codes: + * - std::errc::result_out_of_range if the IR stream is truncated. + * - std::errc::protocol_error if the deserialized node type isn't supported. + * - Forwards `deserialize_schema_tree_node_key_name`'s return values. + * - Forwards `deserialize_schema_tree_node_parent_id`'s return values. */ [[nodiscard]] auto deserialize_ir_unit_schema_tree_node_insertion( ReaderInterface& reader, encoded_tag_t tag, std::string& key_name -) -> OUTCOME_V2_NAMESPACE::std_result; +) -> OUTCOME_V2_NAMESPACE::std_result>; /** * Deserializes a UTC offset change IR unit. diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index c917b1f09..b785c585b 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -49,6 +49,7 @@ class IrUnitHandler { } [[nodiscard]] auto handle_schema_tree_node_insertion( + [[maybe_unused]] bool is_auto_generated, [[maybe_unused]] clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator ) -> IRErrorCode { return IRErrorCode::IRErrorCode_Success; diff --git a/components/core/tests/test-ffi_IrUnitHandlerInterface.cpp b/components/core/tests/test-ffi_IrUnitHandlerInterface.cpp index 8f76a2f1a..e5b87577a 100644 --- a/components/core/tests/test-ffi_IrUnitHandlerInterface.cpp +++ b/components/core/tests/test-ffi_IrUnitHandlerInterface.cpp @@ -39,6 +39,7 @@ class TrivialIrUnitHandler { } [[nodiscard]] auto handle_schema_tree_node_insertion( + [[maybe_unused]] bool is_auto_generated, SchemaTree::NodeLocator schema_tree_node_locator ) -> IRErrorCode { m_schema_tree_node_locator.emplace(schema_tree_node_locator); @@ -109,6 +110,7 @@ auto test_ir_unit_handler_interface(clp::ffi::ir_stream::IrUnitHandlerInterface REQUIRE( (IRErrorCode::IRErrorCode_Success == handler.handle_schema_tree_node_insertion( + true, {SchemaTree::cRootId, cTestSchemaTreeNodeKeyName, SchemaTree::Node::Type::Obj} )) ); diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index b2f07e6f6..ef45515f9 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -111,6 +111,7 @@ class IrUnitHandler { } [[nodiscard]] static auto handle_schema_tree_node_insertion( + [[maybe_unused]] bool is_auto_generated, [[maybe_unused]] clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator ) -> IRErrorCode { return IRErrorCode::IRErrorCode_Success; From d2e906fb249d93f4f08ec10641bff286b0825e2b Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 6 Jan 2025 17:53:54 -0500 Subject: [PATCH 4/6] Add support for deserializing auto-generated kv-pairs --- .../ir_unit_deserialization_methods.cpp | 108 +++++++++++++----- .../ir_unit_deserialization_methods.hpp | 2 +- .../core/tests/test-ir_encoding_methods.cpp | 6 +- 3 files changed, 86 insertions(+), 30 deletions(-) diff --git a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp index 71144ab55..2bf7ad7f2 100644 --- a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp +++ b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp @@ -98,17 +98,24 @@ deserialize_int_val(ReaderInterface& reader, encoded_tag_t tag, value_int_t& val ) -> IRErrorCode; /** - * Deserializes the IDs of all keys in a log event. + * Deserializes the auto-generated node-ID-value pairs and the IDs of all user-generated keys in a + * log event. * @param reader * @param tag Takes the current tag as input and returns the last tag read. - * @return A result containing the deserialized schema or an error code indicating the failure: - * - std::err::protocol_not_supported if the IR stream contains auto-generated keys (TODO: Remove - * this once auto-generated keys are fully supported). - * - Forwards `deserialize_tag`'s return values. - * - Forwards `deserialize_and_decode_schema_tree_node_id`'s return values. + * @return A result containing a pair or an error code indicating the failure: + * - The pair: + * - The auto-generated node-ID-value pairs. + * - The IDs of all user-generated keys. + * - The possible error codes: + * - Forwards `deserialize_tag`'s return values. + * - Forwards `deserialize_and_decode_schema_tree_node_id`'s return values. + * - std::err::protocol_error if the IR stream contains auto-generated key IDs after at least one + * user-generated key ID has been deserialized. */ -[[nodiscard]] auto deserialize_schema(ReaderInterface& reader, encoded_tag_t& tag) - -> OUTCOME_V2_NAMESPACE::std_result; +[[nodiscard]] auto deserialize_auto_gen_node_id_value_pairs_and_user_gen_schema( + ReaderInterface& reader, + encoded_tag_t& tag +) -> OUTCOME_V2_NAMESPACE::std_result>; /** * Deserializes the next value and pushes the result into `node_id_value_pairs`. @@ -292,12 +299,59 @@ auto deserialize_string(ReaderInterface& reader, encoded_tag_t tag, std::string& return IRErrorCode::IRErrorCode_Success; } -auto deserialize_schema(ReaderInterface& reader, encoded_tag_t& tag) - -> OUTCOME_V2_NAMESPACE::std_result { - Schema schema; +auto deserialize_auto_gen_node_id_value_pairs_and_user_gen_schema( + ReaderInterface& reader, + encoded_tag_t& tag +) -> OUTCOME_V2_NAMESPACE::std_result> { + KeyValuePairLogEvent::NodeIdValuePairs auto_gen_node_id_value_pairs; + Schema user_gen_schema; + + // Deserialize auto generated node id value pairs + while (true) { + if (false == is_encoded_key_id_tag(tag)) { + break; + } + + auto const schema_tree_node_id_result{deserialize_and_decode_schema_tree_node_id< + cProtocol::Payload::EncodedSchemaTreeNodeIdByte, + cProtocol::Payload::EncodedSchemaTreeNodeIdShort, + cProtocol::Payload::EncodedSchemaTreeNodeIdInt>(tag, reader)}; + if (schema_tree_node_id_result.has_error()) { + return schema_tree_node_id_result.error(); + } + + // Advance to the next tag. This is needed no matter whether the deserialized node ID is + // auto-generated. + if (auto const err{deserialize_tag(reader, tag)}; IRErrorCode::IRErrorCode_Success != err) { + return ir_error_code_to_errc(err); + } + + auto const [is_auto_generated, node_id]{schema_tree_node_id_result.value()}; + if (false == is_auto_generated) { + // User-generated node ID has been deserialized, pushes the node and terminates + // auto-generated node-ID-value pair deserialization. + user_gen_schema.push_back(node_id); + break; + } + + if (auto const err{deserialize_value_and_insert_to_node_id_value_pairs( + reader, + tag, + node_id, + auto_gen_node_id_value_pairs + )}; + IRErrorCode::IRErrorCode_Success != err) + { + return ir_error_code_to_errc(err); + } + + if (auto const err{deserialize_tag(reader, tag)}; IRErrorCode::IRErrorCode_Success != err) { + return ir_error_code_to_errc(err); + } + } + while (true) { if (false == is_encoded_key_id_tag(tag)) { - // The log event must be an empty value. break; } @@ -310,17 +364,16 @@ auto deserialize_schema(ReaderInterface& reader, encoded_tag_t& tag) } auto const [is_auto_generated, node_id]{schema_tree_node_id_result.value()}; if (is_auto_generated) { - // Currently, we don't support auto-generated keys. - return std::errc::protocol_not_supported; + return std::errc::protocol_error; } - schema.push_back(node_id); + user_gen_schema.push_back(node_id); if (auto const err{deserialize_tag(reader, tag)}; IRErrorCode::IRErrorCode_Success != err) { return ir_error_code_to_errc(err); } } - return schema; + return {std::move(auto_gen_node_id_value_pairs), std::move(user_gen_schema)}; } auto deserialize_value_and_insert_to_node_id_value_pairs( @@ -550,19 +603,22 @@ auto deserialize_ir_unit_kv_pair_log_event( std::shared_ptr user_gen_keys_schema_tree, UtcOffset utc_offset ) -> OUTCOME_V2_NAMESPACE::std_result { - auto const schema_result{deserialize_schema(reader, tag)}; - if (schema_result.has_error()) { - return schema_result.error(); + auto auto_gen_node_id_value_pairs_and_user_gen_schema_result{ + deserialize_auto_gen_node_id_value_pairs_and_user_gen_schema(reader, tag) + }; + if (auto_gen_node_id_value_pairs_and_user_gen_schema_result.has_error()) { + return auto_gen_node_id_value_pairs_and_user_gen_schema_result.error(); } - auto const& schema{schema_result.value()}; + auto& [auto_gen_node_id_value_pairs, + user_gen_schema]{auto_gen_node_id_value_pairs_and_user_gen_schema_result.value()}; - KeyValuePairLogEvent::NodeIdValuePairs node_id_value_pairs; - if (false == schema.empty()) { + KeyValuePairLogEvent::NodeIdValuePairs user_gen_node_id_value_pairs; + if (false == user_gen_schema.empty()) { if (auto const err{deserialize_value_and_construct_node_id_value_pairs( reader, tag, - schema, - node_id_value_pairs + user_gen_schema, + user_gen_node_id_value_pairs )}; IRErrorCode::IRErrorCode_Success != err) { @@ -577,8 +633,8 @@ auto deserialize_ir_unit_kv_pair_log_event( return KeyValuePairLogEvent::create( std::move(auto_gen_keys_schema_tree), std::move(user_gen_keys_schema_tree), - {}, - std::move(node_id_value_pairs), + std::move(auto_gen_node_id_value_pairs), + std::move(user_gen_node_id_value_pairs), utc_offset ); } diff --git a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp index 921df8efe..0efca471c 100644 --- a/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp +++ b/components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp @@ -69,7 +69,7 @@ namespace clp::ffi::ir_stream { * - std::errc::protocol_error if the IR stream is corrupted. * - std::errc::protocol_not_supported if the IR stream contains an unsupported metadata format * or uses an unsupported version. - * - Forwards `deserialize_schema`'s return values. + * - Forwards `deserialize_auto_gen_node_id_value_pairs_and_user_gen_schema`'s return values. * - Forwards `KeyValuePairLogEvent::create`'s return values if the intermediate deserialized result * cannot construct a valid key-value pair log event. */ diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index ef45515f9..78e382fc9 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -1262,7 +1262,7 @@ TEMPLATE_TEST_CASE( {"empty_array", empty_array}}; REQUIRE(unpack_and_serialize_msgpack_bytes( - nlohmann::json::to_msgpack(empty_obj), + nlohmann::json::to_msgpack(basic_obj), nlohmann::json::to_msgpack(basic_obj), serializer )); @@ -1298,7 +1298,7 @@ TEMPLATE_TEST_CASE( recursive_obj.emplace("obj_" + std::to_string(i), recursive_obj); recursive_obj.emplace("array_" + std::to_string(i), recursive_array); REQUIRE(unpack_and_serialize_msgpack_bytes( - nlohmann::json::to_msgpack(empty_obj), + nlohmann::json::to_msgpack(recursive_obj), nlohmann::json::to_msgpack(recursive_obj), serializer )); @@ -1341,7 +1341,7 @@ TEMPLATE_TEST_CASE( auto const serialized_json_result{deserialized_log_event.serialize_to_json()}; REQUIRE_FALSE(serialized_json_result.has_error()); auto const& [auto_generated, user_generated]{serialized_json_result.value()}; - REQUIRE(auto_generated.empty()); + REQUIRE((expect == auto_generated)); REQUIRE((expect == user_generated)); } From 3f3e86890a08502f87c27adc8fb730bdf357e22c Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 7 Jan 2025 17:27:07 -0500 Subject: [PATCH 5/6] Update unit tests --- .../core/tests/test-ir_encoding_methods.cpp | 62 ++++++++++--------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index 78e382fc9..ac4fee4d5 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -1221,7 +1221,7 @@ TEMPLATE_TEST_CASE( eight_byte_encoded_variable_t ) { vector ir_buf; - vector serialized_json_objects; + vector> expected_serialized_json_object_pairs; auto result{Serializer::create()}; REQUIRE((false == result.has_error())); @@ -1230,12 +1230,7 @@ TEMPLATE_TEST_CASE( flush_and_clear_serializer_buffer(serializer, ir_buf); auto const empty_obj = nlohmann::json::parse("{}"); - REQUIRE(unpack_and_serialize_msgpack_bytes( - nlohmann::json::to_msgpack(empty_obj), - nlohmann::json::to_msgpack(empty_obj), - serializer - )); - serialized_json_objects.emplace_back(empty_obj); + expected_serialized_json_object_pairs.emplace_back(empty_obj, empty_obj); // Test encoding basic object constexpr string_view cShortString{"short_string"}; @@ -1260,13 +1255,7 @@ TEMPLATE_TEST_CASE( {"null", nullptr}, {"empty_object", empty_obj}, {"empty_array", empty_array}}; - - REQUIRE(unpack_and_serialize_msgpack_bytes( - nlohmann::json::to_msgpack(basic_obj), - nlohmann::json::to_msgpack(basic_obj), - serializer - )); - serialized_json_objects.emplace_back(basic_obj); + expected_serialized_json_object_pairs.emplace_back(basic_obj, basic_obj); auto basic_array = empty_array; basic_array.emplace_back(1); @@ -1294,17 +1283,22 @@ TEMPLATE_TEST_CASE( auto recursive_array = basic_array; constexpr size_t cRecursiveDepth{6}; for (size_t i{0}; i < cRecursiveDepth; ++i) { + auto const original_obj = recursive_obj; recursive_array.emplace_back(recursive_obj); - recursive_obj.emplace("obj_" + std::to_string(i), recursive_obj); + recursive_obj.emplace("obj_" + std::to_string(i), original_obj); recursive_obj.emplace("array_" + std::to_string(i), recursive_array); + expected_serialized_json_object_pairs.emplace_back(original_obj, recursive_obj); + expected_serialized_json_object_pairs.emplace_back(empty_obj, recursive_obj); + } + + for (auto const& [auto_gen_json_obj, user_gen_json_obj] : expected_serialized_json_object_pairs) + { REQUIRE(unpack_and_serialize_msgpack_bytes( - nlohmann::json::to_msgpack(recursive_obj), - nlohmann::json::to_msgpack(recursive_obj), + nlohmann::json::to_msgpack(auto_gen_json_obj), + nlohmann::json::to_msgpack(user_gen_json_obj), serializer )); - serialized_json_objects.emplace_back(recursive_obj); } - flush_and_clear_serializer_buffer(serializer, ir_buf); ir_buf.push_back(clp::ffi::ir_stream::cProtocol::Eof); @@ -1327,22 +1321,34 @@ TEMPLATE_TEST_CASE( REQUIRE(deserializer.is_stream_completed()); // Check the number of log events deserialized matches the number of log events serialized auto const& deserialized_log_events{ir_unit_handler.get_deserialized_log_events()}; - REQUIRE((serialized_json_objects.size() == deserialized_log_events.size())); + REQUIRE((expected_serialized_json_object_pairs.size() == deserialized_log_events.size())); - auto const num_log_events{serialized_json_objects.size()}; + auto const num_log_events{expected_serialized_json_object_pairs.size()}; for (size_t idx{0}; idx < num_log_events; ++idx) { - auto const& expect{serialized_json_objects.at(idx)}; + auto const& [expected_auto_gen_json_obj, expected_user_gen_json_obj]{ + expected_serialized_json_object_pairs.at(idx) + }; auto const& deserialized_log_event{deserialized_log_events.at(idx)}; - auto const num_leaves_in_json_obj{count_num_leaves(expect)}; - auto const num_kv_pairs{deserialized_log_event.get_user_gen_node_id_value_pairs().size()}; - REQUIRE((num_leaves_in_json_obj == num_kv_pairs)); + auto const num_leaves_in_auto_gen_json_obj{count_num_leaves(expected_auto_gen_json_obj)}; + auto const num_auto_gen_kv_pairs{ + deserialized_log_event.get_auto_gen_node_id_value_pairs().size() + }; + REQUIRE((num_leaves_in_auto_gen_json_obj == num_auto_gen_kv_pairs)); + + auto const num_leaves_in_user_gen_json_obj{count_num_leaves(expected_user_gen_json_obj)}; + auto const num_user_gen_kv_pairs{ + deserialized_log_event.get_user_gen_node_id_value_pairs().size() + }; + REQUIRE((num_leaves_in_user_gen_json_obj == num_user_gen_kv_pairs)); auto const serialized_json_result{deserialized_log_event.serialize_to_json()}; REQUIRE_FALSE(serialized_json_result.has_error()); - auto const& [auto_generated, user_generated]{serialized_json_result.value()}; - REQUIRE((expect == auto_generated)); - REQUIRE((expect == user_generated)); + auto const& [actual_auto_gen_json_obj, actual_user_gen_json_obj]{ + serialized_json_result.value() + }; + REQUIRE((expected_auto_gen_json_obj == actual_auto_gen_json_obj)); + REQUIRE((expected_user_gen_json_obj == actual_user_gen_json_obj)); } auto const eof_result{deserializer.deserialize_next_ir_unit(reader)}; From 7edbecdf665620cd8c25c09981234c167abd1d33 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 7 Jan 2025 17:38:24 -0500 Subject: [PATCH 6/6] Update test case name --- components/core/tests/test-ir_encoding_methods.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index ac4fee4d5..f9fb75740 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -1215,8 +1215,8 @@ TEMPLATE_TEST_CASE( // NOLINTNEXTLINE(readability-function-cognitive-complexity) TEMPLATE_TEST_CASE( - "ffi_ir_stream_Serializer_serialize_msgpack", - "[clp][ffi][ir_stream][Serializer]", + "ffi_ir_stream_kv_pair_log_events_serder", + "[clp][ffi][ir_stream]", four_byte_encoded_variable_t, eight_byte_encoded_variable_t ) {