From aa1581037d3c8169d63d8212fb1d59176cb7b94b Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Tue, 21 Jan 2025 15:11:43 -0700 Subject: [PATCH 01/67] Initial work off of valgur's branch initial seperation Switch to python object pointers remove fluff Add base fields Convert to cached list Partial work time tracking Cahnge time trackign make dir method reasonably fast --- python/bindings/bindings.cpp | 2 +- python/bindings/encoder.cpp | 2 +- python/bindings/json_db_reader.cpp | 6 +- python/bindings/message_database.cpp | 40 +++++++- python/bindings/message_decoder.cpp | 118 ++++++++++++------------ python/bindings/py_database.hpp | 5 +- python/bindings/py_decoded_message.hpp | 27 +++--- python/examples/converter_components.py | 24 +++-- python/novatel_edie/__init__.py | 1 - 9 files changed, 139 insertions(+), 86 deletions(-) diff --git a/python/bindings/bindings.cpp b/python/bindings/bindings.cpp index a4e73c4b5..11a49931a 100644 --- a/python/bindings/bindings.cpp +++ b/python/bindings/bindings.cpp @@ -25,7 +25,6 @@ NB_MODULE(bindings, m) init_common_common(m); init_common_logger(m); init_common_json_db_reader(m); - init_common_message_database(m); init_common_nexcept(m); init_novatel_commander(m); init_novatel_common(m); @@ -39,4 +38,5 @@ NB_MODULE(bindings, m) init_novatel_parser(m); init_novatel_range_decompressor(m); init_novatel_rxconfig_handler(m); + init_common_message_database(m); } diff --git a/python/bindings/encoder.cpp b/python/bindings/encoder.cpp index d7e796172..482ef331f 100644 --- a/python/bindings/encoder.cpp +++ b/python/bindings/encoder.cpp @@ -20,7 +20,7 @@ void init_novatel_encoder(nb::module_& m) .def_prop_ro("logger", [](const oem::Encoder& encoder) { return encoder.GetLogger(); }) .def( "encode", - [](oem::Encoder& encoder, const oem::IntermediateHeader& header, const oem::PyDecodedMessage& py_message, + [](oem::Encoder& encoder, const oem::IntermediateHeader& header, const oem::PyMessageBody& py_message, const oem::MetaDataStruct& metadata, ENCODE_FORMAT format) { MessageDataStruct message_data; if (format == ENCODE_FORMAT::JSON) diff --git a/python/bindings/json_db_reader.cpp b/python/bindings/json_db_reader.cpp index fbeb7c8e7..f3e95c01a 100644 --- a/python/bindings/json_db_reader.cpp +++ b/python/bindings/json_db_reader.cpp @@ -32,7 +32,11 @@ std::string default_json_db_path() PyMessageDatabase::Ptr& MessageDbSingleton::get() { static PyMessageDatabase::Ptr json_db = nullptr; - if (!json_db) { json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path())); } + if (!json_db) { + nb::module_ m = nb::module_::import_("novatel_edie"); + nb::module_ messages_mod = m.def_submodule("messages", "Message types used by NovAtel OEM messages."); + json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path())); + } return json_db; } diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index f043cc098..5875d9cd8 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -4,6 +4,7 @@ #include "bindings_core.hpp" #include "py_database.hpp" +#include "py_decoded_message.hpp" namespace nb = nanobind; using namespace nb::literals; @@ -179,26 +180,38 @@ void init_common_message_database(nb::module_& m) .def("get_msg_def", nb::overload_cast(&PyMessageDatabase::GetMsgDef, nb::const_), "msg_id"_a) .def("get_enum_def", &PyMessageDatabase::GetEnumDefId, "enum_id"_a) .def("get_enum_def", &PyMessageDatabase::GetEnumDefName, "enum_name"_a) - .def_prop_ro("enums", &PyMessageDatabase::GetEnumsByNameDict); + .def_prop_ro("enums", &PyMessageDatabase::GetEnumsByNameDict) + .def_ro("message_types", &PyMessageDatabase::message_types); } -PyMessageDatabase::PyMessageDatabase() { UpdatePythonEnums(); } +PyMessageDatabase::PyMessageDatabase() { + UpdatePythonEnums(); + UpdateMessageTypes(); +} PyMessageDatabase::PyMessageDatabase(std::vector vMessageDefinitions_, std::vector vEnumDefinitions_) : MessageDatabase(std::move(vMessageDefinitions_), std::move(vEnumDefinitions_)) { UpdatePythonEnums(); + UpdateMessageTypes(); } -PyMessageDatabase::PyMessageDatabase(const MessageDatabase& message_db) noexcept : MessageDatabase(message_db) { UpdatePythonEnums(); } +PyMessageDatabase::PyMessageDatabase(const MessageDatabase& message_db) noexcept : MessageDatabase(message_db) { + UpdatePythonEnums(); + UpdateMessageTypes(); +} -PyMessageDatabase::PyMessageDatabase(const MessageDatabase&& message_db) noexcept : MessageDatabase(message_db) { UpdatePythonEnums(); } +PyMessageDatabase::PyMessageDatabase(const MessageDatabase&& message_db) noexcept : MessageDatabase(message_db) { + UpdatePythonEnums(); + UpdateMessageTypes(); +} void PyMessageDatabase::GenerateMappings() { MessageDatabase::GenerateMappings(); UpdatePythonEnums(); + UpdateMessageTypes(); } inline void PyMessageDatabase::UpdatePythonEnums() @@ -218,3 +231,22 @@ inline void PyMessageDatabase::UpdatePythonEnums() enums_by_name[enum_name] = enum_type; } } + +void PyMessageDatabase::UpdateMessageTypes() +{ + nb::module_ messages_mod = nb::module_::import_("novatel_edie.messages"); + nb::handle py_type = nb::type(); + nb::object Type = nb::module_::import_("builtins").attr("type"); + nb::dict type_dict = nb::dict(); + nb::tuple type_tuple = nb::make_tuple(py_type); + for (const auto& message_def : MessageDefinitions()) { + nb::object msg_def = Type(message_def->name + "MessageBody", type_tuple, type_dict); + msg_def.attr("__module__") = "novatel_edie.messages"; + message_types[message_def->name] = msg_def; + messages_mod.attr((message_def->name + "MessageBody").c_str()) = msg_def; + } + nb::object defaut_type = Type("UNKNOWNMessageBody", type_tuple, type_dict); + defaut_type.attr("__module__") = "novatel_edie.messages"; + message_types["UNKNOWN"] = defaut_type; + messages_mod.attr("UNKNOWNMessageBody") = defaut_type; +} diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 863f171eb..2cffc0e3a 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include "bindings_core.hpp" #include "message_db_singleton.hpp" @@ -63,7 +65,7 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C else { // This is an array element of a field array. - return nb::cast(PyDecodedMessage(message_field, {}, parent_db)); + return nb::cast(PyMessageBody(message_field, parent_db)); } } else if (field.fieldDef->conversion == "%id") @@ -81,13 +83,24 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } } -PyDecodedMessage::PyDecodedMessage(std::vector message_, const oem::MetaDataStruct& meta_, PyMessageDatabase::ConstPtr parent_db_) - : fields(std::move(message_)), message_id(meta_.usMessageId), message_crc(meta_.uiMessageCrc), message_name(meta_.acMessageName), time(meta_), - measurement_source(meta_.eMeasurementSource), constellation(meta_.constellation), parent_db_(std::move(parent_db_)) +std::vector PyMessageBody::base_fields = { + "__class__", "__contains__", "__delattr__", "__dir__", "__doc__", "__eq__", "__format__", "__ge__", + "__getattr__", "__getattribute__", "__getitem__", "__getstate__", "__gt__", "__hash__", "__init__", "__init_subclass__", + "__le__", "__len__", "__lt__", "__module__", "__ne__", "__new__", "__reduce__", "__reduce_ex__", + "__repr__", "__setattr__", "__sizeof__", "__str__", "__subclasshook__"}; + +std::vector PyMessageBody::get_field_names() const { + std::vector field_names = base_fields; + for (const auto& [field_name, _] : get_fields()) { field_names.push_back(nb::cast(field_name)); } + return field_names; } -nb::dict& PyDecodedMessage::get_values() const +PyMessageBody::PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_) + : fields(std::move(message_)), parent_db_(std::move(parent_db_)) +{} + +nb::dict& PyMessageBody::get_values() const { if (cached_values_.size() == 0) { @@ -96,7 +109,7 @@ nb::dict& PyDecodedMessage::get_values() const return cached_values_; } -nb::dict& PyDecodedMessage::get_fields() const +nb::dict& PyMessageBody::get_fields() const { if (cached_fields_.size() == 0) { @@ -105,18 +118,18 @@ nb::dict& PyDecodedMessage::get_fields() const return cached_fields_; } -nb::dict PyDecodedMessage::to_dict() const +nb::dict PyMessageBody::to_dict() const { nb::dict dict; for (const auto& [field_name, value] : get_values()) { - if (nb::isinstance(value)) { dict[field_name] = nb::cast(value).to_dict(); } + if (nb::isinstance(value)) { dict[field_name] = nb::cast(value).to_dict(); } else if (nb::isinstance>(value)) { nb::list list; for (const auto& sub_item : nb::cast>(value)) { - if (nb::isinstance(sub_item)) { list.append(nb::cast(sub_item).to_dict()); } + if (nb::isinstance(sub_item)) { list.append(nb::cast(sub_item).to_dict()); } else { list.append(sub_item); } } dict[field_name] = list; @@ -126,23 +139,23 @@ nb::dict PyDecodedMessage::to_dict() const return dict; } -nb::object PyDecodedMessage::getattr(nb::str field_name) const +nb::object PyMessageBody::getattr(nb::str field_name) const { if (!contains(field_name)) { throw nb::attribute_error(field_name.c_str()); } return get_values()[std::move(field_name)]; } -nb::object PyDecodedMessage::getitem(nb::str field_name) const { return get_values()[std::move(field_name)]; } +nb::object PyMessageBody::getitem(nb::str field_name) const { return get_values()[std::move(field_name)]; } -bool PyDecodedMessage::contains(nb::str field_name) const { return get_values().contains(std::move(field_name)); } +bool PyMessageBody::contains(nb::str field_name) const { return get_values().contains(std::move(field_name)); } -size_t PyDecodedMessage::len() const { return fields.size(); } +size_t PyMessageBody::len() const { return fields.size(); } -std::string PyDecodedMessage::repr() const +std::string PyMessageBody::repr() const { std::stringstream repr; - repr << "<"; - if (!message_name.empty()) { repr << message_name << " "; } + repr << "MessageBody("; + //if (!message_name.empty()) { repr << message_name << " "; } bool first = true; for (const auto& [field_name, value] : get_values()) { @@ -150,7 +163,7 @@ std::string PyDecodedMessage::repr() const first = false; repr << nb::str("{}={!r}").format(field_name, value).c_str(); } - repr << ">"; + repr << ")"; return repr.str(); } @@ -185,22 +198,18 @@ void init_novatel_message_decoder(nb::module_& m) .def_rw("milliseconds", &PyGpsTime::milliseconds) .def_rw("status", &PyGpsTime::time_status); - nb::class_(m, "Message") - .def_rw("message_id", &PyDecodedMessage::message_id) - .def_rw("message_crc", &PyDecodedMessage::message_crc) - .def_rw("message_name", &PyDecodedMessage::message_name) - .def_rw("message_time", &PyDecodedMessage::time) - .def_rw("message_measurement_source", &PyDecodedMessage::measurement_source) - .def_rw("message_constellation", &PyDecodedMessage::constellation) - .def_prop_ro("_values", &PyDecodedMessage::get_values) - .def_prop_ro("_fields", &PyDecodedMessage::get_fields) - .def("to_dict", &PyDecodedMessage::to_dict, "Convert the message and its sub-messages into a dict") - .def("__getattr__", &PyDecodedMessage::getattr, "field_name"_a) - .def("__getitem__", &PyDecodedMessage::getitem, "field_name"_a) - .def("__contains__", &PyDecodedMessage::contains, "field_name"_a) - .def("__len__", &PyDecodedMessage::len) - .def("__repr__", &PyDecodedMessage::repr) - .def("__str__", &PyDecodedMessage::repr); + nb::class_(m, "MessageBody") + .def_prop_ro("_values", &PyMessageBody::get_values) + .def_prop_ro("_fields", &PyMessageBody::get_fields) + .def("to_dict", &PyMessageBody::to_dict, "Convert the message and its sub-messages into a dict") + .def("__getattr__", &PyMessageBody::getattr, "field_name"_a) + //.def("__repr__", &PyMessageBody::repr) + .def("__str__", &PyMessageBody::repr) + .def("__dir__", &PyMessageBody::get_field_names); + + nb::class_(m, "Message") + .def_ro("body", &PyMessage::message_body) + .def_ro("header", &PyMessage::header); nb::class_(m, "FieldContainer") .def_rw("value", &FieldContainer::fieldValue) @@ -216,33 +225,24 @@ void init_novatel_message_decoder(nb::module_& m) .def_prop_ro("logger", [](oem::MessageDecoder& decoder) { return decoder.GetLogger(); }) .def( "decode", - [](const oem::MessageDecoder& decoder, const nb::bytes& message_body, oem::MetaDataStruct& metadata) { + [](const oem::MessageDecoder& decoder, const nb::bytes& message_body, nb::object header, oem::MetaDataStruct& metadata) { std::vector fields; STATUS status = decoder.Decode(reinterpret_cast(message_body.c_str()), fields, metadata); - return nb::make_tuple(status, PyDecodedMessage(std::move(fields), metadata, get_parent_db(decoder))); - }, - "message_body"_a, "metadata"_a) - // For internal testing purposes only - .def( - "_decode_ascii", - [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body) { - std::vector fields; - // Copy to ensure that the byte string is zero-delimited - std::string body_str(message_body.c_str(), message_body.size()); - const char* data_ptr = body_str.c_str(); - STATUS status = static_cast(&decoder)->TestDecodeAscii(msg_def_fields, &data_ptr, fields); - return nb::make_tuple(status, PyDecodedMessage(std::move(fields), {}, get_parent_db(decoder))); - }, - "msg_def_fields"_a, "message_body"_a) - .def( - "_decode_binary", - [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body, - uint32_t message_length) { - std::vector fields; - const char* data_ptr = message_body.c_str(); - STATUS status = static_cast(&decoder)->TestDecodeBinary(msg_def_fields, reinterpret_cast(&data_ptr), - fields, message_length); - return nb::make_tuple(status, PyDecodedMessage(std::move(fields), {}, get_parent_db(decoder))); + PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); + nb::handle body_pytype; + try { + body_pytype = parent_db->message_types.at(metadata.MessageName()); + } catch (const std::out_of_range& e) + { + body_pytype = parent_db->message_types.at("UNKNOWN"); + } + bool valid = nb::type_check(body_pytype); + nb::object body_pyinst = nb::inst_alloc(body_pytype); + PyMessageBody* body_cinst = nb::inst_ptr(body_pyinst); + new (body_cinst) PyMessageBody(std::move(fields), parent_db); + nb::inst_mark_ready(body_pyinst); + PyMessage message = PyMessage(body_pyinst, header); + return nb::make_tuple(status, message); }, - "msg_def_fields"_a, "message_body"_a, "message_length"_a); + "message_body"_a, "decoded_header"_a, "metadata"_a); } diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index 4a2a5e8bd..9c7be4c45 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -18,10 +18,13 @@ class PyMessageDatabase final : public MessageDatabase [[nodiscard]] const std::unordered_map& GetEnumsByIdDict() const { return enums_by_id; } [[nodiscard]] const std::unordered_map& GetEnumsByNameDict() const { return enums_by_name; } + std::unordered_map message_types{}; + private: void GenerateMappings() override; void UpdatePythonEnums(); + void UpdateMessageTypes(); std::unordered_map enums_by_id{}; std::unordered_map enums_by_name{}; @@ -31,4 +34,4 @@ class PyMessageDatabase final : public MessageDatabase using ConstPtr = std::shared_ptr; }; -} // namespace novatel::edie \ No newline at end of file +} // namespace novatel::edie diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 705036b35..971442aa9 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -18,9 +18,9 @@ struct PyGpsTime TIME_STATUS time_status{TIME_STATUS::UNKNOWN}; }; -struct PyDecodedMessage +struct PyMessageBody { - explicit PyDecodedMessage(std::vector message_, const MetaDataStruct& meta_, PyMessageDatabase::ConstPtr parent_db_); + explicit PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_); nb::dict& get_values() const; nb::dict& get_fields() const; nb::dict to_dict() const; @@ -31,20 +31,25 @@ struct PyDecodedMessage std::string repr() const; std::vector fields; - - // MetaDataStruct with all fields that are no longer relevant after decoding dropped. - uint16_t message_id; - uint32_t message_crc; - std::string message_name; - PyGpsTime time; - MEASUREMENT_SOURCE measurement_source{MEASUREMENT_SOURCE::PRIMARY}; - CONSTELLATION constellation{CONSTELLATION::UNKNOWN}; + std::vector get_field_names() const; private: + static std::vector base_fields; mutable nb::dict cached_values_; mutable nb::dict cached_fields_; PyMessageDatabase::ConstPtr parent_db_; }; -} // namespace novatel::edie::oem \ No newline at end of file +struct PyMessage +{ + nb::object message_body; + nb::object header; + + PyMessage(nb::object message_body_, nb::object header_) + : message_body(message_body_), header(header_) {} +}; + + + +} // namespace novatel::edie::oem diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index f102068bb..ce45d0d93 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -32,8 +32,11 @@ import argparse import os from binascii import hexlify +import time +import typing import novatel_edie as ne +from novatel_edie.messages import BESTPOSMessageBody from novatel_edie import STATUS @@ -93,6 +96,8 @@ def main(): encoder = ne.Encoder() filter = ne.Filter() + index = 0 + start = time.time() with open(f"{args.input_file}.{encode_format}", "wb") as converted_logs_stream: for framer_status, frame, meta in read_frames(args.input_file, framer): try: @@ -109,18 +114,23 @@ def main(): # Decode the log body. body = frame[meta.header_length:] - status, message = message_decoder.decode(body, meta) + status, message = message_decoder.decode(body, header, meta) status.raise_on_error("MessageDecoder.decode() failed") - # Re-encode the log and write it to the output file. - status, encoded_message = encoder.encode(header, message, meta, encode_format) - status.raise_on_error("Encoder.encode() failed") + index += 1 + if index > 100000: + break + + # # Re-encode the log and write it to the output file. + # status, encoded_message = encoder.encode(header, message, meta, encode_format) + # status.raise_on_error("Encoder.encode() failed") - converted_logs_stream.write(encoded_message.message) - logger.info( f"Encoded ({len(encoded_message.message)}): {format_frame(encoded_message.message, encode_format)}") + # converted_logs_stream.write(encoded_message.message) + # logger.info( f"Encoded ({len(encoded_message.message)}): {format_frame(encoded_message.message, encode_format)}") except ne.DecoderException as e: logger.warn(str(e)) - + end = time.time() + print(f"Time taken: {end - start}") if __name__ == "__main__": main() diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 6f35a060d..8d6375575 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -2,7 +2,6 @@ from .bindings import * - def default_json_db_path(): """Returns a context manager that yields the path to the default JSON database.""" return importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("messages_public.json")) From b93c158c0c43318f695206de0d84e868b005fc8c Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 23 Jan 2025 15:22:47 -0700 Subject: [PATCH 02/67] Change imports --- python/CMakeLists.txt | 2 +- python/bindings/bindings.cpp | 39 ++++++++++++++++------------ python/bindings/json_db_reader.cpp | 2 -- python/bindings/message_database.cpp | 23 +++++++++------- python/bindings/message_decoder.cpp | 4 +-- python/bindings/oem_enums.cpp | 15 ----------- python/bindings/py_database.hpp | 8 ++++-- python/notes | 1 + python/novatel_edie/__init__.py | 2 +- 9 files changed, 46 insertions(+), 50 deletions(-) delete mode 100644 python/bindings/oem_enums.cpp create mode 160000 python/notes diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 14383593c..0a6fe77d4 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -38,7 +38,7 @@ nanobind_add_module(python_bindings ${CMAKE_CURRENT_SOURCE_DIR}/bindings/message_decoder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/nexcept.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/oem_common.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/bindings/oem_enums.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/bindings/init_modules.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/parser.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/range_decompressor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/rxconfig_handler.cpp diff --git a/python/bindings/bindings.cpp b/python/bindings/bindings.cpp index 11a49931a..d3e2aa627 100644 --- a/python/bindings/bindings.cpp +++ b/python/bindings/bindings.cpp @@ -16,27 +16,32 @@ void init_novatel_framer(nb::module_&); void init_novatel_header_decoder(nb::module_&); void init_novatel_message_decoder(nb::module_&); void init_novatel_oem_enums(nb::module_&); +void init_novatel_oem_messages(nb::module_&); void init_novatel_parser(nb::module_&); void init_novatel_range_decompressor(nb::module_&); void init_novatel_rxconfig_handler(nb::module_&); NB_MODULE(bindings, m) { - init_common_common(m); - init_common_logger(m); - init_common_json_db_reader(m); - init_common_nexcept(m); - init_novatel_commander(m); - init_novatel_common(m); - init_novatel_encoder(m); - init_novatel_file_parser(m); - init_novatel_filter(m); - init_novatel_framer(m); - init_novatel_header_decoder(m); - init_novatel_message_decoder(m); - init_novatel_oem_enums(m); - init_novatel_parser(m); - init_novatel_range_decompressor(m); - init_novatel_rxconfig_handler(m); - init_common_message_database(m); + nb::module_ classes_mod = m.def_submodule("classes", "Classes of the module."); + init_common_common(classes_mod); + init_common_logger(classes_mod); + init_common_json_db_reader(classes_mod); + init_common_nexcept(classes_mod); + init_novatel_commander(classes_mod); + init_novatel_common(classes_mod); + init_novatel_encoder(classes_mod); + init_novatel_file_parser(classes_mod); + init_novatel_filter(classes_mod); + init_novatel_framer(classes_mod); + init_novatel_header_decoder(classes_mod); + init_novatel_message_decoder(classes_mod); + init_common_message_database(classes_mod); + init_novatel_parser(classes_mod); + init_novatel_rxconfig_handler(classes_mod); + init_novatel_range_decompressor(classes_mod); + nb::module_ messages_mod = m.def_submodule("messages", "NovAtel OEM message definitions."); + init_novatel_oem_messages(messages_mod); + nb::module_ enums_mod = m.def_submodule("enums", "Enumerations used by NovAtel OEM message fields."); + init_novatel_oem_enums(enums_mod); } diff --git a/python/bindings/json_db_reader.cpp b/python/bindings/json_db_reader.cpp index f3e95c01a..34cdba6fa 100644 --- a/python/bindings/json_db_reader.cpp +++ b/python/bindings/json_db_reader.cpp @@ -33,8 +33,6 @@ PyMessageDatabase::Ptr& MessageDbSingleton::get() { static PyMessageDatabase::Ptr json_db = nullptr; if (!json_db) { - nb::module_ m = nb::module_::import_("novatel_edie"); - nb::module_ messages_mod = m.def_submodule("messages", "Message types used by NovAtel OEM messages."); json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path())); } return json_db; diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index 5875d9cd8..dcbd9e250 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -181,7 +181,7 @@ void init_common_message_database(nb::module_& m) .def("get_enum_def", &PyMessageDatabase::GetEnumDefId, "enum_id"_a) .def("get_enum_def", &PyMessageDatabase::GetEnumDefName, "enum_name"_a) .def_prop_ro("enums", &PyMessageDatabase::GetEnumsByNameDict) - .def_ro("message_types", &PyMessageDatabase::message_types); + .def_prop_ro("messages", &PyMessageDatabase::GetMessagesByNameDict); } PyMessageDatabase::PyMessageDatabase() { @@ -234,19 +234,22 @@ inline void PyMessageDatabase::UpdatePythonEnums() void PyMessageDatabase::UpdateMessageTypes() { - nb::module_ messages_mod = nb::module_::import_("novatel_edie.messages"); - nb::handle py_type = nb::type(); + // clear existing definitions + messages_by_id.clear(); + messages_by_name.clear(); + + // get type constructor nb::object Type = nb::module_::import_("builtins").attr("type"); + + nb::handle py_type = nb::type(); nb::dict type_dict = nb::dict(); nb::tuple type_tuple = nb::make_tuple(py_type); + nb::object msg_def; for (const auto& message_def : MessageDefinitions()) { - nb::object msg_def = Type(message_def->name + "MessageBody", type_tuple, type_dict); - msg_def.attr("__module__") = "novatel_edie.messages"; - message_types[message_def->name] = msg_def; - messages_mod.attr((message_def->name + "MessageBody").c_str()) = msg_def; + msg_def = Type(message_def->name + "MessageBody", type_tuple, type_dict); + messages_by_id[message_def->_id.c_str()] = msg_def; + messages_by_name[message_def->name] = msg_def; } nb::object defaut_type = Type("UNKNOWNMessageBody", type_tuple, type_dict); - defaut_type.attr("__module__") = "novatel_edie.messages"; - message_types["UNKNOWN"] = defaut_type; - messages_mod.attr("UNKNOWNMessageBody") = defaut_type; + messages_by_name["UNKNOWN"] = defaut_type; } diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 2cffc0e3a..fb8cf8360 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -231,10 +231,10 @@ void init_novatel_message_decoder(nb::module_& m) PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); nb::handle body_pytype; try { - body_pytype = parent_db->message_types.at(metadata.MessageName()); + body_pytype = parent_db->GetMessagesByNameDict().at(metadata.MessageName()); } catch (const std::out_of_range& e) { - body_pytype = parent_db->message_types.at("UNKNOWN"); + body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); } bool valid = nb::type_check(body_pytype); nb::object body_pyinst = nb::inst_alloc(body_pytype); diff --git a/python/bindings/oem_enums.cpp b/python/bindings/oem_enums.cpp deleted file mode 100644 index 183f325f1..000000000 --- a/python/bindings/oem_enums.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "bindings_core.hpp" -#include "message_db_singleton.hpp" - -namespace nb = nanobind; -using namespace nb::literals; -using namespace novatel::edie; - -void init_novatel_oem_enums(nb::module_& m) -{ - nb::module_ enums_mod = m.def_submodule("enums", "Enumerations used by NovAtel OEM message fields."); - for (const auto& [name, enum_type] : MessageDbSingleton::get()->GetEnumsByNameDict()) // - { - enums_mod.attr(name.c_str()) = enum_type; - } -} diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index 9c7be4c45..0ba5e7f66 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -15,10 +15,11 @@ class PyMessageDatabase final : public MessageDatabase explicit PyMessageDatabase(const MessageDatabase& message_db) noexcept; explicit PyMessageDatabase(const MessageDatabase&& message_db) noexcept; - [[nodiscard]] const std::unordered_map& GetEnumsByIdDict() const { return enums_by_id; } + [[nodiscard]] const std::unordered_map& GetMessagesByNameDict() const { return messages_by_name; } + [[nodiscard]] const std::unordered_map& GetEnumsByIdDict() const { return enums_by_id; } [[nodiscard]] const std::unordered_map& GetEnumsByNameDict() const { return enums_by_name; } - std::unordered_map message_types{}; + private: @@ -26,6 +27,9 @@ class PyMessageDatabase final : public MessageDatabase void UpdatePythonEnums(); void UpdateMessageTypes(); + std::unordered_map messages_by_id{}; + std::unordered_map messages_by_name{}; + std::unordered_map enums_by_id{}; std::unordered_map enums_by_name{}; diff --git a/python/notes b/python/notes new file mode 160000 index 000000000..1bed21bbb --- /dev/null +++ b/python/notes @@ -0,0 +1 @@ +Subproject commit 1bed21bbbfa8dfa06d521815eedfe442b3b777a0 diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 8d6375575..12190be0e 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -1,6 +1,6 @@ import importlib_resources -from .bindings import * +from .bindings.classes import * def default_json_db_path(): """Returns a context manager that yields the path to the default JSON database.""" From eeca12eff786acef9b9afea03ccec6f790b1cd8b Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 24 Jan 2025 11:06:36 -0700 Subject: [PATCH 03/67] update binding methodology --- python/bindings/bindings.cpp | 34 ++++++++++++------------- python/bindings/init_modules.cpp | 21 +++++++++++++++ python/examples/converter_components.py | 3 +++ python/notes | 1 - python/novatel_edie/__init__.py | 2 +- python/novatel_edie/enums.py | 1 + python/novatel_edie/messages.py | 1 + 7 files changed, 44 insertions(+), 19 deletions(-) create mode 100644 python/bindings/init_modules.cpp delete mode 160000 python/notes create mode 100644 python/novatel_edie/enums.py create mode 100644 python/novatel_edie/messages.py diff --git a/python/bindings/bindings.cpp b/python/bindings/bindings.cpp index d3e2aa627..94916eb9d 100644 --- a/python/bindings/bindings.cpp +++ b/python/bindings/bindings.cpp @@ -23,23 +23,23 @@ void init_novatel_rxconfig_handler(nb::module_&); NB_MODULE(bindings, m) { - nb::module_ classes_mod = m.def_submodule("classes", "Classes of the module."); - init_common_common(classes_mod); - init_common_logger(classes_mod); - init_common_json_db_reader(classes_mod); - init_common_nexcept(classes_mod); - init_novatel_commander(classes_mod); - init_novatel_common(classes_mod); - init_novatel_encoder(classes_mod); - init_novatel_file_parser(classes_mod); - init_novatel_filter(classes_mod); - init_novatel_framer(classes_mod); - init_novatel_header_decoder(classes_mod); - init_novatel_message_decoder(classes_mod); - init_common_message_database(classes_mod); - init_novatel_parser(classes_mod); - init_novatel_rxconfig_handler(classes_mod); - init_novatel_range_decompressor(classes_mod); + //nb::module_ core_mod = m.def_submodule("core", "Core functionality of the module."); + init_common_common(m); + init_common_logger(m); + init_common_json_db_reader(m); + init_common_nexcept(m); + init_novatel_commander(m); + init_novatel_common(m); + init_novatel_encoder(m); + init_novatel_file_parser(m); + init_novatel_filter(m); + init_novatel_framer(m); + init_novatel_header_decoder(m); + init_novatel_message_decoder(m); + init_common_message_database(m); + init_novatel_parser(m); + init_novatel_rxconfig_handler(m); + init_novatel_range_decompressor(m); nb::module_ messages_mod = m.def_submodule("messages", "NovAtel OEM message definitions."); init_novatel_oem_messages(messages_mod); nb::module_ enums_mod = m.def_submodule("enums", "Enumerations used by NovAtel OEM message fields."); diff --git a/python/bindings/init_modules.cpp b/python/bindings/init_modules.cpp new file mode 100644 index 000000000..196e5450a --- /dev/null +++ b/python/bindings/init_modules.cpp @@ -0,0 +1,21 @@ +#include "bindings_core.hpp" +#include "message_db_singleton.hpp" + +namespace nb = nanobind; +using namespace nb::literals; +using namespace novatel::edie; + +void init_novatel_oem_enums(nb::module_& m) +{ + for (const auto& [name, enum_type] : MessageDbSingleton::get()->GetEnumsByNameDict()) // + { + m.attr(name.c_str()) = enum_type; + } +} + +void init_novatel_oem_messages(nb::module_& m) +{ + for (const auto& [name, message_type] : MessageDbSingleton::get()->GetMessagesByNameDict()) { + m.attr((name + "MessageBody").c_str()) = message_type; + } +} diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index ce45d0d93..5965adbdf 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -117,6 +117,9 @@ def main(): status, message = message_decoder.decode(body, header, meta) status.raise_on_error("MessageDecoder.decode() failed") + if isinstance(message.body, BESTPOSMessageBody): + print(message) + index += 1 if index > 100000: break diff --git a/python/notes b/python/notes deleted file mode 160000 index 1bed21bbb..000000000 --- a/python/notes +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1bed21bbbfa8dfa06d521815eedfe442b3b777a0 diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 12190be0e..678fa4ab0 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -1,6 +1,6 @@ import importlib_resources -from .bindings.classes import * +from .bindings import messages, enums, MESSAGE_SIZE_MAX, HEADER_FORMAT, ENCODE_FORMAT, STATUS, string_to_encode_format, pretty_version, Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException def default_json_db_path(): """Returns a context manager that yields the path to the default JSON database.""" diff --git a/python/novatel_edie/enums.py b/python/novatel_edie/enums.py new file mode 100644 index 000000000..50ec26f0a --- /dev/null +++ b/python/novatel_edie/enums.py @@ -0,0 +1 @@ +from .bindings.enums import * diff --git a/python/novatel_edie/messages.py b/python/novatel_edie/messages.py new file mode 100644 index 000000000..cc657ab9c --- /dev/null +++ b/python/novatel_edie/messages.py @@ -0,0 +1 @@ +from .bindings.messages import * From 83ca7ae068dcb0605afb5c24801acf6544155310 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 24 Jan 2025 13:26:07 -0700 Subject: [PATCH 04/67] Get to_dict working --- python/bindings/message_decoder.cpp | 10 ++++++++-- python/bindings/oem_common.cpp | 16 ++++++++++++++++ python/examples/converter_components.py | 2 ++ python/novatel_edie/__init__.py | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index fb8cf8360..c4b17da8f 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -203,13 +203,19 @@ void init_novatel_message_decoder(nb::module_& m) .def_prop_ro("_fields", &PyMessageBody::get_fields) .def("to_dict", &PyMessageBody::to_dict, "Convert the message and its sub-messages into a dict") .def("__getattr__", &PyMessageBody::getattr, "field_name"_a) - //.def("__repr__", &PyMessageBody::repr) + .def("__repr__", &PyMessageBody::repr) .def("__str__", &PyMessageBody::repr) .def("__dir__", &PyMessageBody::get_field_names); nb::class_(m, "Message") .def_ro("body", &PyMessage::message_body) - .def_ro("header", &PyMessage::header); + .def_ro("header", &PyMessage::header) + .def("to_dict", [](const PyMessage& self) { + nb::dict message_dict; + message_dict["header"] = self.header.attr("to_dict")(); + message_dict["body"] = self.message_body.attr("to_dict")(); + return message_dict; + }); nb::class_(m, "FieldContainer") .def_rw("value", &FieldContainer::fieldValue) diff --git a/python/bindings/oem_common.cpp b/python/bindings/oem_common.cpp index c30b54e38..d416769f5 100644 --- a/python/bindings/oem_common.cpp +++ b/python/bindings/oem_common.cpp @@ -121,6 +121,22 @@ void init_novatel_common(nb::module_& m) .def_rw("receiver_status", &oem::IntermediateHeader::uiReceiverStatus) .def_rw("message_definition_crc", &oem::IntermediateHeader::uiMessageDefinitionCrc) .def_rw("receiver_sw_version", &oem::IntermediateHeader::usReceiverSwVersion) + .def("to_dict", [](const oem::IntermediateHeader& self) { + nb::dict header_dict; + header_dict["message_id"] = self.usMessageId; + header_dict["message_type"] = self.ucMessageType; + header_dict["port_address"] = self.uiPortAddress; + header_dict["length"] = self.usLength; + header_dict["sequence"] = self.usSequence; + header_dict["idle_time"] = self.ucIdleTime; + header_dict["time_status"] = self.uiTimeStatus; + header_dict["week"] = self.usWeek; + header_dict["milliseconds"] = self.dMilliseconds; + header_dict["receiver_status"] = self.uiReceiverStatus; + header_dict["message_definition_crc"] = self.uiMessageDefinitionCrc; + header_dict["receiver_sw_version"] = self.usReceiverSwVersion; + return header_dict; + }) .def("__repr__", [](const nb::handle self) { auto& header = nb::cast(self); return nb::str("Header(message_id={!r}, message_type={!r}, port_address={!r}, length={!r}, sequence={!r}, " diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index 5965adbdf..a9735a09f 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -120,6 +120,8 @@ def main(): if isinstance(message.body, BESTPOSMessageBody): print(message) + if isinstance(message.body, ne.messages.RANGEMessageBody): + print(message) index += 1 if index > 100000: break diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 678fa4ab0..449b05704 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -1,6 +1,6 @@ import importlib_resources -from .bindings import messages, enums, MESSAGE_SIZE_MAX, HEADER_FORMAT, ENCODE_FORMAT, STATUS, string_to_encode_format, pretty_version, Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException +from .bindings import messages, enums, MESSAGE_SIZE_MAX, HEADER_FORMAT, ENCODE_FORMAT, STATUS, string_to_encode_format, pretty_version, Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException, Logging def default_json_db_path(): """Returns a context manager that yields the path to the default JSON database.""" From a08a601b239096713f3ea7882ad4829107e5c40f Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 24 Jan 2025 16:09:12 -0700 Subject: [PATCH 05/67] Field type working --- python/bindings/init_modules.cpp | 5 +-- python/bindings/message_database.cpp | 34 ++++++++++++++---- python/bindings/message_decoder.cpp | 47 ++++++++++++++++++------- python/bindings/py_database.hpp | 2 +- python/bindings/py_decoded_message.hpp | 3 +- python/examples/converter_components.py | 9 +++-- 6 files changed, 71 insertions(+), 29 deletions(-) diff --git a/python/bindings/init_modules.cpp b/python/bindings/init_modules.cpp index 196e5450a..cdb2499f2 100644 --- a/python/bindings/init_modules.cpp +++ b/python/bindings/init_modules.cpp @@ -15,7 +15,8 @@ void init_novatel_oem_enums(nb::module_& m) void init_novatel_oem_messages(nb::module_& m) { - for (const auto& [name, message_type] : MessageDbSingleton::get()->GetMessagesByNameDict()) { - m.attr((name + "MessageBody").c_str()) = message_type; + for (const auto& [name, message_type] : MessageDbSingleton::get()->GetMessagesByNameDict()) + { + m.attr(name.c_str()) = message_type; } } diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index dcbd9e250..a95274cbc 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -232,24 +232,44 @@ inline void PyMessageDatabase::UpdatePythonEnums() } } +void PyMessageDatabase::AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, nb::handle type_dict) { + for (const auto& field : fields) { + if (auto* field_array_field = dynamic_cast(field.get())) { + std::string field_name = base_name + "_" + field_array_field->name + "_Field"; + nb::object field_type = type_cons(field_name, type_tuple, type_dict); + messages_by_name[field_name] = field_type; + + AddFieldType(field_array_field->fields, base_name, type_cons, type_tuple, type_dict); + // Handle FieldArrayField case + // Add specific logic for FieldArrayField + } + return; + } +} + + void PyMessageDatabase::UpdateMessageTypes() { // clear existing definitions - messages_by_id.clear(); messages_by_name.clear(); // get type constructor nb::object Type = nb::module_::import_("builtins").attr("type"); - nb::handle py_type = nb::type(); + nb::handle py_type = nb::type(); + nb::handle py_body_type = nb::type(); nb::dict type_dict = nb::dict(); nb::tuple type_tuple = nb::make_tuple(py_type); - nb::object msg_def; + nb::tuple body_type_tuple = nb::make_tuple(py_body_type); for (const auto& message_def : MessageDefinitions()) { - msg_def = Type(message_def->name + "MessageBody", type_tuple, type_dict); - messages_by_id[message_def->_id.c_str()] = msg_def; + nb::object msg_def = Type(message_def->name, type_tuple, type_dict); messages_by_name[message_def->name] = msg_def; + nb::object msg_body_def = Type(message_def->name + "_Body", body_type_tuple, type_dict); + messages_by_name[message_def->name + "_Body"] = msg_body_def; + AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name + "_Body", Type, body_type_tuple, type_dict); } - nb::object defaut_type = Type("UNKNOWNMessageBody", type_tuple, type_dict); - messages_by_name["UNKNOWN"] = defaut_type; + nb::object default_msg_def = Type("UNKNOWN", type_tuple, type_dict); + messages_by_name["UNKNOWN"] = default_msg_def; + nb::object default_msg_body_def = Type("UNKNOWN_Body", body_type_tuple, type_dict); + messages_by_name["UNKNOWN_Body"] = default_msg_body_def; } diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index c4b17da8f..9febc21bc 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -17,7 +17,7 @@ using namespace novatel::edie::oem; NB_MAKE_OPAQUE(std::vector); -nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::ConstPtr& parent_db) +nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::ConstPtr& parent_db, std::string parent) { if (field.fieldDef->type == FIELD_TYPE::ENUM) { @@ -59,13 +59,25 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } std::vector sub_values; sub_values.reserve(message_field.size()); - for (const auto& f : message_field) { sub_values.push_back(convert_field(f, parent_db)); } + for (const auto& f : message_field) { sub_values.push_back(convert_field(f, parent_db, parent)); } return nb::cast(sub_values); } else { // This is an array element of a field array. - return nb::cast(PyMessageBody(message_field, parent_db)); + nb::handle field_ptype; + std::string field_name = parent + "_" + field.fieldDef->name + "_Field"; + try { + field_ptype = parent_db->GetMessagesByNameDict().at(field_name); + } catch (const std::out_of_range& e){ + throw std::runtime_error("Type for " + field_name + "Field not found in the JSON database"); + } + nb::object pyinst = nb::inst_alloc(field_ptype); + PyMessageBody* cinst = nb::inst_ptr(pyinst); + new (cinst) PyMessageBody(message_field, parent_db, field_name+ "Field"); + nb::inst_mark_ready(pyinst); + + return pyinst; } } else if (field.fieldDef->conversion == "%id") @@ -96,15 +108,15 @@ std::vector PyMessageBody::get_field_names() const return field_names; } -PyMessageBody::PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_) - : fields(std::move(message_)), parent_db_(std::move(parent_db_)) +PyMessageBody::PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) + : fields(std::move(message_)), parent_db_(std::move(parent_db_)), name(std::move(name_)) {} nb::dict& PyMessageBody::get_values() const { if (cached_values_.size() == 0) { - for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_); } + for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_, name); } } return cached_values_; } @@ -203,7 +215,7 @@ void init_novatel_message_decoder(nb::module_& m) .def_prop_ro("_fields", &PyMessageBody::get_fields) .def("to_dict", &PyMessageBody::to_dict, "Convert the message and its sub-messages into a dict") .def("__getattr__", &PyMessageBody::getattr, "field_name"_a) - .def("__repr__", &PyMessageBody::repr) + //.def("__repr__", &PyMessageBody::repr) .def("__str__", &PyMessageBody::repr) .def("__dir__", &PyMessageBody::get_field_names); @@ -235,20 +247,29 @@ void init_novatel_message_decoder(nb::module_& m) std::vector fields; STATUS status = decoder.Decode(reinterpret_cast(message_body.c_str()), fields, metadata); PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); + nb::handle message_pytype; nb::handle body_pytype; try { - body_pytype = parent_db->GetMessagesByNameDict().at(metadata.MessageName()); + message_pytype = parent_db->GetMessagesByNameDict().at(metadata.MessageName()); + body_pytype = parent_db->GetMessagesByNameDict().at(metadata.MessageName() + "_Body"); } catch (const std::out_of_range& e) { - body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + message_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN_Body"); } - bool valid = nb::type_check(body_pytype); + + nb::object body_pyinst = nb::inst_alloc(body_pytype); PyMessageBody* body_cinst = nb::inst_ptr(body_pyinst); - new (body_cinst) PyMessageBody(std::move(fields), parent_db); + new (body_cinst) PyMessageBody(std::move(fields), parent_db, metadata.MessageName() + "_Body"); nb::inst_mark_ready(body_pyinst); - PyMessage message = PyMessage(body_pyinst, header); - return nb::make_tuple(status, message); + + nb::object message_pyinst = nb::inst_alloc(message_pytype); + PyMessage* message_cinst = nb::inst_ptr(message_pyinst); + new (message_cinst) PyMessage(body_pyinst, header); + nb::inst_mark_ready(message_pyinst); + + return nb::make_tuple(status, message_pyinst); }, "message_body"_a, "decoded_header"_a, "metadata"_a); } diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index 0ba5e7f66..cd36b5563 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -26,8 +26,8 @@ class PyMessageDatabase final : public MessageDatabase void GenerateMappings() override; void UpdatePythonEnums(); void UpdateMessageTypes(); + void AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, nb::handle type_dict); - std::unordered_map messages_by_id{}; std::unordered_map messages_by_name{}; std::unordered_map enums_by_id{}; diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 971442aa9..1aa340b37 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -20,7 +20,8 @@ struct PyGpsTime struct PyMessageBody { - explicit PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_); + std::string name; + explicit PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name); nb::dict& get_values() const; nb::dict& get_fields() const; nb::dict to_dict() const; diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index a9735a09f..4344f3e1e 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -36,7 +36,7 @@ import typing import novatel_edie as ne -from novatel_edie.messages import BESTPOSMessageBody +from novatel_edie.messages import RANGE from novatel_edie import STATUS @@ -117,11 +117,10 @@ def main(): status, message = message_decoder.decode(body, header, meta) status.raise_on_error("MessageDecoder.decode() failed") - if isinstance(message.body, BESTPOSMessageBody): - print(message) + if isinstance(message, RANGE): + body = message.body + pass - if isinstance(message.body, ne.messages.RANGEMessageBody): - print(message) index += 1 if index > 100000: break From 4f39d9ab4f1cceb19493b68331457efe4421af09 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 08:05:34 -0700 Subject: [PATCH 06/67] Benchmark --- python/examples/converter_components.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index 4344f3e1e..8d3e2a39d 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -33,8 +33,6 @@ import os from binascii import hexlify import time -import typing - import novatel_edie as ne from novatel_edie.messages import RANGE from novatel_edie import STATUS @@ -117,9 +115,9 @@ def main(): status, message = message_decoder.decode(body, header, meta) status.raise_on_error("MessageDecoder.decode() failed") - if isinstance(message, RANGE): - body = message.body - pass + # if isinstance(message, RANGE): + # body = message.body + # pass index += 1 if index > 100000: From 496d1273da4d1071ef13f0390d1d5919f6a269a7 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 09:46:24 -0700 Subject: [PATCH 07/67] Update for tests --- python/bindings/message_decoder.cpp | 4 ++-- python/examples/converter_components.py | 6 +++--- python/novatel_edie/__init__.py | 12 +++++++++++- python/test/test_benchmark.py | 2 +- python/test/test_novatel_types.py | 2 +- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 9febc21bc..10ba79ef9 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -70,11 +70,11 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C try { field_ptype = parent_db->GetMessagesByNameDict().at(field_name); } catch (const std::out_of_range& e){ - throw std::runtime_error("Type for " + field_name + "Field not found in the JSON database"); + field_ptype = parent_db->GetMessagesByNameDict().at("UNKNOWN_Body"); } nb::object pyinst = nb::inst_alloc(field_ptype); PyMessageBody* cinst = nb::inst_ptr(pyinst); - new (cinst) PyMessageBody(message_field, parent_db, field_name+ "Field"); + new (cinst) PyMessageBody(message_field, parent_db, field_name); nb::inst_mark_ready(pyinst); return pyinst; diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index 8d3e2a39d..f4559b403 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -115,9 +115,9 @@ def main(): status, message = message_decoder.decode(body, header, meta) status.raise_on_error("MessageDecoder.decode() failed") - # if isinstance(message, RANGE): - # body = message.body - # pass + if isinstance(message, RANGE): + obs = message.body.obs + pass index += 1 if index > 100000: diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 449b05704..ca9e34e7b 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -1,6 +1,16 @@ import importlib_resources -from .bindings import messages, enums, MESSAGE_SIZE_MAX, HEADER_FORMAT, ENCODE_FORMAT, STATUS, string_to_encode_format, pretty_version, Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException, Logging +from .bindings import ( + messages, enums, + MESSAGE_SIZE_MAX, HEADER_FORMAT, ENCODE_FORMAT, STATUS, FIELD_TYPE, DATA_TYPE, + TIME_STATUS, MEASUREMENT_SOURCE, OEM4_BINARY_HEADER_LENGTH, MAX_ASCII_MESSAGE_LENGTH, + NMEA_SYNC_LENGTH, NMEA_CRC_LENGTH, OEM4_SHORT_BINARY_SYNC_LENGTH, OEM4_SHORT_BINARY_HEADER_LENGTH, + OEM4_BINARY_CRC_LENGTH, OEM4_SHORT_ASCII_SYNC_LENGTH, OEM4_ASCII_CRC_LENGTH, MAX_SHORT_ASCII_MESSAGE_LENGTH, + OEM4_BINARY_SYNC_LENGTH, MAX_BINARY_MESSAGE_LENGTH, OEM4_ASCII_SYNC_LENGTH, + string_to_encode_format, pretty_version, get_default_database, + MetaData, MessageData, Commander, JsonDbReader, BaseField, RangeDecompressor, + Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException, Logging, LogLevel, + Parser, RxConfigHandler, FileParser) def default_json_db_path(): """Returns a context manager that yields the path to the default JSON database.""" diff --git a/python/test/test_benchmark.py b/python/test/test_benchmark.py index 66909d037..e8f2fb1bd 100644 --- a/python/test/test_benchmark.py +++ b/python/test/test_benchmark.py @@ -59,7 +59,7 @@ def run(self, log, encode_format): failed_once = True break body = log[meta_data.header_length:] - status, message = self.message_decoder.decode(body, meta_data) + status, message = self.message_decoder.decode(body, header, meta_data) if status != STATUS.SUCCESS: print("Failed to decode message: ", status) failed_once = True diff --git a/python/test/test_novatel_types.py b/python/test/test_novatel_types.py index d61884a1c..9f5678e64 100644 --- a/python/test/test_novatel_types.py +++ b/python/test/test_novatel_types.py @@ -662,7 +662,7 @@ def test_simple_field_width_valid(helper): input = b"TRUE,0x63,227,56,2734,-3842,38283,54244,-4359,5293,79338432,-289834,2.54,5.44061788e+03" status, intermediate_format = helper.test_decode_ascii(helper.msg_def_fields, input) status.raise_on_error() - + assert intermediate_format.field0 is True # bool assert intermediate_format.field1 == 99 # uint8_t assert intermediate_format.field2 == 227 # uint8_t From 14714ea8944c90415348c00faeda21706ea6dc7f Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 10:11:44 -0700 Subject: [PATCH 08/67] Update encoder and tests --- python/bindings/encoder.cpp | 10 ++-- python/novatel_edie/__init__.py | 4 +- python/test/test_benchmark.py | 2 +- python/test/test_decode_encode.py | 82 +++++++++++++++---------------- 4 files changed, 51 insertions(+), 47 deletions(-) diff --git a/python/bindings/encoder.cpp b/python/bindings/encoder.cpp index 482ef331f..775d369b1 100644 --- a/python/bindings/encoder.cpp +++ b/python/bindings/encoder.cpp @@ -20,9 +20,11 @@ void init_novatel_encoder(nb::module_& m) .def_prop_ro("logger", [](const oem::Encoder& encoder) { return encoder.GetLogger(); }) .def( "encode", - [](oem::Encoder& encoder, const oem::IntermediateHeader& header, const oem::PyMessageBody& py_message, + [](oem::Encoder& encoder, const oem::PyMessage& py_message, const oem::MetaDataStruct& metadata, ENCODE_FORMAT format) { MessageDataStruct message_data; + oem::IntermediateHeader* header_cinst = nb::inst_ptr(py_message.header); + oem::PyMessageBody* body_cinst = nb::inst_ptr(py_message.message_body); if (format == ENCODE_FORMAT::JSON) { // Allocate more space for JSON messages. @@ -31,7 +33,7 @@ void init_novatel_encoder(nb::module_& m) uint8_t buffer[MESSAGE_SIZE_MAX * 3]; auto* buf_ptr = reinterpret_cast(&buffer); uint32_t buf_size = MESSAGE_SIZE_MAX * 3; - STATUS status = encoder.Encode(&buf_ptr, buf_size, header, py_message.fields, message_data, metadata, format); + STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, body_cinst->fields, message_data, metadata, format); return nb::make_tuple(status, oem::PyMessageData(message_data)); } else @@ -39,9 +41,9 @@ void init_novatel_encoder(nb::module_& m) uint8_t buffer[MESSAGE_SIZE_MAX]; auto buf_ptr = reinterpret_cast(&buffer); uint32_t buf_size = MESSAGE_SIZE_MAX; - STATUS status = encoder.Encode(&buf_ptr, buf_size, header, py_message.fields, message_data, metadata, format); + STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, body_cinst->fields, message_data, metadata, format); return nb::make_tuple(status, oem::PyMessageData(message_data)); } }, - "header"_a, "message"_a, "metadata"_a, "encode_format"_a); + "message"_a, "metadata"_a, "encode_format"_a); } diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index ca9e34e7b..068abe841 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -6,8 +6,10 @@ TIME_STATUS, MEASUREMENT_SOURCE, OEM4_BINARY_HEADER_LENGTH, MAX_ASCII_MESSAGE_LENGTH, NMEA_SYNC_LENGTH, NMEA_CRC_LENGTH, OEM4_SHORT_BINARY_SYNC_LENGTH, OEM4_SHORT_BINARY_HEADER_LENGTH, OEM4_BINARY_CRC_LENGTH, OEM4_SHORT_ASCII_SYNC_LENGTH, OEM4_ASCII_CRC_LENGTH, MAX_SHORT_ASCII_MESSAGE_LENGTH, - OEM4_BINARY_SYNC_LENGTH, MAX_BINARY_MESSAGE_LENGTH, OEM4_ASCII_SYNC_LENGTH, + OEM4_BINARY_SYNC_LENGTH, MAX_BINARY_MESSAGE_LENGTH, OEM4_ASCII_SYNC_LENGTH, OEM4_BINARY_SYNC1, OEM4_BINARY_SYNC2, + OEM4_BINARY_SYNC3, string_to_encode_format, pretty_version, get_default_database, + Oem4BinaryHeader, MetaData, MessageData, Commander, JsonDbReader, BaseField, RangeDecompressor, Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException, Logging, LogLevel, Parser, RxConfigHandler, FileParser) diff --git a/python/test/test_benchmark.py b/python/test/test_benchmark.py index e8f2fb1bd..b51070259 100644 --- a/python/test/test_benchmark.py +++ b/python/test/test_benchmark.py @@ -64,7 +64,7 @@ def run(self, log, encode_format): print("Failed to decode message: ", status) failed_once = True break - status, message_data = self.encoder.encode(header, message, meta_data, encode_format) + status, message_data = self.encoder.encode(message, meta_data, encode_format) if status != STATUS.SUCCESS: print("Failed to encode message: ", status) failed_once = True diff --git a/python/test/test_decode_encode.py b/python/test/test_decode_encode.py index 0b80292c6..f48828ade 100644 --- a/python/test/test_decode_encode.py +++ b/python/test/test_decode_encode.py @@ -66,11 +66,11 @@ def DecodeEncode(self, encode_format: ne.ENCODE_FORMAT, message_input: bytes, me return (Result.HEADER_DECODER_ERROR, None) if not return_message else (Result.HEADER_DECODER_ERROR, None, None) body = message_input[meta_data.header_length:] - status, message = self.message_decoder.decode(body, meta_data) + status, message = self.message_decoder.decode(body, header, meta_data) if status != STATUS.SUCCESS: return (Result.MESSAGE_DECODER_ERROR, None) if not return_message else (Result.MESSAGE_DECODER_ERROR, None, None) - status, message_data = self.encoder.encode(header, message, meta_data, encode_format) + status, message_data = self.encoder.encode(message, meta_data, encode_format) if status != STATUS.SUCCESS: return (Result.ENCODER_ERROR, None) if not return_message else (Result.ENCODER_ERROR, None, None) @@ -78,7 +78,7 @@ def DecodeEncode(self, encode_format: ne.ENCODE_FORMAT, message_input: bytes, me return Result.SUCCESS, message_data, message return Result.SUCCESS, message_data - + def TestDecodeEncode(self, format_, encoded_message): return self.DecodeEncode(format_, encoded_message, ne.MetaData())[0] @@ -107,29 +107,29 @@ def TestSameFormatCompare(self, format_: ne.ENCODE_FORMAT, expected_message_data return Result.MESSAGEDATA_COMPARISON_ERROR return Result.SUCCESS - + def TestConversion(self, format_: ENCODE_FORMAT, message_input: bytes, expected_message_data: ExpectedMessageData, expected_meta_data: ne.MetaData=None): test_meta_data = ne.MetaData() ret_code, test_message_data = self.DecodeEncode(format_, message_input, test_meta_data) if ret_code != Result.SUCCESS: return ret_code - + if expected_message_data.header != test_message_data.header: print(f"MessageData.header error (expected {len(expected_message_data.header)}, got {len(test_message_data.header)})") return Result.HEADER_LENGTH_ERROR - + if expected_message_data.message != test_message_data.message: print(f"MessageData.message error (expected {len(expected_message_data.message)}, got {len(test_message_data.message)})") return Result.LENGTH_ERROR - + if expected_meta_data is not None: if not compare_metadata(test_meta_data, expected_meta_data): return Result.METADATA_COMPARISON_ERROR - + if not compare_message_data(test_message_data, expected_message_data): return Result.MESSAGEDATA_COMPARISON_ERROR - + return Result.SUCCESS @@ -292,35 +292,35 @@ def test_ascii_log_roundtrip_gloephem(helper): ret_code, message_data, glo_ephemeris = helper.DecodeEncode(ENCODE_FORMAT.FLATTENED_BINARY, log, return_message=True) assert ret_code == Result.SUCCESS - assert glo_ephemeris.sloto == 51 - assert glo_ephemeris.freqo == 0 - assert glo_ephemeris.sat_type == 1 - assert glo_ephemeris.false_iod == 80 - assert glo_ephemeris.ephem_week == 2168 - assert glo_ephemeris.ephem_time == 161118000 - assert glo_ephemeris.time_offset == 10782 - assert glo_ephemeris.nt == 573 - assert glo_ephemeris.GLOEPHEMERIS_reserved == 0 - assert glo_ephemeris.GLOEPHEMERIS_reserved_9 == 0 - assert glo_ephemeris.issue == 95 - assert glo_ephemeris.broadcast_health == 0 - assert glo_ephemeris.pos_x == -2.3917966796875000e+07 - assert glo_ephemeris.pos_y == 4.8163881835937500e+06 - assert glo_ephemeris.pos_z == 7.4258510742187500e+06 - assert glo_ephemeris.vel_x == -1.0062713623046875e+03 - assert glo_ephemeris.vel_y == 1.8321990966796875e+02 - assert glo_ephemeris.vel_z == -3.3695755004882813e+03 - assert glo_ephemeris.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) - assert glo_ephemeris.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) - assert glo_ephemeris.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) - assert glo_ephemeris.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) - assert glo_ephemeris.delta_tau == 5.587935448e-09 - assert glo_ephemeris.gamma == approx(0.00000000000000000, abs=0.0000000000000001) - assert glo_ephemeris.tk == 84600 - assert glo_ephemeris.p == 3 - assert glo_ephemeris.ft == 2 - assert glo_ephemeris.age == 0 - assert glo_ephemeris.flags == 13 + assert glo_ephemeris.body.sloto == 51 + assert glo_ephemeris.body.freqo == 0 + assert glo_ephemeris.body.sat_type == 1 + assert glo_ephemeris.body.false_iod == 80 + assert glo_ephemeris.body.ephem_week == 2168 + assert glo_ephemeris.body.ephem_time == 161118000 + assert glo_ephemeris.body.time_offset == 10782 + assert glo_ephemeris.body.nt == 573 + assert glo_ephemeris.body.GLOEPHEMERIS_reserved == 0 + assert glo_ephemeris.body.GLOEPHEMERIS_reserved_9 == 0 + assert glo_ephemeris.body.issue == 95 + assert glo_ephemeris.body.broadcast_health == 0 + assert glo_ephemeris.body.pos_x == -2.3917966796875000e+07 + assert glo_ephemeris.body.pos_y == 4.8163881835937500e+06 + assert glo_ephemeris.body.pos_z == 7.4258510742187500e+06 + assert glo_ephemeris.body.vel_x == -1.0062713623046875e+03 + assert glo_ephemeris.body.vel_y == 1.8321990966796875e+02 + assert glo_ephemeris.body.vel_z == -3.3695755004882813e+03 + assert glo_ephemeris.body.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) + assert glo_ephemeris.body.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) + assert glo_ephemeris.body.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) + assert glo_ephemeris.body.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) + assert glo_ephemeris.body.delta_tau == 5.587935448e-09 + assert glo_ephemeris.body.gamma == approx(0.00000000000000000, abs=0.0000000000000001) + assert glo_ephemeris.body.tk == 84600 + assert glo_ephemeris.body.p == 3 + assert glo_ephemeris.body.ft == 2 + assert glo_ephemeris.body.age == 0 + assert glo_ephemeris.body.flags == 13 def test_ascii_log_roundtrip_loglist(helper): @@ -629,8 +629,8 @@ def test_flat_binary_log_decode_validmodelsb(helper): assert compare_binary_headers(test_log_header, log_header) # Check the populated parts of the log - assert len(validmodels.models) == 1 - for models in validmodels.models: + assert len(validmodels.body.models) == 1 + for models in validmodels.body.models: assert models.model == "FFNRNNCBN" @@ -664,7 +664,7 @@ def test_flat_binary_log_decode_version(helper): assert log_header.receiver_sw_version == 32768 # Check the populated parts of the log - versions = version.versions + versions = version.body.versions assert len(versions) == 4 # Check GPSCARD fields From 031e3f35a440ee5b68dc08f8ffd846bbd0233852 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 10:46:13 -0700 Subject: [PATCH 09/67] Fix most tests --- python/novatel_edie/__init__.py | 2 +- python/test/test_decode_encode.py | 192 +++++++++++++++--------------- 2 files changed, 100 insertions(+), 94 deletions(-) diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 068abe841..d2b623d37 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -9,7 +9,7 @@ OEM4_BINARY_SYNC_LENGTH, MAX_BINARY_MESSAGE_LENGTH, OEM4_ASCII_SYNC_LENGTH, OEM4_BINARY_SYNC1, OEM4_BINARY_SYNC2, OEM4_BINARY_SYNC3, string_to_encode_format, pretty_version, get_default_database, - Oem4BinaryHeader, + Oem4BinaryHeader, Oem4BinaryShortHeader, MetaData, MessageData, Commander, JsonDbReader, BaseField, RangeDecompressor, Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException, Logging, LogLevel, Parser, RxConfigHandler, FileParser) diff --git a/python/test/test_decode_encode.py b/python/test/test_decode_encode.py index f48828ade..230424a0b 100644 --- a/python/test/test_decode_encode.py +++ b/python/test/test_decode_encode.py @@ -502,27 +502,27 @@ def test_flat_binary_log_decode_bestpos(helper): assert log_header.receiver_sw_version == 32768 # SOL_COMPUTED SINGLE 51.15043711386 -114.03067767000 1097.2099 -17.0000 WGS84 0.9038 0.8534 1.7480 \"\" 0.000 0.000 35 30 30 30 00 06 39 33\r\n" - assert bestpos.solution_status == ne.enums.SolStatus.SOL_COMPUTED - assert bestpos.position_type == ne.enums.SolType.SINGLE - assert bestpos.latitude == 51.15043711386 - assert bestpos.longitude == -114.03067767000 - assert bestpos.orthometric_height == 1097.2099 - assert bestpos.undulation == -17.0000 - assert bestpos.datum_id == ne.enums.Datum.WGS84 - assert bestpos.latitude_std_dev == approx(0.9038, abs=1e-5) - assert bestpos.longitude_std_dev == approx(0.8534, abs=1e-5) - assert bestpos.height_std_dev == approx(1.7480, abs=1e-5) - # assert bestpos.base_id == "" - assert bestpos.diff_age == 0.000 - assert bestpos.solution_age == 0.000 - assert bestpos.num_svs == 35 - assert bestpos.num_soln_svs == 30 - assert bestpos.num_soln_L1_svs == 30 - assert bestpos.num_soln_multi_svs == 30 - assert bestpos.extended_solution_status2 == 0x00 - assert bestpos.ext_sol_stat == 0x06 - assert bestpos.gal_and_bds_mask == 0x39 - assert bestpos.gps_and_glo_mask == 0x33 + assert bestpos.body.solution_status == ne.enums.SolStatus.SOL_COMPUTED + assert bestpos.body.position_type == ne.enums.SolType.SINGLE + assert bestpos.body.latitude == 51.15043711386 + assert bestpos.body.longitude == -114.03067767000 + assert bestpos.body.orthometric_height == 1097.2099 + assert bestpos.body.undulation == -17.0000 + assert bestpos.body.datum_id == ne.enums.Datum.WGS84 + assert bestpos.body.latitude_std_dev == approx(0.9038, abs=1e-5) + assert bestpos.body.longitude_std_dev == approx(0.8534, abs=1e-5) + assert bestpos.body.height_std_dev == approx(1.7480, abs=1e-5) + # assert bestpos.body.base_id == "" + assert bestpos.body.diff_age == 0.000 + assert bestpos.body.solution_age == 0.000 + assert bestpos.body.num_svs == 35 + assert bestpos.body.num_soln_svs == 30 + assert bestpos.body.num_soln_L1_svs == 30 + assert bestpos.body.num_soln_multi_svs == 30 + assert bestpos.body.extended_solution_status2 == 0x00 + assert bestpos.body.ext_sol_stat == 0x06 + assert bestpos.body.gal_and_bds_mask == 0x39 + assert bestpos.body.gps_and_glo_mask == 0x33 def test_flat_binary_log_decode_gloephema(helper): @@ -548,35 +548,35 @@ def test_flat_binary_log_decode_gloephema(helper): assert log_header.msg_def_crc == 0x8d29 assert log_header.receiver_sw_version == 32768 - assert gloephemeris.sloto == 51 - assert gloephemeris.freqo == 0 - assert gloephemeris.sat_type == 1 - assert gloephemeris.false_iod == 80 - assert gloephemeris.ephem_week == 2168 - assert gloephemeris.ephem_time == 161118000 - assert gloephemeris.time_offset == 10782 - assert gloephemeris.nt == 573 - assert gloephemeris.GLOEPHEMERIS_reserved == 0 - assert gloephemeris.GLOEPHEMERIS_reserved_9 == 0 - assert gloephemeris.issue == 95 - assert gloephemeris.broadcast_health == 0 - assert gloephemeris.pos_x == -2.3917966796875000e+07 - assert gloephemeris.pos_y == 4.8163881835937500e+06 - assert gloephemeris.pos_z == 7.4258510742187500e+06 - assert gloephemeris.vel_x == -1.0062713623046875e+03 - assert gloephemeris.vel_y == 1.8321990966796875e+02 - assert gloephemeris.vel_z == -3.3695755004882813e+03 - assert gloephemeris.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) - assert gloephemeris.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) - assert gloephemeris.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) - assert gloephemeris.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) - assert gloephemeris.delta_tau == 5.587935448e-09 - assert gloephemeris.gamma == approx(0.00000000000000000, abs=0.0000000000000001) - assert gloephemeris.tk == 84600 - assert gloephemeris.p == 3 - assert gloephemeris.ft == 2 - assert gloephemeris.age == 0 - assert gloephemeris.flags == 13 + assert gloephemeris.body.sloto == 51 + assert gloephemeris.body.freqo == 0 + assert gloephemeris.body.sat_type == 1 + assert gloephemeris.body.false_iod == 80 + assert gloephemeris.body.ephem_week == 2168 + assert gloephemeris.body.ephem_time == 161118000 + assert gloephemeris.body.time_offset == 10782 + assert gloephemeris.body.nt == 573 + assert gloephemeris.body.GLOEPHEMERIS_reserved == 0 + assert gloephemeris.body.GLOEPHEMERIS_reserved_9 == 0 + assert gloephemeris.body.issue == 95 + assert gloephemeris.body.broadcast_health == 0 + assert gloephemeris.body.pos_x == -2.3917966796875000e+07 + assert gloephemeris.body.pos_y == 4.8163881835937500e+06 + assert gloephemeris.body.pos_z == 7.4258510742187500e+06 + assert gloephemeris.body.vel_x == -1.0062713623046875e+03 + assert gloephemeris.body.vel_y == 1.8321990966796875e+02 + assert gloephemeris.body.vel_z == -3.3695755004882813e+03 + assert gloephemeris.body.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) + assert gloephemeris.body.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) + assert gloephemeris.body.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) + assert gloephemeris.body.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) + assert gloephemeris.body.delta_tau == 5.587935448e-09 + assert gloephemeris.body.gamma == approx(0.00000000000000000, abs=0.0000000000000001) + assert gloephemeris.body.tk == 84600 + assert gloephemeris.body.p == 3 + assert gloephemeris.body.ft == 2 + assert gloephemeris.body.age == 0 + assert gloephemeris.body.flags == 13 def test_flat_binary_log_decode_portstatsb(helper): @@ -593,8 +593,8 @@ def test_flat_binary_log_decode_portstatsb(helper): assert compare_binary_headers(test_log_header, log_header) # Check the populated parts of the log - assert len(portstats.port_statistics) == 23 - for port_statistics, expected_port in zip(portstats.port_statistics, portstats_port_fields): + assert len(portstats.body.port_statistics) == 23 + for port_statistics, expected_port in zip(portstats.body.port_statistics, portstats_port_fields): assert port_statistics.port == expected_port @@ -612,8 +612,8 @@ def test_flat_binary_log_decode_psrdopb(helper): assert compare_binary_headers(test_log_header, log_header) # Check the populated parts of the log - assert len(psrdop.sats) == 35 - for sat, expected in zip(psrdop.sats, psrdop_sat_fields): + assert len(psrdop.body.sats) == 35 + for sat, expected in zip(psrdop.body.sats, psrdop_sat_fields): assert sat == expected @@ -733,7 +733,7 @@ def test_flat_binary_log_decode_versiona(helper): assert log_header.receiver_sw_version == 16248 # Check the populated parts of the log - versions = version.versions + versions = version.body.versions assert len(versions) == 8 # Check GPSCARD fields @@ -1027,8 +1027,8 @@ def ASSERT_SHORT_HEADER_EQ(short_header_, header_): def ASSERT_BESTSATS_EQ(message1, message2): - assert len(message1.satellite_entries) == len(message2.satellite_entries) - for satellite_entries1, satellite_entries2 in zip(message1.satellite_entries, message2.satellite_entries): + assert len(message1.body.satellite_entries) == len(message2.body.satellite_entries) + for satellite_entries1, satellite_entries2 in zip(message1.body.satellite_entries, message2.body.satellite_entries): assert satellite_entries1.system_type == satellite_entries2.system_type assert satellite_entries1.id.prn_or_slot == satellite_entries2.id.prn_or_slot assert satellite_entries1.id.frequency_channel == satellite_entries2.id.frequency_channel @@ -1037,36 +1037,38 @@ def ASSERT_BESTSATS_EQ(message1, message2): def ASSERT_BESTPOS_EQ(message1, message2): - assert message1.solution_status == message2.solution_status - assert message1.position_type == message2.position_type - assert message1.latitude == approx(message2.latitude, abs=1e-11) - assert message1.longitude == approx(message2.longitude, abs=1e-11) - if hasattr(message1, "orthometric_height") and hasattr(message2, "orthometric_height"): - assert message1.orthometric_height == approx(message2.orthometric_height, abs=1e-4) + body1 = message1.body + body2 = message2.body + assert body1.solution_status == body2.solution_status + assert body1.position_type == body2.position_type + assert body1.latitude == approx(body2.latitude, abs=1e-11) + assert body1.longitude == approx(body2.longitude, abs=1e-11) + if hasattr(body1, "orthometric_height") and hasattr(body2, "orthometric_height"): + assert body1.orthometric_height == approx(body2.orthometric_height, abs=1e-4) else: - assert message1.height == approx(message2.height, abs=1e-4) - assert message1.undulation == approx(message2.undulation, rel=1e-6) - assert message1.datum_id == message2.datum_id - assert message1.latitude_std_dev == approx(message2.latitude_std_dev, abs=1e-4) - assert message1.longitude_std_dev == approx(message2.longitude_std_dev, abs=1e-4) - assert message1.height_std_dev == approx(message2.height_std_dev, abs=1e-4) - assert message1.base_id == message2.base_id - assert message1.diff_age == approx(message2.diff_age, rel=1e-6) - assert message1.solution_age == approx(message2.solution_age, rel=1e-6) - assert message1.num_svs == message2.num_svs - assert message1.num_soln_svs == message2.num_soln_svs - assert message1.num_soln_L1_svs == message2.num_soln_L1_svs - assert message1.num_soln_multi_svs == message2.num_soln_multi_svs - if hasattr(message1, "extended_solution_status2") and hasattr(message2, "extended_solution_status2"): - assert message1.extended_solution_status2 == message2.extended_solution_status2 - assert message1.ext_sol_stat == message2.ext_sol_stat - assert message1.gal_and_bds_mask == message2.gal_and_bds_mask - assert message1.gps_and_glo_mask == message2.gps_and_glo_mask + assert body1.height == approx(body2.height, abs=1e-4) + assert body1.undulation == approx(body2.undulation, rel=1e-6) + assert body1.datum_id == body2.datum_id + assert body1.latitude_std_dev == approx(body2.latitude_std_dev, abs=1e-4) + assert body1.longitude_std_dev == approx(body2.longitude_std_dev, abs=1e-4) + assert body1.height_std_dev == approx(body2.height_std_dev, abs=1e-4) + assert body1.base_id == body2.base_id + assert body1.diff_age == approx(body2.diff_age, rel=1e-6) + assert body1.solution_age == approx(body2.solution_age, rel=1e-6) + assert body1.num_svs == body2.num_svs + assert body1.num_soln_svs == body2.num_soln_svs + assert body1.num_soln_L1_svs == body2.num_soln_L1_svs + assert body1.num_soln_multi_svs == body2.num_soln_multi_svs + if hasattr(body1, "extended_solution_status2") and hasattr(body2, "extended_solution_status2"): + assert body1.extended_solution_status2 == body2.extended_solution_status2 + assert body1.ext_sol_stat == body2.ext_sol_stat + assert body1.gal_and_bds_mask == body2.gal_and_bds_mask + assert body1.gps_and_glo_mask == body2.gps_and_glo_mask def ASSERT_LOGLIST_EQ(message1, message2): - assert len(message1.log_list) == len(message2.log_list) - for log_list1, log_list2 in zip(message1.log_list, message2.log_list): + assert len(message1.body.log_list) == len(message2.body.log_list) + for log_list1, log_list2 in zip(message1.body.log_list, message2.body.log_list): assert log_list1.log_port_address == log_list2.log_port_address assert log_list1.message_id == log_list2.message_id assert log_list1.trigger == log_list2.trigger @@ -1076,19 +1078,23 @@ def ASSERT_LOGLIST_EQ(message1, message2): def ASSERT_RAWGPSSUBFRAME_EQ(message1, message2): - assert message1.frame_decoder_number == message2.frame_decoder_number - assert message1.satellite_id == message2.satellite_id - assert message1.sub_frame_id == message2.sub_frame_id - assert message1.raw_sub_frame_data == message2.raw_sub_frame_data - assert message1.signal_channel_number == message2.signal_channel_number + body1 = message1.body + body2 = message2.body + assert body1.frame_decoder_number == body2.frame_decoder_number + assert body1.satellite_id == body2.satellite_id + assert body1.sub_frame_id == body2.sub_frame_id + assert body1.raw_sub_frame_data == body2.raw_sub_frame_data + assert body1.signal_channel_number == body2.signal_channel_number def ASSERT_TRACKSTAT_EQ(message1, message2): - assert message1.position_status == message2.position_status - assert message1.position_type == message2.position_type - assert message1.tracking_elevation_cutoff == message2.tracking_elevation_cutoff - assert len(message1.chan_status) == len(message2.chan_status) - for chan_status1, chan_status2 in zip(message1.chan_status, message2.chan_status): + body1 = message1.body + body2 = message2.body + assert body1.position_status == body2.position_status + assert body1.position_type == body2.position_type + assert body1.tracking_elevation_cutoff == body2.tracking_elevation_cutoff + assert len(body1.chan_status) == len(body2.chan_status) + for chan_status1, chan_status2 in zip(body1.chan_status, body2.chan_status): assert chan_status1.prn == chan_status2.prn assert chan_status1.freq == chan_status2.freq assert chan_status1.channel_status == chan_status2.channel_status From e940f22259d3cec2c13c7c8a0c34e2a0b3d6dbbe Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 11:10:03 -0700 Subject: [PATCH 10/67] Readd decode functions --- python/bindings/message_decoder.cpp | 26 ++++++++++++++++++++++++-- python/novatel_edie/__init__.py | 2 +- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 10ba79ef9..e076dc3fb 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -271,5 +271,27 @@ void init_novatel_message_decoder(nb::module_& m) return nb::make_tuple(status, message_pyinst); }, - "message_body"_a, "decoded_header"_a, "metadata"_a); -} + "message_body"_a, "decoded_header"_a, "metadata"_a) + .def( + "_decode_ascii", + [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body) { + std::vector fields; + // Copy to ensure that the byte string is zero-delimited + std::string body_str(message_body.c_str(), message_body.size()); + const char* data_ptr = body_str.c_str(); + STATUS status = static_cast(&decoder)->TestDecodeAscii(msg_def_fields, &data_ptr, fields); + return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN_Body")); + }, + "msg_def_fields"_a, "message_body"_a) + .def( + "_decode_binary", + [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body, + uint32_t message_length) { + std::vector fields; + const char* data_ptr = message_body.c_str(); + STATUS status = static_cast(&decoder)->TestDecodeBinary(msg_def_fields, reinterpret_cast(&data_ptr), + fields, message_length); + return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN_Body")); + }, + "msg_def_fields"_a, "message_body"_a, "message_length"_a); + } diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index d2b623d37..581a43be6 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -8,7 +8,7 @@ OEM4_BINARY_CRC_LENGTH, OEM4_SHORT_ASCII_SYNC_LENGTH, OEM4_ASCII_CRC_LENGTH, MAX_SHORT_ASCII_MESSAGE_LENGTH, OEM4_BINARY_SYNC_LENGTH, MAX_BINARY_MESSAGE_LENGTH, OEM4_ASCII_SYNC_LENGTH, OEM4_BINARY_SYNC1, OEM4_BINARY_SYNC2, OEM4_BINARY_SYNC3, - string_to_encode_format, pretty_version, get_default_database, + string_to_encode_format, pretty_version, get_default_database, MessageDatabase, Oem4BinaryHeader, Oem4BinaryShortHeader, MetaData, MessageData, Commander, JsonDbReader, BaseField, RangeDecompressor, Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException, Logging, LogLevel, From 5953d7f35216faf8c3f566041e2b91d108b3ff5a Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 13:05:54 -0700 Subject: [PATCH 11/67] Cleanup comments --- python/bindings/bindings.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/python/bindings/bindings.cpp b/python/bindings/bindings.cpp index 94916eb9d..639b7651c 100644 --- a/python/bindings/bindings.cpp +++ b/python/bindings/bindings.cpp @@ -23,7 +23,6 @@ void init_novatel_rxconfig_handler(nb::module_&); NB_MODULE(bindings, m) { - //nb::module_ core_mod = m.def_submodule("core", "Core functionality of the module."); init_common_common(m); init_common_logger(m); init_common_json_db_reader(m); From 41abc5929900a3038c0dabf93dfe2d52f8db681c Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 13:08:58 -0700 Subject: [PATCH 12/67] undo formatting --- python/bindings/json_db_reader.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/bindings/json_db_reader.cpp b/python/bindings/json_db_reader.cpp index 34cdba6fa..fbeb7c8e7 100644 --- a/python/bindings/json_db_reader.cpp +++ b/python/bindings/json_db_reader.cpp @@ -32,9 +32,7 @@ std::string default_json_db_path() PyMessageDatabase::Ptr& MessageDbSingleton::get() { static PyMessageDatabase::Ptr json_db = nullptr; - if (!json_db) { - json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path())); - } + if (!json_db) { json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path())); } return json_db; } From d47a552fb8e7fad37312cbb4f744dfba3a442904 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 13:37:25 -0700 Subject: [PATCH 13/67] Cleanup message database code --- python/bindings/message_database.cpp | 38 +++++++++++++++------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index a95274cbc..9828386f0 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -232,44 +232,46 @@ inline void PyMessageDatabase::UpdatePythonEnums() } } -void PyMessageDatabase::AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, nb::handle type_dict) { +void PyMessageDatabase::AddFieldType(std::vector> fields, std::string base_name, nb::handle type_constructor, + nb::handle type_tuple, nb::handle type_dict) +{ + // rescursively add field types for each field array element within the provided vector for (const auto& field : fields) { if (auto* field_array_field = dynamic_cast(field.get())) { std::string field_name = base_name + "_" + field_array_field->name + "_Field"; - nb::object field_type = type_cons(field_name, type_tuple, type_dict); + nb::object field_type = type_constructor(field_name, type_tuple, type_dict); messages_by_name[field_name] = field_type; - AddFieldType(field_array_field->fields, base_name, type_cons, type_tuple, type_dict); - // Handle FieldArrayField case - // Add specific logic for FieldArrayField + AddFieldType(field_array_field->fields, field_name, type_constructor, type_tuple, type_dict); } - return; } } - void PyMessageDatabase::UpdateMessageTypes() { // clear existing definitions messages_by_name.clear(); // get type constructor - nb::object Type = nb::module_::import_("builtins").attr("type"); - - nb::handle py_type = nb::type(); - nb::handle py_body_type = nb::type(); + nb::object type_constructor = nb::module_::import_("builtins").attr("type"); + // specify the python superclasses for the new message and message body types + nb::tuple type_tuple = nb::make_tuple(nb::type()); + nb::tuple body_type_tuple = nb::make_tuple(nb::type()); + // provide no additional attributes via `__dict__` nb::dict type_dict = nb::dict(); - nb::tuple type_tuple = nb::make_tuple(py_type); - nb::tuple body_type_tuple = nb::make_tuple(py_body_type); + + // add message and message body types for each message definition for (const auto& message_def : MessageDefinitions()) { - nb::object msg_def = Type(message_def->name, type_tuple, type_dict); + nb::object msg_def = type_constructor(message_def->name, type_tuple, type_dict); messages_by_name[message_def->name] = msg_def; - nb::object msg_body_def = Type(message_def->name + "_Body", body_type_tuple, type_dict); + nb::object msg_body_def = type_constructor(message_def->name + "_Body", body_type_tuple, type_dict); messages_by_name[message_def->name + "_Body"] = msg_body_def; - AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name + "_Body", Type, body_type_tuple, type_dict); + // add additional MessageBody types for each field array element within the message definition + AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name + "_Body", type_constructor, body_type_tuple, type_dict); } - nb::object default_msg_def = Type("UNKNOWN", type_tuple, type_dict); + // provide UNKNOWN types for undecodable messages + nb::object default_msg_def = type_constructor("UNKNOWN", type_tuple, type_dict); messages_by_name["UNKNOWN"] = default_msg_def; - nb::object default_msg_body_def = Type("UNKNOWN_Body", body_type_tuple, type_dict); + nb::object default_msg_body_def = type_constructor("UNKNOWN_Body", body_type_tuple, type_dict); messages_by_name["UNKNOWN_Body"] = default_msg_body_def; } From 1718230b7b67b877e5d5cbf6ab56e44c85ea9ea2 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 14:31:49 -0700 Subject: [PATCH 14/67] Imporve dir implementation --- python/bindings/message_decoder.cpp | 30 +++++++++++++------------ python/bindings/py_decoded_message.hpp | 2 -- python/examples/converter_components.py | 7 +++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index e076dc3fb..dc2b49489 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -95,19 +95,6 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } } -std::vector PyMessageBody::base_fields = { - "__class__", "__contains__", "__delattr__", "__dir__", "__doc__", "__eq__", "__format__", "__ge__", - "__getattr__", "__getattribute__", "__getitem__", "__getstate__", "__gt__", "__hash__", "__init__", "__init_subclass__", - "__le__", "__len__", "__lt__", "__module__", "__ne__", "__new__", "__reduce__", "__reduce_ex__", - "__repr__", "__setattr__", "__sizeof__", "__str__", "__subclasshook__"}; - -std::vector PyMessageBody::get_field_names() const -{ - std::vector field_names = base_fields; - for (const auto& [field_name, _] : get_fields()) { field_names.push_back(nb::cast(field_name)); } - return field_names; -} - PyMessageBody::PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) : fields(std::move(message_)), parent_db_(std::move(parent_db_)), name(std::move(name_)) {} @@ -217,7 +204,22 @@ void init_novatel_message_decoder(nb::module_& m) .def("__getattr__", &PyMessageBody::getattr, "field_name"_a) //.def("__repr__", &PyMessageBody::repr) .def("__str__", &PyMessageBody::repr) - .def("__dir__", &PyMessageBody::get_field_names); + .def("__dir__", [](nb::object self) { + // get required Python builtin functions + nb::module_ builtins = nb::module_::import_("builtins"); + nb::handle super = builtins.attr("super"); + nb::handle type = builtins.attr("type"); + // get base MessageBody type from concrete instance + nb::object body_type = (type(self).attr("__bases__"))[0]; + // retrieve base list based on superclass method + nb::object super_type = super(body_type, self); + nb::list base_list = nb::cast(super_type.attr("__dir__")()); + // add dynamic fields to the list + PyMessageBody* body = nb::inst_ptr(self); + for (const auto& [field_name, _] : body->get_fields()) { base_list.append(field_name); } + + return base_list; + }); nb::class_(m, "Message") .def_ro("body", &PyMessage::message_body) diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 1aa340b37..0893ea1a8 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -32,10 +32,8 @@ struct PyMessageBody std::string repr() const; std::vector fields; - std::vector get_field_names() const; private: - static std::vector base_fields; mutable nb::dict cached_values_; mutable nb::dict cached_fields_; diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index f4559b403..e71172c51 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -115,9 +115,10 @@ def main(): status, message = message_decoder.decode(body, header, meta) status.raise_on_error("MessageDecoder.decode() failed") - if isinstance(message, RANGE): - obs = message.body.obs - pass + # if isinstance(message, RANGE): + + # obs = message.body.obs + # pass index += 1 if index > 100000: From 079afd2e16a51139bab9f83bd14d2493b655e223 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 15:43:02 -0700 Subject: [PATCH 15/67] Finish example --- python/bindings/message_database.cpp | 7 ++++--- python/bindings/message_decoder.cpp | 25 +++++++++++++++---------- python/bindings/py_decoded_message.hpp | 10 ++++++++-- python/examples/converter_components.py | 25 ++++++++----------------- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index 9828386f0..4f5ddc560 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -237,13 +237,14 @@ void PyMessageDatabase::AddFieldType(std::vector> fie { // rescursively add field types for each field array element within the provided vector for (const auto& field : fields) { - if (auto* field_array_field = dynamic_cast(field.get())) { + if (field->type == FIELD_TYPE::FIELD_ARRAY) + { + auto* field_array_field = dynamic_cast(field.get()); std::string field_name = base_name + "_" + field_array_field->name + "_Field"; nb::object field_type = type_constructor(field_name, type_tuple, type_dict); messages_by_name[field_name] = field_type; - AddFieldType(field_array_field->fields, field_name, type_constructor, type_tuple, type_dict); - } + } } } diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index dc2b49489..343bdae9d 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -153,8 +153,7 @@ size_t PyMessageBody::len() const { return fields.size(); } std::string PyMessageBody::repr() const { std::stringstream repr; - repr << "MessageBody("; - //if (!message_name.empty()) { repr << message_name << " "; } + repr << name << "("; bool first = true; for (const auto& [field_name, value] : get_values()) { @@ -202,7 +201,7 @@ void init_novatel_message_decoder(nb::module_& m) .def_prop_ro("_fields", &PyMessageBody::get_fields) .def("to_dict", &PyMessageBody::to_dict, "Convert the message and its sub-messages into a dict") .def("__getattr__", &PyMessageBody::getattr, "field_name"_a) - //.def("__repr__", &PyMessageBody::repr) + .def("__repr__", &PyMessageBody::repr) .def("__str__", &PyMessageBody::repr) .def("__dir__", [](nb::object self) { // get required Python builtin functions @@ -210,9 +209,9 @@ void init_novatel_message_decoder(nb::module_& m) nb::handle super = builtins.attr("super"); nb::handle type = builtins.attr("type"); // get base MessageBody type from concrete instance - nb::object body_type = (type(self).attr("__bases__"))[0]; + nb::handle body_type = (type(self).attr("__bases__"))[0]; // retrieve base list based on superclass method - nb::object super_type = super(body_type, self); + nb::handle super_type = super(body_type, self); nb::list base_list = nb::cast(super_type.attr("__dir__")()); // add dynamic fields to the list PyMessageBody* body = nb::inst_ptr(self); @@ -224,12 +223,15 @@ void init_novatel_message_decoder(nb::module_& m) nb::class_(m, "Message") .def_ro("body", &PyMessage::message_body) .def_ro("header", &PyMessage::header) + .def_ro("name", &PyMessage::name) .def("to_dict", [](const PyMessage& self) { nb::dict message_dict; message_dict["header"] = self.header.attr("to_dict")(); message_dict["body"] = self.message_body.attr("to_dict")(); return message_dict; - }); + }) + .def("__repr__", &PyMessage::repr) + .def("__str__", &PyMessage::repr); nb::class_(m, "FieldContainer") .def_rw("value", &FieldContainer::fieldValue) @@ -251,9 +253,12 @@ void init_novatel_message_decoder(nb::module_& m) PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); nb::handle message_pytype; nb::handle body_pytype; + const std::string message_name = metadata.MessageName(); + const std::string message_body_name = metadata.MessageName() + "_Body"; + try { - message_pytype = parent_db->GetMessagesByNameDict().at(metadata.MessageName()); - body_pytype = parent_db->GetMessagesByNameDict().at(metadata.MessageName() + "_Body"); + message_pytype = parent_db->GetMessagesByNameDict().at(message_name); + body_pytype = parent_db->GetMessagesByNameDict().at(message_body_name); } catch (const std::out_of_range& e) { message_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); @@ -263,12 +268,12 @@ void init_novatel_message_decoder(nb::module_& m) nb::object body_pyinst = nb::inst_alloc(body_pytype); PyMessageBody* body_cinst = nb::inst_ptr(body_pyinst); - new (body_cinst) PyMessageBody(std::move(fields), parent_db, metadata.MessageName() + "_Body"); + new (body_cinst) PyMessageBody(std::move(fields), parent_db, message_body_name); nb::inst_mark_ready(body_pyinst); nb::object message_pyinst = nb::inst_alloc(message_pytype); PyMessage* message_cinst = nb::inst_ptr(message_pyinst); - new (message_cinst) PyMessage(body_pyinst, header); + new (message_cinst) PyMessage(body_pyinst, header, message_name); nb::inst_mark_ready(message_pyinst); return nb::make_tuple(status, message_pyinst); diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 0893ea1a8..a785e3d22 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -44,9 +44,15 @@ struct PyMessage { nb::object message_body; nb::object header; + std::string name; + + PyMessage(nb::object message_body_, nb::object header_, std::string name_) + : message_body(message_body_), header(header_), name(name_) {} - PyMessage(nb::object message_body_, nb::object header_) - : message_body(message_body_), header(header_) {} + std::string repr() const + { + return "<" + name + " Message>"; + } }; diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index e71172c51..a38cb1b09 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -94,8 +94,6 @@ def main(): encoder = ne.Encoder() filter = ne.Filter() - index = 0 - start = time.time() with open(f"{args.input_file}.{encode_format}", "wb") as converted_logs_stream: for framer_status, frame, meta in read_frames(args.input_file, framer): try: @@ -115,25 +113,18 @@ def main(): status, message = message_decoder.decode(body, header, meta) status.raise_on_error("MessageDecoder.decode() failed") - # if isinstance(message, RANGE): + # Get info from the log. + if isinstance(message, RANGE): + observations = message.body.obs - # obs = message.body.obs - # pass + # Re-encode the log and write it to the output file. + status, encoded_message = encoder.encode(message, meta, encode_format) + status.raise_on_error("Encoder.encode() failed") - index += 1 - if index > 100000: - break - - # # Re-encode the log and write it to the output file. - # status, encoded_message = encoder.encode(header, message, meta, encode_format) - # status.raise_on_error("Encoder.encode() failed") - - # converted_logs_stream.write(encoded_message.message) - # logger.info( f"Encoded ({len(encoded_message.message)}): {format_frame(encoded_message.message, encode_format)}") + converted_logs_stream.write(encoded_message.message) + logger.info( f"Encoded ({len(encoded_message.message)}): {format_frame(encoded_message.message, encode_format)}") except ne.DecoderException as e: logger.warn(str(e)) - end = time.time() - print(f"Time taken: {end - start}") if __name__ == "__main__": main() From ae42f7d6cc0d856aece41ebcec68eb50821f7ea7 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 27 Jan 2025 16:01:50 -0700 Subject: [PATCH 16/67] add submodules --- python/novatel_edie/__init__.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 581a43be6..eaa63fe17 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -2,17 +2,20 @@ from .bindings import ( messages, enums, - MESSAGE_SIZE_MAX, HEADER_FORMAT, ENCODE_FORMAT, STATUS, FIELD_TYPE, DATA_TYPE, - TIME_STATUS, MEASUREMENT_SOURCE, OEM4_BINARY_HEADER_LENGTH, MAX_ASCII_MESSAGE_LENGTH, - NMEA_SYNC_LENGTH, NMEA_CRC_LENGTH, OEM4_SHORT_BINARY_SYNC_LENGTH, OEM4_SHORT_BINARY_HEADER_LENGTH, - OEM4_BINARY_CRC_LENGTH, OEM4_SHORT_ASCII_SYNC_LENGTH, OEM4_ASCII_CRC_LENGTH, MAX_SHORT_ASCII_MESSAGE_LENGTH, - OEM4_BINARY_SYNC_LENGTH, MAX_BINARY_MESSAGE_LENGTH, OEM4_ASCII_SYNC_LENGTH, OEM4_BINARY_SYNC1, OEM4_BINARY_SYNC2, - OEM4_BINARY_SYNC3, - string_to_encode_format, pretty_version, get_default_database, MessageDatabase, - Oem4BinaryHeader, Oem4BinaryShortHeader, - MetaData, MessageData, Commander, JsonDbReader, BaseField, RangeDecompressor, - Framer, HeaderDecoder, MessageDecoder, Encoder, Filter, DecoderException, Logging, LogLevel, - Parser, RxConfigHandler, FileParser) + HEADER_FORMAT, ENCODE_FORMAT, STATUS, FIELD_TYPE, DATA_TYPE, TIME_STATUS, MEASUREMENT_SOURCE, + MESSAGE_SIZE_MAX, MAX_ASCII_MESSAGE_LENGTH, MAX_SHORT_ASCII_MESSAGE_LENGTH, MAX_BINARY_MESSAGE_LENGTH, + NMEA_SYNC_LENGTH, NMEA_CRC_LENGTH, + OEM4_BINARY_HEADER_LENGTH, OEM4_SHORT_BINARY_HEADER_LENGTH, + OEM4_BINARY_CRC_LENGTH, OEM4_ASCII_CRC_LENGTH, + OEM4_BINARY_SYNC_LENGTH, OEM4_SHORT_ASCII_SYNC_LENGTH, OEM4_ASCII_SYNC_LENGTH, OEM4_SHORT_BINARY_SYNC_LENGTH, + OEM4_BINARY_SYNC1, OEM4_BINARY_SYNC2, OEM4_BINARY_SYNC3, + string_to_encode_format, pretty_version, + JsonDbReader, MessageDatabase, get_default_database, + Oem4BinaryHeader, Oem4BinaryShortHeader, MetaData, MessageData, BaseField, + Framer, Filter, HeaderDecoder, MessageDecoder, DecoderException, Encoder, Commander, Parser, FileParser, + RangeDecompressor, RxConfigHandler, + Logging, LogLevel, +) def default_json_db_path(): """Returns a context manager that yields the path to the default JSON database.""" From 047de33cf4317e23e1ebb511b517c454a8512aca Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Tue, 28 Jan 2025 08:50:39 -0700 Subject: [PATCH 17/67] optimize field conversion --- python/bindings/message_decoder.cpp | 41 ++++++++++++++----------- python/examples/converter_components.py | 1 + 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 343bdae9d..ab6b286c5 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -39,7 +39,29 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C // Empty array return nb::list(); } - else if (message_field[0].fieldDef->type == field.fieldDef->type && message_field[0].fieldDef->name == field.fieldDef->name) + else if (field.fieldDef->type == FIELD_TYPE::FIELD_ARRAY) + { + // Field Array + std::vector sub_values; + sub_values.reserve(message_field.size()); + nb::handle field_ptype; + std::string field_name = parent + "_" + field.fieldDef->name + "_Field"; + try { + field_ptype = parent_db->GetMessagesByNameDict().at(field_name); + } catch (const std::out_of_range& e) { + field_ptype = parent_db->GetMessagesByNameDict().at("UNKNOWN_Body"); + } + for (const auto& subfield : message_field) { + nb::object pyinst = nb::inst_alloc(field_ptype); + PyMessageBody* cinst = nb::inst_ptr(pyinst); + const auto& message_subfield = std::get>(subfield.fieldValue); + new (cinst) PyMessageBody(message_subfield, parent_db, field_name); + nb::inst_mark_ready(pyinst); + sub_values.push_back(pyinst); + } + return nb::cast(sub_values); + } + else { // Fixed-length, variable-length and field arrays are stored as a field // with a list of sub-fields of the same type and name. @@ -62,23 +84,6 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C for (const auto& f : message_field) { sub_values.push_back(convert_field(f, parent_db, parent)); } return nb::cast(sub_values); } - else - { - // This is an array element of a field array. - nb::handle field_ptype; - std::string field_name = parent + "_" + field.fieldDef->name + "_Field"; - try { - field_ptype = parent_db->GetMessagesByNameDict().at(field_name); - } catch (const std::out_of_range& e){ - field_ptype = parent_db->GetMessagesByNameDict().at("UNKNOWN_Body"); - } - nb::object pyinst = nb::inst_alloc(field_ptype); - PyMessageBody* cinst = nb::inst_ptr(pyinst); - new (cinst) PyMessageBody(message_field, parent_db, field_name); - nb::inst_mark_ready(pyinst); - - return pyinst; - } } else if (field.fieldDef->conversion == "%id") { diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index a38cb1b09..c225d6d53 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -116,6 +116,7 @@ def main(): # Get info from the log. if isinstance(message, RANGE): observations = message.body.obs + pass # Re-encode the log and write it to the output file. status, encoded_message = encoder.encode(message, meta, encode_format) From 63c3f778b4d8fc58e111e79bc0daa857a2a92338 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Tue, 28 Jan 2025 08:57:52 -0700 Subject: [PATCH 18/67] Fix dir breaking change --- python/bindings/message_decoder.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index ab6b286c5..21669f0eb 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -216,8 +216,8 @@ void init_novatel_message_decoder(nb::module_& m) // get base MessageBody type from concrete instance nb::handle body_type = (type(self).attr("__bases__"))[0]; // retrieve base list based on superclass method - nb::handle super_type = super(body_type, self); - nb::list base_list = nb::cast(super_type.attr("__dir__")()); + nb::object super_obj = super(body_type, self); + nb::list base_list = nb::cast(super_obj.attr("__dir__")()); // add dynamic fields to the list PyMessageBody* body = nb::inst_ptr(self); for (const auto& [field_name, _] : body->get_fields()) { base_list.append(field_name); } From 3a0b4854fbc06ea5433a75d9b568746cff7f1d06 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Tue, 28 Jan 2025 10:14:33 -0700 Subject: [PATCH 19/67] Remove unused imports --- python/examples/converter_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index c225d6d53..8f30ce1c8 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -32,7 +32,7 @@ import argparse import os from binascii import hexlify -import time + import novatel_edie as ne from novatel_edie.messages import RANGE from novatel_edie import STATUS From 6f4262b565072f755d001b5e55e7a28feb675177 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Tue, 28 Jan 2025 10:18:07 -0700 Subject: [PATCH 20/67] Fix formatting --- python/bindings/message_decoder.cpp | 44 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 21669f0eb..ac4a2e6c8 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -284,26 +284,26 @@ void init_novatel_message_decoder(nb::module_& m) return nb::make_tuple(status, message_pyinst); }, "message_body"_a, "decoded_header"_a, "metadata"_a) - .def( - "_decode_ascii", - [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body) { - std::vector fields; - // Copy to ensure that the byte string is zero-delimited - std::string body_str(message_body.c_str(), message_body.size()); - const char* data_ptr = body_str.c_str(); - STATUS status = static_cast(&decoder)->TestDecodeAscii(msg_def_fields, &data_ptr, fields); - return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN_Body")); - }, - "msg_def_fields"_a, "message_body"_a) - .def( - "_decode_binary", - [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body, - uint32_t message_length) { - std::vector fields; - const char* data_ptr = message_body.c_str(); - STATUS status = static_cast(&decoder)->TestDecodeBinary(msg_def_fields, reinterpret_cast(&data_ptr), - fields, message_length); - return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN_Body")); - }, - "msg_def_fields"_a, "message_body"_a, "message_length"_a); + .def( + "_decode_ascii", + [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body) { + std::vector fields; + // Copy to ensure that the byte string is zero-delimited + std::string body_str(message_body.c_str(), message_body.size()); + const char* data_ptr = body_str.c_str(); + STATUS status = static_cast(&decoder)->TestDecodeAscii(msg_def_fields, &data_ptr, fields); + return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN_Body")); + }, + "msg_def_fields"_a, "message_body"_a) + .def( + "_decode_binary", + [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body, + uint32_t message_length) { + std::vector fields; + const char* data_ptr = message_body.c_str(); + STATUS status = static_cast(&decoder)->TestDecodeBinary(msg_def_fields, reinterpret_cast(&data_ptr), + fields, message_length); + return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN_Body")); + }, + "msg_def_fields"_a, "message_body"_a, "message_length"_a); } From 8d2f6a5e5daee7f2c2c02711c0216a545ebbb5fa Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 30 Jan 2025 15:48:23 -0700 Subject: [PATCH 21/67] doc updates --- python/bindings/message_decoder.cpp | 10 +++++----- python/bindings/py_database.hpp | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index ac4a2e6c8..7aa7251b6 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -21,6 +21,7 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C { if (field.fieldDef->type == FIELD_TYPE::ENUM) { + // Handle Enums const std::string& enumId = static_cast(field.fieldDef.get())->enumId; auto it = parent_db->GetEnumsByIdDict().find(enumId); if (it == parent_db->GetEnumsByIdDict().end()) @@ -36,12 +37,12 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C const auto& message_field = std::get>(field.fieldValue); if (message_field.empty()) { - // Empty array + // Handle Empty Arrays return nb::list(); } else if (field.fieldDef->type == FIELD_TYPE::FIELD_ARRAY) { - // Field Array + // Handle Field Arrays std::vector sub_values; sub_values.reserve(message_field.size()); nb::handle field_ptype; @@ -63,9 +64,7 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } else { - // Fixed-length, variable-length and field arrays are stored as a field - // with a list of sub-fields of the same type and name. - // This needs to be un-nested for the translated Python structure. + // Handle Fixed or Variable-Length Arrays if (field.fieldDef->conversion == "%s") { // The array is actually a string @@ -87,6 +86,7 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } else if (field.fieldDef->conversion == "%id") { + // Handle Satellite IDs const uint32_t temp_id = std::get(field.fieldValue); SatelliteId sat_id; sat_id.usPrnOrSlot = temp_id & 0x0000FFFF; diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index cd36b5563..e37cd5c7f 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -24,7 +24,22 @@ class PyMessageDatabase final : public MessageDatabase private: void GenerateMappings() override; + //----------------------------------------------------------------------- + //! \brief Creates Python Enums for each enum definition in the database. + //! + //! These classes are stored by ID in the enums_by_id map and by name in the enums_by_name map. + //----------------------------------------------------------------------- void UpdatePythonEnums(); + //----------------------------------------------------------------------- + //! \brief Creates Python types for each component of all message definitions in the database. + //! + //! A message named "MESSAGE" will be mapped to a Python class named "MESSAGE". + //! The message's body will be mapped to a class named "MESSAGE_Body". + //! A field of that body named "FIELD" will be mapped to a class named "MESSAGE_Body_FIELD_Field". + //! A subfield of that field named "SUBFIELD" will be mapped to a class named "MESSAGE_Body_FIELD_Field_SUBFIELD_Field". + //! + //! These classes are stored by name in the messages_by_name map. + //----------------------------------------------------------------------- void UpdateMessageTypes(); void AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, nb::handle type_dict); From f39da7a1552a3c1c81cbdebf62d0da94cb8d3e2e Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 30 Jan 2025 15:51:36 -0700 Subject: [PATCH 22/67] rename for consistency --- python/bindings/message_database.cpp | 12 ++++++------ python/bindings/py_database.hpp | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index 4f5ddc560..f32376d63 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -186,7 +186,7 @@ void init_common_message_database(nb::module_& m) PyMessageDatabase::PyMessageDatabase() { UpdatePythonEnums(); - UpdateMessageTypes(); + UpdatePythonMessageTypes(); } PyMessageDatabase::PyMessageDatabase(std::vector vMessageDefinitions_, @@ -194,24 +194,24 @@ PyMessageDatabase::PyMessageDatabase(std::vector vM : MessageDatabase(std::move(vMessageDefinitions_), std::move(vEnumDefinitions_)) { UpdatePythonEnums(); - UpdateMessageTypes(); + UpdatePythonMessageTypes(); } PyMessageDatabase::PyMessageDatabase(const MessageDatabase& message_db) noexcept : MessageDatabase(message_db) { UpdatePythonEnums(); - UpdateMessageTypes(); + UpdatePythonMessageTypes(); } PyMessageDatabase::PyMessageDatabase(const MessageDatabase&& message_db) noexcept : MessageDatabase(message_db) { UpdatePythonEnums(); - UpdateMessageTypes(); + UpdatePythonMessageTypes(); } void PyMessageDatabase::GenerateMappings() { MessageDatabase::GenerateMappings(); UpdatePythonEnums(); - UpdateMessageTypes(); + UpdatePythonMessageTypes(); } inline void PyMessageDatabase::UpdatePythonEnums() @@ -248,7 +248,7 @@ void PyMessageDatabase::AddFieldType(std::vector> fie } } -void PyMessageDatabase::UpdateMessageTypes() +void PyMessageDatabase::UpdatePythonMessageTypes() { // clear existing definitions messages_by_name.clear(); diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index e37cd5c7f..1e7a687a5 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -40,7 +40,7 @@ class PyMessageDatabase final : public MessageDatabase //! //! These classes are stored by name in the messages_by_name map. //----------------------------------------------------------------------- - void UpdateMessageTypes(); + void UpdatePythonMessageTypes(); void AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, nb::handle type_dict); std::unordered_map messages_by_name{}; From 2379072d3c1ea931424b0fb6b43b29400afea429 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 31 Jan 2025 08:35:15 -0700 Subject: [PATCH 23/67] use this-> --- python/bindings/message_decoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 7aa7251b6..bca943fd0 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -108,7 +108,7 @@ nb::dict& PyMessageBody::get_values() const { if (cached_values_.size() == 0) { - for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_, name); } + for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_, this->name); } } return cached_values_; } From e5fba46907a98fd15bec84056105a4d06d5f633b Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 29 Jan 2025 08:26:29 -0700 Subject: [PATCH 24/67] Add custom installer --- scripts/novatel_edie_installer.py | 108 ++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 scripts/novatel_edie_installer.py diff --git a/scripts/novatel_edie_installer.py b/scripts/novatel_edie_installer.py new file mode 100644 index 000000000..d02b8a252 --- /dev/null +++ b/scripts/novatel_edie_installer.py @@ -0,0 +1,108 @@ +import os +import shutil +import sys +from contextlib import contextmanager +import argparse +import subprocess +import toml + +@contextmanager +def open_library_clone(library: str, relative_filepaths: list[str]): + """Creates a clone of specified files in a library and moves into the temp directory. + + Args: + library: The name of the library to clone. + relative_filepaths: A list of relative filepaths to clone. + """ + library_path = os.path.join(sys.exec_prefix, 'Lib', 'site-packages', library) + temp_dir = os.path.join(os.getcwd(), 'temp_dir') + destination_directory = os.path.join(temp_dir, library) + if not os.path.exists(library_path): + raise FileNotFoundError(f"Source library {library} not found.") + + if not os.path.exists(destination_directory): + os.makedirs(destination_directory) + + cwd = os.getcwd() + + try: + for rel_path in relative_filepaths: + s = os.path.join(library_path, rel_path) + if not os.path.exists(s): + raise FileNotFoundError(f"File {rel_path} not found in source library {library}.") + d = os.path.join(destination_directory, rel_path) + shutil.copy2(s, d) + + os.chdir(temp_dir) + yield + finally: + os.chdir(cwd) + shutil.rmtree(temp_dir) + +def copy_file_to_cwd(file_path: str): + """Copies a file to the current working directory. + + Args: + file_path: The path of the file to copy. + """ + if not os.path.exists(file_path): + raise FileNotFoundError(f"File {file_path} not found.") + + destination_path = os.path.join(os.getcwd(), os.path.basename(file_path)) + shutil.copy2(file_path, destination_path) + +def install_package(): + """Installs a package in the current working directory. + + Args: + package_name: The name of the package to install. + """ + try: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '.']) + except subprocess.CalledProcessError as e: + print(f"Failed to install package: {e}") + +def create_pyproject_toml(package_name: str, version: str, description: str, author: str, author_email: str, license_str: str = 'MIT'): + """Creates a basic pyproject.toml file. + + Args: + package_name: The name of the package. + version: The version of the package. + description: A short description of the package. + author: The name of the author. + author_email: The email of the author. + """ + pyproject_content = { + 'tool': { + 'poetry': { + 'name': package_name, + 'version': version, + 'description': description, + 'authors': [f"{author} <{author_email}>"], + 'license': license_str + } + }, + 'build-system': { + 'requires': ['poetry-core>=1.0.0'], + 'build-backend': 'poetry.core.masonry.api' + } + } + + with open('pyproject.toml', 'w') as f: + toml.dump(pyproject_content, f) + + +def main(args): + library_name = 'novatel_edie' + relative_filepaths = ['__init__.py', 'bindings.pyd'] + with open_library_clone(library_name, relative_filepaths): + copy_file_to_cwd(args.database) + create_pyproject_toml('novatel_edie', '0.1.0', 'A package for decoding NovAtel EDIE messages', 'Author', 'MIT') + install_package() + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Process some files.') + parser.add_argument('database', type=str, help='The database to process') + main(parser.parse_args()) From 9e91d085c74f7a112bd8461f2a04a1ed32ed6e36 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 29 Jan 2025 16:27:07 -0700 Subject: [PATCH 25/67] Stubs almost working --- CMakeLists.txt | 2 +- .../{messages_public.json => database.json} | 0 .../__init__.py | 23 +++ .../installer.py | 133 ++++++++++++++++ .../stubgen.py | 150 ++++++++++++++++++ install_customizer/pyproject.toml | 9 ++ python/CMakeLists.txt | 2 +- python/bindings/json_db_reader.cpp | 4 +- python/bindings/message_database.cpp | 16 +- python/examples/converter_components.py | 8 +- python/novatel_edie/__init__.py | 6 +- scripts/novatel_edie_installer.py | 108 ------------- 12 files changed, 342 insertions(+), 119 deletions(-) rename database/{messages_public.json => database.json} (100%) create mode 100644 install_customizer/novatel_edie_install_customizer/__init__.py create mode 100644 install_customizer/novatel_edie_install_customizer/installer.py create mode 100644 install_customizer/novatel_edie_install_customizer/stubgen.py create mode 100644 install_customizer/pyproject.toml delete mode 100644 scripts/novatel_edie_installer.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d151bca2..b949f6b92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,5 +82,5 @@ if(BUILD_BENCHMARKS OR BUILD_EXAMPLES OR BUILD_TESTS) endif() install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -install(FILES database/messages_public.json DESTINATION ${CMAKE_INSTALL_DATADIR}/novatel_edie) +install(FILES database/database.json DESTINATION ${CMAKE_INSTALL_DATADIR}/novatel_edie) install_novatel_edie_cmake_config() diff --git a/database/messages_public.json b/database/database.json similarity index 100% rename from database/messages_public.json rename to database/database.json diff --git a/install_customizer/novatel_edie_install_customizer/__init__.py b/install_customizer/novatel_edie_install_customizer/__init__.py new file mode 100644 index 000000000..a104f7c5a --- /dev/null +++ b/install_customizer/novatel_edie_install_customizer/__init__.py @@ -0,0 +1,23 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" diff --git a/install_customizer/novatel_edie_install_customizer/installer.py b/install_customizer/novatel_edie_install_customizer/installer.py new file mode 100644 index 000000000..0412032be --- /dev/null +++ b/install_customizer/novatel_edie_install_customizer/installer.py @@ -0,0 +1,133 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +A module concerning the installation of a custom package. +""" + +import os +import shutil +import sys +from contextlib import contextmanager +import argparse +import subprocess + +from novatel_edie_install_customizer.stubgen import StubGenerator + +@contextmanager +def open_library_clone(library: str): + """Creates a clone of specified files in a library and moves into the temp directory. + + Args: + library: The name of the library to clone. + """ + packages_path = os.path.join(sys.exec_prefix, 'Lib', 'site-packages') + library_path = os.path.join(packages_path, library) + if not os.path.exists(library_path): + raise FileNotFoundError(f"Source library {library} not found.") + library_dist_info = [ + d for d in os.listdir(packages_path) + if d.startswith(library) and d.endswith('.dist-info')] + if not library_dist_info: + raise FileNotFoundError( + f"No .dist-info directory found for library {library}.") + library_dist_info = library_dist_info[0] + library_dist_info_path = os.path.join(packages_path, library_dist_info) + + temp_dir = os.path.join(os.getcwd(), 'temp_dir') + destination_dir = os.path.join(temp_dir, 'wheel') + + cwd = os.getcwd() + + try: + if not os.path.exists(destination_dir): + os.makedirs(destination_dir) + shutil.copytree( + library_path, + os.path.join(destination_dir, library), + dirs_exist_ok=True) + shutil.copytree( + library_dist_info_path, + os.path.join(destination_dir, library_dist_info), + dirs_exist_ok=True) + + os.chdir(temp_dir) + yield + finally: + os.chdir(cwd) + shutil.rmtree(temp_dir) + +def copy_file(file_path: str, destination_path: str = None): + """Copies a file to the current working directory. + + Args: + file_path: The path of the file to copy. + """ + if not os.path.exists(file_path): + raise FileNotFoundError(f"File {file_path} not found.") + + destination_path = (os.path.basename(file_path) + if destination_path is None + else destination_path) + + destination_path = os.path.join(os.getcwd(), destination_path) + shutil.copy2(file_path, destination_path) + +def install_package(): + """Installs a package in the current working directory. + + Args: + package_name: The name of the package to install. + """ + try: + subprocess.check_call([ + sys.executable, '-m', 'wheel', 'pack', './wheel']) + pass + except subprocess.CalledProcessError as e: + print(f"Failed to pack package: {e}") + wheel_files = [f for f in os.listdir('.') if f.endswith('.whl')] + for wheel_file in wheel_files: + try: + subprocess.check_call([ + sys.executable, '-m', 'pip', + 'install', wheel_file, '--force-reinstall']) + except subprocess.CalledProcessError as e: + print(f"Failed to install {wheel_file}: {e}") + + +def main(args): + library_name = 'novatel_edie' + with open_library_clone(library_name): + copy_file( + args.database, os.path.join('wheel', library_name, 'database.json')) + + database = StubGenerator(args.database) + database.write_stub_files(os.path.join('wheel', library_name)) + + install_package() + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Process some files.') + parser.add_argument('database', type=str, help='The database to process') + main(parser.parse_args()) diff --git a/install_customizer/novatel_edie_install_customizer/stubgen.py b/install_customizer/novatel_edie_install_customizer/stubgen.py new file mode 100644 index 000000000..883e244c4 --- /dev/null +++ b/install_customizer/novatel_edie_install_customizer/stubgen.py @@ -0,0 +1,150 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +A module concerning the generation of type hint stub files for the novatel_edie package. +""" +import os + +import nanobind + +import nanobind.stubgen +import novatel_edie as ne +from novatel_edie import MessageDatabase, JsonDbReader, DATA_TYPE, MessageDefinition + +class StubGenerator: + """Generator of type hint stub files for the novatel_edie package.""" + data_type_to_pytype = { + DATA_TYPE.INT: 'int', + DATA_TYPE.UINT: 'int', + DATA_TYPE.BOOL: 'bool', + DATA_TYPE.CHAR: 'int', + DATA_TYPE.UCHAR: 'int', + DATA_TYPE.SHORT: 'int', + DATA_TYPE.USHORT: 'int', + DATA_TYPE.LONG: 'int', + DATA_TYPE.ULONG: 'int', + DATA_TYPE.LONGLONG: 'int', + DATA_TYPE.ULONGLONG: 'int', + DATA_TYPE.FLOAT: 'float', + DATA_TYPE.DOUBLE: 'float', + DATA_TYPE.HEXBYTE: 'int', + DATA_TYPE.SATELLITEID: 'SatelliteId', + DATA_TYPE.UNKNOWN: 'bytes' + } + def __init__(self, database: str | MessageDatabase): + if isinstance(database, str): + database = JsonDbReader.load_file(database) + if not isinstance(database, MessageDatabase): + raise TypeError( + 'database must be a MessageDatabase object or a path to a JSON database file.') + self.database = database + + + def write_stub_files(self, file_path: str): + """Writes package stub files to the specified directory.""" + + file_specs = {'__init__.pyi': self.get_core_stubs(), + 'enums.pyi': self.get_enum_stubs(), + 'messages.pyi': self.get_message_stubs()} + + if not os.path.exists(file_path): + os.makedirs(file_path) + for file_name, file_contents in file_specs.items(): + with open(os.path.join(file_path, file_name), 'w') as f: + f.write(file_contents) + + def get_core_stubs(self) -> str: + stub_gen = nanobind.stubgen.StubGen(ne.bindings) + stub_gen.put(ne.bindings) + stub_str = stub_gen.get() + return stub_str + + def _convert_enum_def(self, enum_def) -> str: + """Create a type hint string for an enum definition.""" + type_hint = f'class {enum_def.__name__}(Enum):\n' + for enumeration in enum_def: + type_hint += f' {enumeration.name} = {enumeration.value}\n' + return type_hint + + def get_enum_stubs(self) -> str: + """Get a stub string for all enums in the database.""" + stub_str = ('from enum import Enum\n' + 'from typing import Any\n\n') + enums = list(self.database.enums.values()) + type_hints = [self._convert_enum_def(enum_def) for enum_def in enums] + type_hints_str = '\n'.join(type_hints) + stub_str += type_hints_str + return stub_str + + def _convert_message_def(self, message_def: MessageDefinition) -> str: + """Create a type hint string for a message definition.""" + type_hint = f'class {message_def.name}_Body(MessageBody):\n' + for field in message_def.fields[message_def.latest_message_crc]: + field_type = None + if field.type == ne.FIELD_TYPE.SIMPLE: + field_type = self.data_type_to_pytype.get(field.data_type.name) + if field.type == ne.FIELD_TYPE.ENUM: + field_type = field.enum_def.name if field.enum_def else 'Enum' + if (field.type in { + ne.FIELD_TYPE.FIXED_LENGTH_ARRAY, + ne.FIELD_TYPE.VARIABLE_LENGTH_ARRAY}): + if field.conversion == r'%s': + field_type = 'str' + else: + field_type = f'list[{self.data_type_to_pytype.get(field.data_type.name)}]' + + if not field_type: + field_type = 'Any' + + type_hint += ' @property\n' + type_hint += f' def {field.name}(self) -> {field_type}: ...\n\n' + + type_hint += f'class {message_def.name}(Message):\n' + type_hint += ' @property\n' + type_hint += ' def header(self) -> Header: ...\n\n' + type_hint += ' @property\n' + type_hint += f' def body(self) -> {message_def.name}_Body: ...\n\n' + + return type_hint + + def get_message_stubs(self) -> str: + """Get a stub string for all messages in the database.""" + stub_str = 'from typing import Any\n\n' + stub_str += 'from novatel_edie import Header, Message, MessageBody, SatelliteId\n' + stub_str += 'from novatel_edie.enums import *\n\n' + + message_list = [key for key in self.database.messages + if not key.endswith('_Body') + and not key.endswith('_Field') + and key != 'UNKNOWN'] + message_defs = [self.database.get_msg_def(msg) + for msg in message_list] + type_hints = [self._convert_message_def(msg_def) + for msg_def in message_defs] + type_hints_str = '\n'.join(type_hints) + stub_str += type_hints_str + + return stub_str + +if __name__ == '__main__': + StubGenerator(ne.JsonDbReader.load_file('database/database.json')).write_stub_files('stubs') diff --git a/install_customizer/pyproject.toml b/install_customizer/pyproject.toml new file mode 100644 index 000000000..d6667ed66 --- /dev/null +++ b/install_customizer/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "novatel_edie_install_customizer" +description = "Tool for creating a custom installation of novatel_edie package" +version = "3.2.27" +license = {text = "MIT"} +requires-python = ">=3.8" +dependencies = [ + "novatel_edie", +] diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 0a6fe77d4..cf42ca825 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -66,6 +66,6 @@ install(TARGETS python_bindings install(DIRECTORY novatel_edie/ DESTINATION ${PYTHON_INSTALL_DIR} COMPONENT python) -install(FILES ${CMAKE_SOURCE_DIR}/database/messages_public.json +install(FILES ${CMAKE_SOURCE_DIR}/database/database.json DESTINATION ${PYTHON_INSTALL_DIR} COMPONENT python) diff --git a/python/bindings/json_db_reader.cpp b/python/bindings/json_db_reader.cpp index fbeb7c8e7..2e8cd5ee8 100644 --- a/python/bindings/json_db_reader.cpp +++ b/python/bindings/json_db_reader.cpp @@ -13,11 +13,11 @@ std::string default_json_db_path() { // Does the following, but using nanobind: // import importlib_resources - // path_ctx = importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("messages_public.json")) + // path_ctx = importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("database.json")) // with path_ctx as path: // return path nb::object ir = nb::module_::import_("importlib_resources"); - nb::object path_ctx = ir.attr("as_file")(ir.attr("files")("novatel_edie").attr("joinpath")("messages_public.json")); + nb::object path_ctx = ir.attr("as_file")(ir.attr("files")("novatel_edie").attr("joinpath")("database.json")); auto py_path = path_ctx.attr("__enter__")(); if (!nb::cast(py_path.attr("is_file")())) { diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index f32376d63..b9568fa97 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -214,6 +214,16 @@ void PyMessageDatabase::GenerateMappings() UpdatePythonMessageTypes(); } +void cleanString(std::string& str) +{ + // Remove special characters from the string to make it a valid python attribute name + for (char& c : str) + { + if (c == '-' || c == '+' || c == '.' || c == '/' || c == '(' || c == ')') { c = '_'; } + } + if (isdigit(str[0])) { str = "_" + str; } +} + inline void PyMessageDatabase::UpdatePythonEnums() { nb::object IntEnum = nb::module_::import_("enum").attr("IntEnum"); @@ -223,7 +233,11 @@ inline void PyMessageDatabase::UpdatePythonEnums() { nb::dict values; const char* enum_name = enum_def->name.c_str(); - for (const auto& enumerator : enum_def->enumerators) { values[enumerator.name.c_str()] = enumerator.value; } + for (const auto& enumerator : enum_def->enumerators) { + std::string enumerator_name = enumerator.name; + cleanString(enumerator_name); + values[enumerator_name.c_str()] = enumerator.value; + } nb::object enum_type = IntEnum(enum_name, values); enum_type.attr("_name") = enum_name; enum_type.attr("_id") = enum_def->_id; diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index 8f30ce1c8..51e45cffc 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -34,8 +34,9 @@ from binascii import hexlify import novatel_edie as ne -from novatel_edie.messages import RANGE +from novatel_edie.messages import BESTPOS from novatel_edie import STATUS +from novatel_edie.enums import Datum def read_frames(input_file, framer): @@ -114,8 +115,9 @@ def main(): status.raise_on_error("MessageDecoder.decode() failed") # Get info from the log. - if isinstance(message, RANGE): - observations = message.body.obs + if isinstance(message, BESTPOS): + base_id = message.body.base_id + assert(isinstance(message.body.datum_id, Datum)) pass # Re-encode the log and write it to the output file. diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index eaa63fe17..4dd99afa0 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -11,12 +11,12 @@ OEM4_BINARY_SYNC1, OEM4_BINARY_SYNC2, OEM4_BINARY_SYNC3, string_to_encode_format, pretty_version, JsonDbReader, MessageDatabase, get_default_database, - Oem4BinaryHeader, Oem4BinaryShortHeader, MetaData, MessageData, BaseField, + Oem4BinaryHeader, Oem4BinaryShortHeader, MetaData, MessageData, MessageDefinition, BaseField, Framer, Filter, HeaderDecoder, MessageDecoder, DecoderException, Encoder, Commander, Parser, FileParser, RangeDecompressor, RxConfigHandler, - Logging, LogLevel, + Logging, LogLevel ) def default_json_db_path(): """Returns a context manager that yields the path to the default JSON database.""" - return importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("messages_public.json")) + return importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("database.json")) diff --git a/scripts/novatel_edie_installer.py b/scripts/novatel_edie_installer.py deleted file mode 100644 index d02b8a252..000000000 --- a/scripts/novatel_edie_installer.py +++ /dev/null @@ -1,108 +0,0 @@ -import os -import shutil -import sys -from contextlib import contextmanager -import argparse -import subprocess -import toml - -@contextmanager -def open_library_clone(library: str, relative_filepaths: list[str]): - """Creates a clone of specified files in a library and moves into the temp directory. - - Args: - library: The name of the library to clone. - relative_filepaths: A list of relative filepaths to clone. - """ - library_path = os.path.join(sys.exec_prefix, 'Lib', 'site-packages', library) - temp_dir = os.path.join(os.getcwd(), 'temp_dir') - destination_directory = os.path.join(temp_dir, library) - if not os.path.exists(library_path): - raise FileNotFoundError(f"Source library {library} not found.") - - if not os.path.exists(destination_directory): - os.makedirs(destination_directory) - - cwd = os.getcwd() - - try: - for rel_path in relative_filepaths: - s = os.path.join(library_path, rel_path) - if not os.path.exists(s): - raise FileNotFoundError(f"File {rel_path} not found in source library {library}.") - d = os.path.join(destination_directory, rel_path) - shutil.copy2(s, d) - - os.chdir(temp_dir) - yield - finally: - os.chdir(cwd) - shutil.rmtree(temp_dir) - -def copy_file_to_cwd(file_path: str): - """Copies a file to the current working directory. - - Args: - file_path: The path of the file to copy. - """ - if not os.path.exists(file_path): - raise FileNotFoundError(f"File {file_path} not found.") - - destination_path = os.path.join(os.getcwd(), os.path.basename(file_path)) - shutil.copy2(file_path, destination_path) - -def install_package(): - """Installs a package in the current working directory. - - Args: - package_name: The name of the package to install. - """ - try: - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '.']) - except subprocess.CalledProcessError as e: - print(f"Failed to install package: {e}") - -def create_pyproject_toml(package_name: str, version: str, description: str, author: str, author_email: str, license_str: str = 'MIT'): - """Creates a basic pyproject.toml file. - - Args: - package_name: The name of the package. - version: The version of the package. - description: A short description of the package. - author: The name of the author. - author_email: The email of the author. - """ - pyproject_content = { - 'tool': { - 'poetry': { - 'name': package_name, - 'version': version, - 'description': description, - 'authors': [f"{author} <{author_email}>"], - 'license': license_str - } - }, - 'build-system': { - 'requires': ['poetry-core>=1.0.0'], - 'build-backend': 'poetry.core.masonry.api' - } - } - - with open('pyproject.toml', 'w') as f: - toml.dump(pyproject_content, f) - - -def main(args): - library_name = 'novatel_edie' - relative_filepaths = ['__init__.py', 'bindings.pyd'] - with open_library_clone(library_name, relative_filepaths): - copy_file_to_cwd(args.database) - create_pyproject_toml('novatel_edie', '0.1.0', 'A package for decoding NovAtel EDIE messages', 'Author', 'MIT') - install_package() - - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Process some files.') - parser.add_argument('database', type=str, help='The database to process') - main(parser.parse_args()) From 15c40c5edebbba7cca6a9960875926242fa4ff06 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 30 Jan 2025 13:08:12 -0700 Subject: [PATCH 26/67] Working with subfields --- .../stubgen.py | 93 +++++++++++++------ python/examples/converter_components.py | 12 ++- 2 files changed, 74 insertions(+), 31 deletions(-) diff --git a/install_customizer/novatel_edie_install_customizer/stubgen.py b/install_customizer/novatel_edie_install_customizer/stubgen.py index 883e244c4..fa2225a4d 100644 --- a/install_customizer/novatel_edie_install_customizer/stubgen.py +++ b/install_customizer/novatel_edie_install_customizer/stubgen.py @@ -96,36 +96,77 @@ def get_enum_stubs(self) -> str: stub_str += type_hints_str return stub_str + def _get_field_pytype(self, field, parent: str) -> str: + """Get a type hint string for a base level field.""" + python_type = None + if field.type == ne.FIELD_TYPE.SIMPLE: + python_type = self.data_type_to_pytype.get(field.data_type.name) + if field.type == ne.FIELD_TYPE.ENUM: + python_type = field.enum_def.name if field.enum_def else 'Enum' + if field.type == ne.FIELD_TYPE.STRING: + python_type = 'str' + if (field.type in { + ne.FIELD_TYPE.FIXED_LENGTH_ARRAY, + ne.FIELD_TYPE.VARIABLE_LENGTH_ARRAY}): + if field.conversion == r'%s': + python_type = 'str' + else: + python_type = f'list[{self.data_type_to_pytype.get(field.data_type.name)}]' + if field.type == ne.FIELD_TYPE.FIELD_ARRAY: + python_type = f'list[{parent}_{field.name}_Field]' + if not python_type: + python_type = 'Any' + return python_type + + def _convert_field_array_def(self, field_array_def, parent: str) -> str: + """Convert a field array definition to a type hint string.""" + subfield_hints = [] + + # Create MessageBodyField type hint + name = f'{parent}_{field_array_def.name}_Field' + type_hint = f'class {name}(MessageBody):\n' + for field in field_array_def.fields: + python_type = self._get_field_pytype(field, parent) + type_hint += (' @property\n' + f' def {field.name}(self) -> {python_type}: ...\n\n') + # Create hints for any subfields + if field.type == ne.FIELD_TYPE.FIELD_ARRAY: + subfield_hints.append(self._convert_field_array_def(field, name)) + + # Combine all hints + hints = subfield_hints + [type_hint] + hint_str = '\n'.join(hints) + + return hint_str + def _convert_message_def(self, message_def: MessageDefinition) -> str: """Create a type hint string for a message definition.""" - type_hint = f'class {message_def.name}_Body(MessageBody):\n' + subfield_hints = [] + + # Create the Message type hint + message_hint = (f'class {message_def.name}(Message):\n' + ' @property\n' + ' def header(self) -> Header: ...\n\n' + ' @property\n' + f' def body(self) -> {message_def.name}_Body: ...\n\n') + + # Create the MessageBody type hint + body_name = f'{message_def.name}_Body' + body_hint = f'class {body_name}(MessageBody):\n' for field in message_def.fields[message_def.latest_message_crc]: - field_type = None - if field.type == ne.FIELD_TYPE.SIMPLE: - field_type = self.data_type_to_pytype.get(field.data_type.name) - if field.type == ne.FIELD_TYPE.ENUM: - field_type = field.enum_def.name if field.enum_def else 'Enum' - if (field.type in { - ne.FIELD_TYPE.FIXED_LENGTH_ARRAY, - ne.FIELD_TYPE.VARIABLE_LENGTH_ARRAY}): - if field.conversion == r'%s': - field_type = 'str' - else: - field_type = f'list[{self.data_type_to_pytype.get(field.data_type.name)}]' - - if not field_type: - field_type = 'Any' - - type_hint += ' @property\n' - type_hint += f' def {field.name}(self) -> {field_type}: ...\n\n' - - type_hint += f'class {message_def.name}(Message):\n' - type_hint += ' @property\n' - type_hint += ' def header(self) -> Header: ...\n\n' - type_hint += ' @property\n' - type_hint += f' def body(self) -> {message_def.name}_Body: ...\n\n' + python_type = self._get_field_pytype(field, body_name) + body_hint += ' @property\n' + body_hint += f' def {field.name}(self) -> {python_type}: ...\n\n' - return type_hint + # Create hints for any subfields + if field.type == ne.FIELD_TYPE.FIELD_ARRAY: + subfield_hints.append(self._convert_field_array_def(field, body_name)) + + # Combine all hints + hints = subfield_hints + [body_hint, message_hint] + hint_str = '\n'.join(hints) + + return hint_str def get_message_stubs(self) -> str: """Get a stub string for all messages in the database.""" diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index 51e45cffc..243006c89 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -34,7 +34,7 @@ from binascii import hexlify import novatel_edie as ne -from novatel_edie.messages import BESTPOS +from novatel_edie.messages import RANGE from novatel_edie import STATUS from novatel_edie.enums import Datum @@ -115,10 +115,12 @@ def main(): status.raise_on_error("MessageDecoder.decode() failed") # Get info from the log. - if isinstance(message, BESTPOS): - base_id = message.body.base_id - assert(isinstance(message.body.datum_id, Datum)) - pass + if isinstance(message, RANGE): + obs = message.body.obs + for ob in obs: + value = ob.psr + pass + # Re-encode the log and write it to the output file. status, encoded_message = encoder.encode(message, meta, encode_format) From 57908866a714582f15ad2de2a3091008087fa38e Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 11:46:29 -0700 Subject: [PATCH 27/67] Prototype --- pyproject.toml | 5 +- python/CMakeLists.txt | 17 ++ python/bindings/json_db_reader.cpp | 76 ++++++--- python/bindings/message_db_singleton.hpp | 2 +- .../novatel_edie_customizer}/__init__.py | 0 .../novatel_edie_customizer/__main__.py | 27 +++ .../novatel_edie_customizer/cli.py | 39 +++++ .../novatel_edie_customizer}/installer.py | 28 ++-- .../novatel_edie_customizer}/stubgen.py | 157 ++++++++++-------- .../novatel_edie_customizer}/pyproject.toml | 10 +- 10 files changed, 251 insertions(+), 110 deletions(-) rename {install_customizer/novatel_edie_install_customizer => python/novatel_edie_customizer/novatel_edie_customizer}/__init__.py (100%) create mode 100644 python/novatel_edie_customizer/novatel_edie_customizer/__main__.py create mode 100644 python/novatel_edie_customizer/novatel_edie_customizer/cli.py rename {install_customizer/novatel_edie_install_customizer => python/novatel_edie_customizer/novatel_edie_customizer}/installer.py (88%) rename {install_customizer/novatel_edie_install_customizer => python/novatel_edie_customizer/novatel_edie_customizer}/stubgen.py (57%) rename {install_customizer => python/novatel_edie_customizer}/pyproject.toml (51%) diff --git a/pyproject.toml b/pyproject.toml index 00f321000..614024bb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ requires = [ "scikit-build-core >=0.4.3", "nanobind >=2.0.0", "conan >=2.4.0", + "typer >=0.15" ] build-backend = "scikit_build_core.build" @@ -34,10 +35,10 @@ build-backend = "scikit_build_core.build" [tool.scikit-build] cmake.minimum-version = "3.24" -cmake.targets = ["python_bindings"] +cmake.targets = ["python_bindings", "bindings_stub", "dynamic_stubs"] install.components = ["python"] wheel.license-files = ["LICENSE"] -build-dir = "build/{wheel_tag}" +build-dir = "out/build/{wheel_tag}" # Build stable ABI wheels for CPython 3.12+ wheel.py-api = "cp312" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index cf42ca825..5690fab16 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -54,6 +54,20 @@ target_compile_definitions(python_bindings PRIVATE HAVE_SNPRINTF ) +nanobind_add_stub( + bindings_stub + MODULE bindings + OUTPUT stubs/bindings.pyi + PYTHON_PATH $ + DEPENDS bindings +) + +add_custom_target( + dynamic_stubs ALL + COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs + COMMENT "Generating dynamic stubs" +) + if(DEFINED SKBUILD_PROJECT_NAME) set(PYTHON_INSTALL_DIR ${SKBUILD_PROJECT_NAME}) elseif(NOT DEFINED PYTHON_INSTALL_DIR) @@ -69,3 +83,6 @@ install(DIRECTORY novatel_edie/ install(FILES ${CMAKE_SOURCE_DIR}/database/database.json DESTINATION ${PYTHON_INSTALL_DIR} COMPONENT python) +install (DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/stubs/ + DESTINATION ${PYTHON_INSTALL_DIR} + COMPONENT python) diff --git a/python/bindings/json_db_reader.cpp b/python/bindings/json_db_reader.cpp index 2e8cd5ee8..a52d17aeb 100644 --- a/python/bindings/json_db_reader.cpp +++ b/python/bindings/json_db_reader.cpp @@ -8,31 +8,65 @@ namespace nb = nanobind; using namespace nb::literals; using namespace novatel::edie; -namespace { -std::string default_json_db_path() -{ - // Does the following, but using nanobind: - // import importlib_resources - // path_ctx = importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("database.json")) - // with path_ctx as path: - // return path - nb::object ir = nb::module_::import_("importlib_resources"); - nb::object path_ctx = ir.attr("as_file")(ir.attr("files")("novatel_edie").attr("joinpath")("database.json")); - auto py_path = path_ctx.attr("__enter__")(); - if (!nb::cast(py_path.attr("is_file")())) - { - throw NExcept((std::string("Could not find the default JSON DB file at ") + nb::str(py_path).c_str()).c_str()); - } - auto path = nb::cast(nb::str(py_path)); - path_ctx.attr("__exit__")(nb::none(), nb::none(), nb::none()); - return path; -} -} // namespace +//namespace { +//std::string default_json_db_path() +//{ +// // Does the following, but using nanobind: +// // import importlib_resources +// // path_ctx = importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("database.json")) +// // with path_ctx as path: +// // return path +// nb::object import_lib_resources = nb::module_::import_("importlib_resources"); +// +// nb::object package_files = import_lib_resources.attr("files")("novatel_edie"); +// nb::object db_path = package_files.attr("joinpath")("database.json"); +// +// nb::object path_ctx = import_lib_resources.attr("as_file")(db_path); +// auto py_path = path_ctx.attr("__enter__")(); +// if (!nb::cast(py_path.attr("is_file")())) +// { +// throw NExcept((std::string("Could not find the default JSON DB file at ") + nb::str(py_path).c_str()).c_str()); +// } +// auto path = nb::cast(nb::str(py_path)); +// path_ctx.attr("__exit__")(nb::none(), nb::none(), nb::none()); +// return path; +//} +//} // namespace PyMessageDatabase::Ptr& MessageDbSingleton::get() { static PyMessageDatabase::Ptr json_db = nullptr; - if (!json_db) { json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path())); } + + // If the database has already been loaded, return it + if (json_db) { return json_db; } + + // Import necessary modules for locating the default database + nb::object builtins = nb::module_::import_("builtins"); + nb::object os_path = nb::module_::import_("os.path"); + nb::object import_lib_util = nb::module_::import_("importlib.util"); + + // Determine if the novatel_edie package exists within the current Python environment + nb::object module_spec = import_lib_util.attr("find_spec")("novatel_edie"); + bool module_exists = nb::cast(builtins.attr("bool")(import_lib_util.attr("find_spec")("novatel_edie"))); + // If the package does not exist, return an empty database + if (!module_exists) { + json_db = std::make_shared(MessageDatabase()); + return json_db; + } + + // Determine the path to the database file within the novatel_edie package + nb::object novatel_edie_path = os_path.attr("dirname")(module_spec.attr("origin")); + nb::object db_path = os_path.attr("join")(novatel_edie_path, "database.json"); + // If the database file does not exist, return an empty database + bool db_exists = nb::cast(os_path.attr("isfile")(db_path)); + if (!db_exists) + { + json_db = std::make_shared(MessageDatabase()); + return json_db; + } + // If the database file exists, load it + std::string default_json_db_path = nb::cast(db_path); + json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path)); return json_db; } diff --git a/python/bindings/message_db_singleton.hpp b/python/bindings/message_db_singleton.hpp index 017cbef60..d8420f103 100644 --- a/python/bindings/message_db_singleton.hpp +++ b/python/bindings/message_db_singleton.hpp @@ -12,4 +12,4 @@ class MessageDbSingleton [[nodiscard]] static PyMessageDatabase::Ptr& get(); }; -} // namespace novatel::edie \ No newline at end of file +} // namespace novatel::edie diff --git a/install_customizer/novatel_edie_install_customizer/__init__.py b/python/novatel_edie_customizer/novatel_edie_customizer/__init__.py similarity index 100% rename from install_customizer/novatel_edie_install_customizer/__init__.py rename to python/novatel_edie_customizer/novatel_edie_customizer/__init__.py diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py b/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py new file mode 100644 index 000000000..b2ccb37ed --- /dev/null +++ b/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py @@ -0,0 +1,27 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +if __name__ == "__main__": + from .cli import app + app() diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/cli.py b/python/novatel_edie_customizer/novatel_edie_customizer/cli.py new file mode 100644 index 000000000..541bf9578 --- /dev/null +++ b/python/novatel_edie_customizer/novatel_edie_customizer/cli.py @@ -0,0 +1,39 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import typer + +from .stubgen import generate_stubs +from .installer import install_custom + +app = typer.Typer() + +# sub_app = typer.Typer() +# sub_app.command()(generate_stubs) + +app.command()(generate_stubs) +app.command()(install_custom) + +if __name__ == "__main__": + app() diff --git a/install_customizer/novatel_edie_install_customizer/installer.py b/python/novatel_edie_customizer/novatel_edie_customizer/installer.py similarity index 88% rename from install_customizer/novatel_edie_install_customizer/installer.py rename to python/novatel_edie_customizer/novatel_edie_customizer/installer.py index 0412032be..f9ec80ac6 100644 --- a/install_customizer/novatel_edie_install_customizer/installer.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/installer.py @@ -28,10 +28,12 @@ import shutil import sys from contextlib import contextmanager -import argparse import subprocess -from novatel_edie_install_customizer.stubgen import StubGenerator +import typer +from typing_extensions import Annotated + +from novatel_edie_customizer.stubgen import StubGenerator @contextmanager def open_library_clone(library: str): @@ -113,21 +115,23 @@ def install_package(): except subprocess.CalledProcessError as e: print(f"Failed to install {wheel_file}: {e}") +def install_custom( + database: Annotated[ + str, + typer.Argument(help='A path to a database file.') + ]): + """Generate type hint stub files for a provided database. -def main(args): + Args: + database: A path to a database file. + """ + database = os.path.abspath(database) library_name = 'novatel_edie' with open_library_clone(library_name): copy_file( - args.database, os.path.join('wheel', library_name, 'database.json')) + database, os.path.join('wheel', library_name, 'database.json')) - database = StubGenerator(args.database) + database = StubGenerator(database) database.write_stub_files(os.path.join('wheel', library_name)) install_package() - - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Process some files.') - parser.add_argument('database', type=str, help='The database to process') - main(parser.parse_args()) diff --git a/install_customizer/novatel_edie_install_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py similarity index 57% rename from install_customizer/novatel_edie_install_customizer/stubgen.py rename to python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py index fa2225a4d..3fb5e4d33 100644 --- a/install_customizer/novatel_edie_install_customizer/stubgen.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py @@ -24,48 +24,47 @@ A module concerning the generation of type hint stub files for the novatel_edie package. """ import os +import json +import re -import nanobind - -import nanobind.stubgen -import novatel_edie as ne -from novatel_edie import MessageDatabase, JsonDbReader, DATA_TYPE, MessageDefinition +import typer +from typing_extensions import Annotated class StubGenerator: """Generator of type hint stub files for the novatel_edie package.""" data_type_to_pytype = { - DATA_TYPE.INT: 'int', - DATA_TYPE.UINT: 'int', - DATA_TYPE.BOOL: 'bool', - DATA_TYPE.CHAR: 'int', - DATA_TYPE.UCHAR: 'int', - DATA_TYPE.SHORT: 'int', - DATA_TYPE.USHORT: 'int', - DATA_TYPE.LONG: 'int', - DATA_TYPE.ULONG: 'int', - DATA_TYPE.LONGLONG: 'int', - DATA_TYPE.ULONGLONG: 'int', - DATA_TYPE.FLOAT: 'float', - DATA_TYPE.DOUBLE: 'float', - DATA_TYPE.HEXBYTE: 'int', - DATA_TYPE.SATELLITEID: 'SatelliteId', - DATA_TYPE.UNKNOWN: 'bytes' + 'INT': 'int', + 'UNIT': 'int', + 'BOOL': 'bool', + 'CHAR': 'int', + 'UCHAR': 'int', + 'SHORT': 'int', + 'USHORT': 'int', + 'LONG': 'int', + 'ULONG': 'int', + 'LONGLONG': 'int', + 'ULONGLONG': 'int', + 'FLOAT': 'float', + 'DOUBLE': 'float', + 'HEXBYTE': 'int', + 'SATELLITEID': 'SatelliteId', + 'UNKNOWN': 'bytes' } - def __init__(self, database: str | MessageDatabase): + def __init__(self, database: str | dict): if isinstance(database, str): - database = JsonDbReader.load_file(database) - if not isinstance(database, MessageDatabase): + with open(database, 'r') as f: + database = json.load(f) + if not isinstance(database, dict): raise TypeError( - 'database must be a MessageDatabase object or a path to a JSON database file.') + 'database must be a dict or a path to a JSON database file.') self.database = database - + self.database['enums_by_id'] = {enum['_id']: enum for enum in database['enums']} def write_stub_files(self, file_path: str): """Writes package stub files to the specified directory.""" - - file_specs = {'__init__.pyi': self.get_core_stubs(), - 'enums.pyi': self.get_enum_stubs(), - 'messages.pyi': self.get_message_stubs()} + file_specs = { + 'enums.pyi': self.get_enum_stubs(), + 'messages.pyi': self.get_message_stubs()} if not os.path.exists(file_path): os.makedirs(file_path) @@ -73,24 +72,22 @@ def write_stub_files(self, file_path: str): with open(os.path.join(file_path, file_name), 'w') as f: f.write(file_contents) - def get_core_stubs(self) -> str: - stub_gen = nanobind.stubgen.StubGen(ne.bindings) - stub_gen.put(ne.bindings) - stub_str = stub_gen.get() - return stub_str - def _convert_enum_def(self, enum_def) -> str: """Create a type hint string for an enum definition.""" - type_hint = f'class {enum_def.__name__}(Enum):\n' - for enumeration in enum_def: - type_hint += f' {enumeration.name} = {enumeration.value}\n' + type_hint = f'class {enum_def['name']}(Enum):\n' + for enumerator in enum_def['enumerators']: + name = enumerator['name'] + name = re.sub(r'[)(\-+./\\]', '_', name) + if name[0].isdigit(): + name = f'_{name}' + type_hint += f' {name} = {enumerator['value']}\n' return type_hint def get_enum_stubs(self) -> str: """Get a stub string for all enums in the database.""" stub_str = ('from enum import Enum\n' 'from typing import Any\n\n') - enums = list(self.database.enums.values()) + enums = self.database['enums'] type_hints = [self._convert_enum_def(enum_def) for enum_def in enums] type_hints_str = '\n'.join(type_hints) stub_str += type_hints_str @@ -99,21 +96,22 @@ def get_enum_stubs(self) -> str: def _get_field_pytype(self, field, parent: str) -> str: """Get a type hint string for a base level field.""" python_type = None - if field.type == ne.FIELD_TYPE.SIMPLE: - python_type = self.data_type_to_pytype.get(field.data_type.name) - if field.type == ne.FIELD_TYPE.ENUM: - python_type = field.enum_def.name if field.enum_def else 'Enum' - if field.type == ne.FIELD_TYPE.STRING: + if field['type'] == 'SIMPLE': + python_type = self.data_type_to_pytype.get(field['dataType']['name']) + if field['type'] == 'ENUM': + enum_def = self.database['enums_by_id'].get(field['enumID']) + python_type = enum_def['name'] if enum_def else 'Any' + if field['type'] == 'STRING': python_type = 'str' - if (field.type in { - ne.FIELD_TYPE.FIXED_LENGTH_ARRAY, - ne.FIELD_TYPE.VARIABLE_LENGTH_ARRAY}): - if field.conversion == r'%s': + if (field['type'] in { + 'FIXED_LENGTH_ARRAY', + 'VARIABLE_LENGTH_ARRAY'}): + if field['conversionString'] == r'%s': python_type = 'str' else: - python_type = f'list[{self.data_type_to_pytype.get(field.data_type.name)}]' - if field.type == ne.FIELD_TYPE.FIELD_ARRAY: - python_type = f'list[{parent}_{field.name}_Field]' + python_type = f'list[{self.data_type_to_pytype.get(field['dataType']['name'])}]' + if field['type'] == 'FIELD_ARRAY': + python_type = f'list[{parent}_{field['name']}_Field]' if not python_type: python_type = 'Any' return python_type @@ -123,14 +121,14 @@ def _convert_field_array_def(self, field_array_def, parent: str) -> str: subfield_hints = [] # Create MessageBodyField type hint - name = f'{parent}_{field_array_def.name}_Field' + name = f'{parent}_{field_array_def['name']}_Field' type_hint = f'class {name}(MessageBody):\n' - for field in field_array_def.fields: + for field in field_array_def['fields']: python_type = self._get_field_pytype(field, parent) type_hint += (' @property\n' - f' def {field.name}(self) -> {python_type}: ...\n\n') + f' def {field['name']}(self) -> {python_type}: ...\n\n') # Create hints for any subfields - if field.type == ne.FIELD_TYPE.FIELD_ARRAY: + if field['type'] == 'FIELD_ARRAY': subfield_hints.append(self._convert_field_array_def(field, name)) # Combine all hints @@ -139,27 +137,30 @@ def _convert_field_array_def(self, field_array_def, parent: str) -> str: return hint_str - def _convert_message_def(self, message_def: MessageDefinition) -> str: + def _convert_message_def(self, message_def: dict) -> str: """Create a type hint string for a message definition.""" subfield_hints = [] # Create the Message type hint - message_hint = (f'class {message_def.name}(Message):\n' + message_hint = (f'class {message_def['name']}(Message):\n' ' @property\n' ' def header(self) -> Header: ...\n\n' ' @property\n' - f' def body(self) -> {message_def.name}_Body: ...\n\n') + f' def body(self) -> {message_def['name']}_Body: ...\n\n') # Create the MessageBody type hint - body_name = f'{message_def.name}_Body' + body_name = f'{message_def['name']}_Body' body_hint = f'class {body_name}(MessageBody):\n' - for field in message_def.fields[message_def.latest_message_crc]: + fields = message_def['fields'][message_def['latestMsgDefCrc']] + if not fields: + body_hint += ' pass\n\n' + for field in fields: python_type = self._get_field_pytype(field, body_name) body_hint += ' @property\n' - body_hint += f' def {field.name}(self) -> {python_type}: ...\n\n' + body_hint += f' def {field['name']}(self) -> {python_type}: ...\n\n' # Create hints for any subfields - if field.type == ne.FIELD_TYPE.FIELD_ARRAY: + if field['type'] == 'FIELD_ARRAY': subfield_hints.append(self._convert_field_array_def(field, body_name)) # Combine all hints @@ -174,18 +175,32 @@ def get_message_stubs(self) -> str: stub_str += 'from novatel_edie import Header, Message, MessageBody, SatelliteId\n' stub_str += 'from novatel_edie.enums import *\n\n' - message_list = [key for key in self.database.messages - if not key.endswith('_Body') - and not key.endswith('_Field') - and key != 'UNKNOWN'] - message_defs = [self.database.get_msg_def(msg) - for msg in message_list] + messages = self.database['messages'] type_hints = [self._convert_message_def(msg_def) - for msg_def in message_defs] + for msg_def in messages] type_hints_str = '\n'.join(type_hints) stub_str += type_hints_str return stub_str + +def generate_stubs( + database: Annotated[ + str, + typer.Argument(help='A path to a database file.') + ], + output_dir: Annotated[ + str, + typer.Argument(help='The directory to write stub files to.') + ] = './stubs' + ): + """Generate type hint stub files for a provided database. + + Args: + database: A path to a database file. + output_dir: The directory to write stub files to. + """ + StubGenerator(database).write_stub_files(output_dir) + if __name__ == '__main__': - StubGenerator(ne.JsonDbReader.load_file('database/database.json')).write_stub_files('stubs') + typer.run(generate_stubs) diff --git a/install_customizer/pyproject.toml b/python/novatel_edie_customizer/pyproject.toml similarity index 51% rename from install_customizer/pyproject.toml rename to python/novatel_edie_customizer/pyproject.toml index d6667ed66..d10ecd1b3 100644 --- a/install_customizer/pyproject.toml +++ b/python/novatel_edie_customizer/pyproject.toml @@ -1,9 +1,13 @@ [project] -name = "novatel_edie_install_customizer" +name = "novatel_edie_customizer" description = "Tool for creating a custom installation of novatel_edie package" -version = "3.2.27" +version = "0.0.1" license = {text = "MIT"} requires-python = ">=3.8" dependencies = [ - "novatel_edie", + "typer>=0.15" ] + + +[project.scripts] +novatel_edie_customizer = "novatel_edie_customizer.cli:app" From 62e4e27c8c2fcac239810991b204999e94267204 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 11:54:46 -0700 Subject: [PATCH 28/67] Fix delimiters --- .../novatel_edie_customizer/stubgen.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py index 3fb5e4d33..c661f397c 100644 --- a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py @@ -74,13 +74,13 @@ def write_stub_files(self, file_path: str): def _convert_enum_def(self, enum_def) -> str: """Create a type hint string for an enum definition.""" - type_hint = f'class {enum_def['name']}(Enum):\n' + type_hint = f'class {enum_def["name"]}(Enum):\n' for enumerator in enum_def['enumerators']: name = enumerator['name'] name = re.sub(r'[)(\-+./\\]', '_', name) if name[0].isdigit(): name = f'_{name}' - type_hint += f' {name} = {enumerator['value']}\n' + type_hint += f' {name} = {enumerator["value"]}\n' return type_hint def get_enum_stubs(self) -> str: @@ -109,9 +109,9 @@ def _get_field_pytype(self, field, parent: str) -> str: if field['conversionString'] == r'%s': python_type = 'str' else: - python_type = f'list[{self.data_type_to_pytype.get(field['dataType']['name'])}]' + python_type = f'list[{self.data_type_to_pytype.get(field["dataType"]["name"])}]' if field['type'] == 'FIELD_ARRAY': - python_type = f'list[{parent}_{field['name']}_Field]' + python_type = f'list[{parent}_{field["name"]}_Field]' if not python_type: python_type = 'Any' return python_type @@ -121,12 +121,12 @@ def _convert_field_array_def(self, field_array_def, parent: str) -> str: subfield_hints = [] # Create MessageBodyField type hint - name = f'{parent}_{field_array_def['name']}_Field' + name = f'{parent}_{field_array_def["name"]}_Field' type_hint = f'class {name}(MessageBody):\n' for field in field_array_def['fields']: python_type = self._get_field_pytype(field, parent) type_hint += (' @property\n' - f' def {field['name']}(self) -> {python_type}: ...\n\n') + f' def {field["name"]}(self) -> {python_type}: ...\n\n') # Create hints for any subfields if field['type'] == 'FIELD_ARRAY': subfield_hints.append(self._convert_field_array_def(field, name)) @@ -142,14 +142,14 @@ def _convert_message_def(self, message_def: dict) -> str: subfield_hints = [] # Create the Message type hint - message_hint = (f'class {message_def['name']}(Message):\n' + message_hint = (f'class {message_def["name"]}(Message):\n' ' @property\n' ' def header(self) -> Header: ...\n\n' ' @property\n' - f' def body(self) -> {message_def['name']}_Body: ...\n\n') + f' def body(self) -> {message_def["name"]}_Body: ...\n\n') # Create the MessageBody type hint - body_name = f'{message_def['name']}_Body' + body_name = f'{message_def["name"]}_Body' body_hint = f'class {body_name}(MessageBody):\n' fields = message_def['fields'][message_def['latestMsgDefCrc']] if not fields: @@ -157,7 +157,7 @@ def _convert_message_def(self, message_def: dict) -> str: for field in fields: python_type = self._get_field_pytype(field, body_name) body_hint += ' @property\n' - body_hint += f' def {field['name']}(self) -> {python_type}: ...\n\n' + body_hint += f' def {field["name"]}(self) -> {python_type}: ...\n\n' # Create hints for any subfields if field['type'] == 'FIELD_ARRAY': From d5aa28d541dfef773f201752a335b9061cde3b2c Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 13:26:04 -0700 Subject: [PATCH 29/67] Fix unit tests --- src/decoders/common/test/main.cpp | 2 +- src/decoders/oem/test/main.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/decoders/common/test/main.cpp b/src/decoders/common/test/main.cpp index 5c437c9a5..c2590eac3 100644 --- a/src/decoders/common/test/main.cpp +++ b/src/decoders/common/test/main.cpp @@ -38,7 +38,7 @@ int main(int argc, char** argv) if (argc != 2) { throw std::invalid_argument("1 argument required.\nUsage: "); } - std::string strDatabaseVar = std::string(argv[1]) + "/database/messages_public.json"; + std::string strDatabaseVar = std::string(argv[1]) + "/database/database.json"; #ifdef _WIN32 if (_putenv_s("TEST_DATABASE_PATH", strDatabaseVar.c_str()) != 0) { throw std::runtime_error("Failed to set db path."); } diff --git a/src/decoders/oem/test/main.cpp b/src/decoders/oem/test/main.cpp index 517acbdca..d484b8661 100644 --- a/src/decoders/oem/test/main.cpp +++ b/src/decoders/oem/test/main.cpp @@ -38,7 +38,7 @@ int main(int argc, char** argv) if (argc != 2) { throw std::invalid_argument("1 argument required.\nUsage: "); } - std::string strDatabaseVar = std::string(argv[1]) + "/database/messages_public.json"; + std::string strDatabaseVar = std::string(argv[1]) + "/database/database.json"; std::string strResourceVar = std::string(argv[1]) + "/src/decoders/oem/test/resources/"; #ifdef _WIN32 From 8d4c4dc358491fed7b1a7b26ddf71541fc1422cf Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 13:27:44 -0700 Subject: [PATCH 30/67] Fix cmake formatting --- python/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 5690fab16..068164346 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -55,11 +55,11 @@ target_compile_definitions(python_bindings PRIVATE ) nanobind_add_stub( - bindings_stub - MODULE bindings - OUTPUT stubs/bindings.pyi - PYTHON_PATH $ - DEPENDS bindings + bindings_stub + MODULE bindings + OUTPUT stubs/bindings.pyi + PYTHON_PATH $ + DEPENDS bindings ) add_custom_target( From ef319a27ddc56bac2a248ad5c4c07c46b516a893 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 13:37:42 -0700 Subject: [PATCH 31/67] Change database ref --- regression/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression/run_tests.sh b/regression/run_tests.sh index 4a9b04939..876eeca26 100755 --- a/regression/run_tests.sh +++ b/regression/run_tests.sh @@ -10,7 +10,7 @@ fi failures=() for FORMAT in "ASCII" "ABBREV_ASCII" "BINARY" "FLATTENED_BINARY" "JSON"; do - "$converter_components" "$script_dir/../database/messages_public.json" "$script_dir/BESTUTMBIN.GPS" $FORMAT > /dev/null + "$converter_components" "$script_dir/../database/database.json" "$script_dir/BESTUTMBIN.GPS" $FORMAT > /dev/null retval=$? if [ $retval -ne 0 ]; then echo "converter_components failed for $FORMAT failed with status $retval" From ac83d9254607bfec06014d0277d498c672074a30 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 13:43:18 -0700 Subject: [PATCH 32/67] Update remaining old db path refs --- .github/workflows/cpp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 7ed2bc303..123dd1d7d 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -228,7 +228,7 @@ jobs: - name: Generate documentation for EDIE run: | sphinx-build docs docs/build - python3 scripts/database_to_html.py database/messages_public.json + python3 scripts/database_to_html.py database/database.json - name: Archive documentation artifacts uses: actions/upload-artifact@v4.3.0 From 89b6bc735dc700fea8a36449907562289c0d103a Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 13:46:15 -0700 Subject: [PATCH 33/67] Backwards compatible union hint --- .../novatel_edie_customizer/novatel_edie_customizer/stubgen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py index c661f397c..fccc1a3ad 100644 --- a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py @@ -26,6 +26,7 @@ import os import json import re +from typing import Union import typer from typing_extensions import Annotated @@ -50,7 +51,7 @@ class StubGenerator: 'SATELLITEID': 'SatelliteId', 'UNKNOWN': 'bytes' } - def __init__(self, database: str | dict): + def __init__(self, database: Union[str, dict]): if isinstance(database, str): with open(database, 'r') as f: database = json.load(f) From 434b55a26212bb7338f3a59149a6eefd58427ed7 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 14:23:52 -0700 Subject: [PATCH 34/67] try different image --- .github/workflows/python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 8347bfeb3..489b612a4 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -19,7 +19,7 @@ jobs: - [Windows, windows-latest, AMD64] # - [Windows, windows-latest, x86] - [Windows, windows-latest, ARM64] - - [MacOS, macos-latest, x86_64] + - [MacOS, macos-latest-large, x86_64] - [MacOS, macos-latest, arm64] # spdlog fails to build with the correct architecture on universal2 # - [MacOS, macos-latest, universal2] From 77a080790274695f82999452966bfebb94458e96 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 14:25:10 -0700 Subject: [PATCH 35/67] Disable incompatible runners --- .github/workflows/python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 489b612a4..cb27cbb5f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -18,8 +18,8 @@ jobs: # - [Linux, ubuntu-latest, i686] - [Windows, windows-latest, AMD64] # - [Windows, windows-latest, x86] - - [Windows, windows-latest, ARM64] - - [MacOS, macos-latest-large, x86_64] + # - [Windows, windows-latest, ARM64] + # - [MacOS, macos-latest, x86_64] - [MacOS, macos-latest, arm64] # spdlog fails to build with the correct architecture on universal2 # - [MacOS, macos-latest, universal2] From bce947d6377d41a073b72fb0097ed7008bb39687 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 14:38:22 -0700 Subject: [PATCH 36/67] Delete dead code --- python/bindings/json_db_reader.cpp | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/python/bindings/json_db_reader.cpp b/python/bindings/json_db_reader.cpp index a52d17aeb..264da4e08 100644 --- a/python/bindings/json_db_reader.cpp +++ b/python/bindings/json_db_reader.cpp @@ -8,31 +8,6 @@ namespace nb = nanobind; using namespace nb::literals; using namespace novatel::edie; -//namespace { -//std::string default_json_db_path() -//{ -// // Does the following, but using nanobind: -// // import importlib_resources -// // path_ctx = importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("database.json")) -// // with path_ctx as path: -// // return path -// nb::object import_lib_resources = nb::module_::import_("importlib_resources"); -// -// nb::object package_files = import_lib_resources.attr("files")("novatel_edie"); -// nb::object db_path = package_files.attr("joinpath")("database.json"); -// -// nb::object path_ctx = import_lib_resources.attr("as_file")(db_path); -// auto py_path = path_ctx.attr("__enter__")(); -// if (!nb::cast(py_path.attr("is_file")())) -// { -// throw NExcept((std::string("Could not find the default JSON DB file at ") + nb::str(py_path).c_str()).c_str()); -// } -// auto path = nb::cast(nb::str(py_path)); -// path_ctx.attr("__exit__")(nb::none(), nb::none(), nb::none()); -// return path; -//} -//} // namespace - PyMessageDatabase::Ptr& MessageDbSingleton::get() { static PyMessageDatabase::Ptr json_db = nullptr; From 264d2875de1df817c73d1f4367a1614af895b5d9 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 14:48:25 -0700 Subject: [PATCH 37/67] Add docs to C++ --- python/bindings/message_db_singleton.hpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/bindings/message_db_singleton.hpp b/python/bindings/message_db_singleton.hpp index d8420f103..becc77d5c 100644 --- a/python/bindings/message_db_singleton.hpp +++ b/python/bindings/message_db_singleton.hpp @@ -5,10 +5,26 @@ namespace novatel::edie { +//============================================================================ +//! \class MessageDbSingleton +//! \brief The singular default MessageDatabase based on the JSON database +//! file included in the novatel_edie package. +//! +//! If the package does not contain a JSON database file or the C++ bindings +//! submodule is imported outside of the package, the default database will +//! be empty. +//============================================================================ class MessageDbSingleton { public: MessageDbSingleton() = delete; + + + //---------------------------------------------------------------------------- + //! \brief Method to get the MessageDbSingleton. + //! + //! If the instance does not yet exist, it will be created and returned. + //---------------------------------------------------------------------------- [[nodiscard]] static PyMessageDatabase::Ptr& get(); }; From 0003e07e5ce105ec94b8bb2aa7c404c086ae7b51 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Wed, 5 Feb 2025 15:18:57 -0700 Subject: [PATCH 38/67] Add docstrings --- .../novatel_edie_customizer/installer.py | 18 +++--- .../novatel_edie_customizer/stubgen.py | 64 ++++++++++++++++--- 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/installer.py b/python/novatel_edie_customizer/novatel_edie_customizer/installer.py index f9ec80ac6..3e8e3a937 100644 --- a/python/novatel_edie_customizer/novatel_edie_customizer/installer.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/installer.py @@ -37,10 +37,11 @@ @contextmanager def open_library_clone(library: str): - """Creates a clone of specified files in a library and moves into the temp directory. + """Creates a clone of a specified library within a temporary directory. Args: - library: The name of the library to clone. + library: The name of the library to clone. Must be within + the current Python environment. """ packages_path = os.path.join(sys.exec_prefix, 'Lib', 'site-packages') library_path = os.path.join(packages_path, library) @@ -83,6 +84,8 @@ def copy_file(file_path: str, destination_path: str = None): Args: file_path: The path of the file to copy. + destination_path: A relative path within current working directory + to copy the file to. """ if not os.path.exists(file_path): raise FileNotFoundError(f"File {file_path} not found.") @@ -95,11 +98,7 @@ def copy_file(file_path: str, destination_path: str = None): shutil.copy2(file_path, destination_path) def install_package(): - """Installs a package in the current working directory. - - Args: - package_name: The name of the package to install. - """ + """Installs a package in the current working directory.""" try: subprocess.check_call([ sys.executable, '-m', 'wheel', 'pack', './wheel']) @@ -120,7 +119,10 @@ def install_custom( str, typer.Argument(help='A path to a database file.') ]): - """Generate type hint stub files for a provided database. + """Create a custom installation of novatel_edie based on the provided DB. + + The custom installation will have all messages and enums of the provided + database be directly importable from the 'enums' and 'messages' submodules. Args: database: A path to a database file. diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py index fccc1a3ad..957c02afd 100644 --- a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py @@ -52,6 +52,12 @@ class StubGenerator: 'UNKNOWN': 'bytes' } def __init__(self, database: Union[str, dict]): + """Initializes a StubGenerator. + + Args: + database: A path to a message database json file or + a dictionary representation of a database. + """ if isinstance(database, str): with open(database, 'r') as f: database = json.load(f) @@ -62,7 +68,11 @@ def __init__(self, database: Union[str, dict]): self.database['enums_by_id'] = {enum['_id']: enum for enum in database['enums']} def write_stub_files(self, file_path: str): - """Writes package stub files to the specified directory.""" + """Writes package stub files to the specified directory. + + Args: + file_path: The directory to write the stub files to. + """ file_specs = { 'enums.pyi': self.get_enum_stubs(), 'messages.pyi': self.get_message_stubs()} @@ -74,7 +84,13 @@ def write_stub_files(self, file_path: str): f.write(file_contents) def _convert_enum_def(self, enum_def) -> str: - """Create a type hint string for an enum definition.""" + """Create a type hint string for an enum definition. + + Args: + enum_def: A dictionary containing an enum definition. + Returns: + A string containing a type hint for the enum definition. + """ type_hint = f'class {enum_def["name"]}(Enum):\n' for enumerator in enum_def['enumerators']: name = enumerator['name'] @@ -85,7 +101,11 @@ def _convert_enum_def(self, enum_def) -> str: return type_hint def get_enum_stubs(self) -> str: - """Get a stub string for all enums in the database.""" + """Get a stub string for all enums in the database. + + Returns: + A string containing type hint stubs for all enums in the database. + """ stub_str = ('from enum import Enum\n' 'from typing import Any\n\n') enums = self.database['enums'] @@ -94,8 +114,16 @@ def get_enum_stubs(self) -> str: stub_str += type_hints_str return stub_str - def _get_field_pytype(self, field, parent: str) -> str: - """Get a type hint string for a base level field.""" + def _get_field_pytype(self, field: dict, parent: str) -> str: + """Get a type hint string for a field definition. + + Args: + field: A dictionary containing a field definition. + parent: The name of the parent class. May be used for naming + subfield classes. + Returns: + A string containing a type hint for the field definition. + """ python_type = None if field['type'] == 'SIMPLE': python_type = self.data_type_to_pytype.get(field['dataType']['name']) @@ -117,8 +145,16 @@ def _get_field_pytype(self, field, parent: str) -> str: python_type = 'Any' return python_type - def _convert_field_array_def(self, field_array_def, parent: str) -> str: - """Convert a field array definition to a type hint string.""" + def _convert_field_array_def(self, field_array_def: dict, parent: str) -> str: + """Convert a field array definition to a type hint string. + + Args: + field_array_def: A dictionary containing a field array definition. + parent: The name of the parent class. The name for the field array + class will be based on this. + Returns: + A string containing type hint stubs for the field array definition. + """ subfield_hints = [] # Create MessageBodyField type hint @@ -139,7 +175,13 @@ def _convert_field_array_def(self, field_array_def, parent: str) -> str: return hint_str def _convert_message_def(self, message_def: dict) -> str: - """Create a type hint string for a message definition.""" + """Create a type hint string for a message definition. + + Args: + message_def: A dictionary containing a message definition. + Returns: + A string containing type hint stubs for the message definition. + """ subfield_hints = [] # Create the Message type hint @@ -171,7 +213,11 @@ def _convert_message_def(self, message_def: dict) -> str: return hint_str def get_message_stubs(self) -> str: - """Get a stub string for all messages in the database.""" + """Get a stub string for all messages in the database. + + Returns: + A string containing type hint stubs for all messages in the database. + """ stub_str = 'from typing import Any\n\n' stub_str += 'from novatel_edie import Header, Message, MessageBody, SatelliteId\n' stub_str += 'from novatel_edie.enums import *\n\n' From 850be5a17f36271a5522402f3a5483e829d86c87 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 6 Feb 2025 11:11:27 -0700 Subject: [PATCH 39/67] fix remaining jsondb refs --- benchmarks/benchmark.cpp | 2 +- conanfile.py | 2 +- examples/novatel/command_encoding/command_encoding.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/benchmark.cpp b/benchmarks/benchmark.cpp index af0f63b9e..d559abd9a 100644 --- a/benchmarks/benchmark.cpp +++ b/benchmarks/benchmark.cpp @@ -212,7 +212,7 @@ int main(int argc, char** argv) { if (argc < 2) { throw std::invalid_argument("1 argument required.\nUsage: [benchmark options]"); } - std::string strDatabaseVar = std::string(argv[1]) + "/database/messages_public.json"; + std::string strDatabaseVar = std::string(argv[1]) + "/database/database.json"; #ifdef _WIN32 if (_putenv_s("TEST_DATABASE_PATH", strDatabaseVar.c_str()) != 0) { throw std::runtime_error("Failed to set db path."); } diff --git a/conanfile.py b/conanfile.py index a753a6f55..cb2d79827 100644 --- a/conanfile.py +++ b/conanfile.py @@ -104,5 +104,5 @@ def package_info(self): "edie_decoders_common", "edie_common", ] - db_path = os.path.join(self.package_folder, "res", "novatel_edie", "messages_public.json") + db_path = os.path.join(self.package_folder, "res", "novatel_edie", "database.json") self.runenv_info.define_path("EDIE_DATABASE_FILE", db_path) diff --git a/examples/novatel/command_encoding/command_encoding.cpp b/examples/novatel/command_encoding/command_encoding.cpp index 2867f6ce0..3e325aa17 100644 --- a/examples/novatel/command_encoding/command_encoding.cpp +++ b/examples/novatel/command_encoding/command_encoding.cpp @@ -52,7 +52,7 @@ int main(int argc, char* argv[]) if (argc < 3) { logger->error("Format: command_encoding \n"); - logger->error("Example: command_encoding database/messages_public.json ASCII \"RTKTIMEOUT 30\"\n"); + logger->error("Example: command_encoding database/database.json ASCII \"RTKTIMEOUT 30\"\n"); return 1; } From 7bda66f0d73e8deb7ffbee80c2c1f902d1ec57bd Mon Sep 17 00:00:00 2001 From: riley-kinahan Date: Thu, 6 Feb 2025 11:13:27 -0700 Subject: [PATCH 40/67] Update python/CMakeLists.txt Add CMake stub dependencies for automatic regeneration Co-authored-by: Martin Valgur --- python/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 068164346..6eee65e97 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -66,6 +66,9 @@ add_custom_target( dynamic_stubs ALL COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs COMMENT "Generating dynamic stubs" + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie_customizer/novatel_edie_customizer/stubgen.py + ${CMAKE_SOURCE_DIR}/database/database.json ) if(DEFINED SKBUILD_PROJECT_NAME) From 2fda1774dda50ef8619aa7a2eccb7fc82984c547 Mon Sep 17 00:00:00 2001 From: riley-kinahan Date: Thu, 6 Feb 2025 11:15:49 -0700 Subject: [PATCH 41/67] Update python/bindings/message_database.cpp Simplify string cleaning Co-authored-by: Martin Valgur --- python/bindings/message_database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index b9568fa97..76f00bb76 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -219,7 +219,7 @@ void cleanString(std::string& str) // Remove special characters from the string to make it a valid python attribute name for (char& c : str) { - if (c == '-' || c == '+' || c == '.' || c == '/' || c == '(' || c == ')') { c = '_'; } + if (!isalnum(c)) { c = '_'; } } if (isdigit(str[0])) { str = "_" + str; } } From 63cf5a2fedd5c5eb92225ba834b54a97aa15761a Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 6 Feb 2025 11:18:50 -0700 Subject: [PATCH 42/67] Add out to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 18357e305..d6f4f6196 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ build/ docs/build bin/ +out/ CMakeSettings.json CMakeUserPresets.json /include/novatel_edie/version.h From 32a11e5a749bbaf101abdf99e96cf975c711322a Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 6 Feb 2025 11:21:46 -0700 Subject: [PATCH 43/67] Expand .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index d6f4f6196..4c54cfaf9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,11 @@ build/ docs/build bin/ out/ +CMakeFiles/ +**.cmake +**.dir/ +Debug/ +conan_host_profile CMakeSettings.json CMakeUserPresets.json /include/novatel_edie/version.h From adb9036b43cc26018faa11031c79134dd859b331 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 6 Feb 2025 11:22:17 -0700 Subject: [PATCH 44/67] nicer text formatting --- .../novatel_edie_customizer/stubgen.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py index 957c02afd..127fcb196 100644 --- a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py @@ -26,6 +26,7 @@ import os import json import re +import textwrap from typing import Union import typer @@ -185,11 +186,14 @@ def _convert_message_def(self, message_def: dict) -> str: subfield_hints = [] # Create the Message type hint - message_hint = (f'class {message_def["name"]}(Message):\n' - ' @property\n' - ' def header(self) -> Header: ...\n\n' - ' @property\n' - f' def body(self) -> {message_def["name"]}_Body: ...\n\n') + name = message_def['name'] + message_hint = textwrap.dedent(f"""\ + class {name}(Message): + @property + def header(self) -> Header: ... + @property + def body(self) -> {name}_Body: ... + """) # Create the MessageBody type hint body_name = f'{message_def["name"]}_Body' From 5ceb378a96c6187f40deb45deef8acb0931b5e60 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 6 Feb 2025 11:27:45 -0700 Subject: [PATCH 45/67] Fix compliler warning --- python/bindings/message_decoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index bca943fd0..8ce7122ba 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -101,7 +101,7 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } PyMessageBody::PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) - : fields(std::move(message_)), parent_db_(std::move(parent_db_)), name(std::move(name_)) + : name(std::move(name_)), fields(std::move(message_)), parent_db_(std::move(parent_db_)) {} nb::dict& PyMessageBody::get_values() const From 31e51a0b8f384bd5f6e2cd7e0f387e3a691ed8f3 Mon Sep 17 00:00:00 2001 From: Martin Valgur Date: Thu, 6 Feb 2025 22:10:19 +0200 Subject: [PATCH 46/67] Add type hinting support - drop PyMessage subclasses (#97) * Rename MESSAGE_Body -> MESSAGE * Drop PyMessage subclasses * Improve PyMessage repr * Tidy test_decode_encode.py --------- Co-authored-by: riley-kinahan --- python/CMakeLists.txt | 2 +- python/bindings/message_database.cpp | 35 ++-- python/bindings/message_decoder.cpp | 70 +++---- python/bindings/py_decoded_message.hpp | 9 +- .../novatel_edie_customizer/stubgen.py | 10 +- python/test/test_decode_encode.py | 174 +++++++++--------- 6 files changed, 154 insertions(+), 146 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6eee65e97..2f128b15b 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -64,7 +64,7 @@ nanobind_add_stub( add_custom_target( dynamic_stubs ALL - COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs + COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie_customizer/novatel_edie_customizer/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs COMMENT "Generating dynamic stubs" DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie_customizer/novatel_edie_customizer/stubgen.py diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index 76f00bb76..46e6cb321 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -184,7 +184,8 @@ void init_common_message_database(nb::module_& m) .def_prop_ro("messages", &PyMessageDatabase::GetMessagesByNameDict); } -PyMessageDatabase::PyMessageDatabase() { +PyMessageDatabase::PyMessageDatabase() +{ UpdatePythonEnums(); UpdatePythonMessageTypes(); } @@ -197,12 +198,14 @@ PyMessageDatabase::PyMessageDatabase(std::vector vM UpdatePythonMessageTypes(); } -PyMessageDatabase::PyMessageDatabase(const MessageDatabase& message_db) noexcept : MessageDatabase(message_db) { +PyMessageDatabase::PyMessageDatabase(const MessageDatabase& message_db) noexcept : MessageDatabase(message_db) +{ UpdatePythonEnums(); UpdatePythonMessageTypes(); } -PyMessageDatabase::PyMessageDatabase(const MessageDatabase&& message_db) noexcept : MessageDatabase(message_db) { +PyMessageDatabase::PyMessageDatabase(const MessageDatabase&& message_db) noexcept : MessageDatabase(message_db) +{ UpdatePythonEnums(); UpdatePythonMessageTypes(); } @@ -233,7 +236,8 @@ inline void PyMessageDatabase::UpdatePythonEnums() { nb::dict values; const char* enum_name = enum_def->name.c_str(); - for (const auto& enumerator : enum_def->enumerators) { + for (const auto& enumerator : enum_def->enumerators) + { std::string enumerator_name = enumerator.name; cleanString(enumerator_name); values[enumerator_name.c_str()] = enumerator.value; @@ -250,7 +254,8 @@ void PyMessageDatabase::AddFieldType(std::vector> fie nb::handle type_tuple, nb::handle type_dict) { // rescursively add field types for each field array element within the provided vector - for (const auto& field : fields) { + for (const auto& field : fields) + { if (field->type == FIELD_TYPE::FIELD_ARRAY) { auto* field_array_field = dynamic_cast(field.get()); @@ -258,7 +263,7 @@ void PyMessageDatabase::AddFieldType(std::vector> fie nb::object field_type = type_constructor(field_name, type_tuple, type_dict); messages_by_name[field_name] = field_type; AddFieldType(field_array_field->fields, field_name, type_constructor, type_tuple, type_dict); - } + } } } @@ -270,23 +275,19 @@ void PyMessageDatabase::UpdatePythonMessageTypes() // get type constructor nb::object type_constructor = nb::module_::import_("builtins").attr("type"); // specify the python superclasses for the new message and message body types - nb::tuple type_tuple = nb::make_tuple(nb::type()); nb::tuple body_type_tuple = nb::make_tuple(nb::type()); // provide no additional attributes via `__dict__` nb::dict type_dict = nb::dict(); // add message and message body types for each message definition - for (const auto& message_def : MessageDefinitions()) { - nb::object msg_def = type_constructor(message_def->name, type_tuple, type_dict); - messages_by_name[message_def->name] = msg_def; - nb::object msg_body_def = type_constructor(message_def->name + "_Body", body_type_tuple, type_dict); - messages_by_name[message_def->name + "_Body"] = msg_body_def; + for (const auto& message_def : MessageDefinitions()) + { + nb::object msg_body_def = type_constructor(message_def->name, body_type_tuple, type_dict); + messages_by_name[message_def->name] = msg_body_def; // add additional MessageBody types for each field array element within the message definition - AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name + "_Body", type_constructor, body_type_tuple, type_dict); + AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name, type_constructor, body_type_tuple, type_dict); } // provide UNKNOWN types for undecodable messages - nb::object default_msg_def = type_constructor("UNKNOWN", type_tuple, type_dict); - messages_by_name["UNKNOWN"] = default_msg_def; - nb::object default_msg_body_def = type_constructor("UNKNOWN_Body", body_type_tuple, type_dict); - messages_by_name["UNKNOWN_Body"] = default_msg_body_def; + nb::object default_msg_body_def = type_constructor("UNKNOWN", body_type_tuple, type_dict); + messages_by_name["UNKNOWN"] = default_msg_body_def; } diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 8ce7122ba..60aad0b4e 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -1,9 +1,9 @@ #include "novatel_edie/decoders/oem/message_decoder.hpp" #include -#include #include #include +#include #include "bindings_core.hpp" #include "message_db_singleton.hpp" @@ -47,18 +47,22 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C sub_values.reserve(message_field.size()); nb::handle field_ptype; std::string field_name = parent + "_" + field.fieldDef->name + "_Field"; - try { + try + { field_ptype = parent_db->GetMessagesByNameDict().at(field_name); - } catch (const std::out_of_range& e) { - field_ptype = parent_db->GetMessagesByNameDict().at("UNKNOWN_Body"); } - for (const auto& subfield : message_field) { + catch (const std::out_of_range& e) + { + field_ptype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + } + for (const auto& subfield : message_field) + { nb::object pyinst = nb::inst_alloc(field_ptype); PyMessageBody* cinst = nb::inst_ptr(pyinst); const auto& message_subfield = std::get>(subfield.fieldValue); new (cinst) PyMessageBody(message_subfield, parent_db, field_name); nb::inst_mark_ready(pyinst); - sub_values.push_back(pyinst); + sub_values.push_back(pyinst); } return nb::cast(sub_values); } @@ -102,7 +106,8 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C PyMessageBody::PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) : name(std::move(name_)), fields(std::move(message_)), parent_db_(std::move(parent_db_)) -{} +{ +} nb::dict& PyMessageBody::get_values() const { @@ -229,12 +234,13 @@ void init_novatel_message_decoder(nb::module_& m) .def_ro("body", &PyMessage::message_body) .def_ro("header", &PyMessage::header) .def_ro("name", &PyMessage::name) - .def("to_dict", [](const PyMessage& self) { - nb::dict message_dict; - message_dict["header"] = self.header.attr("to_dict")(); - message_dict["body"] = self.message_body.attr("to_dict")(); - return message_dict; - }) + .def("to_dict", + [](const PyMessage& self) { + nb::dict message_dict; + message_dict["header"] = self.header.attr("to_dict")(); + message_dict["body"] = self.message_body.attr("to_dict")(); + return message_dict; + }) .def("__repr__", &PyMessage::repr) .def("__str__", &PyMessage::repr); @@ -256,30 +262,24 @@ void init_novatel_message_decoder(nb::module_& m) std::vector fields; STATUS status = decoder.Decode(reinterpret_cast(message_body.c_str()), fields, metadata); PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); - nb::handle message_pytype; nb::handle body_pytype; const std::string message_name = metadata.MessageName(); - const std::string message_body_name = metadata.MessageName() + "_Body"; - try { - message_pytype = parent_db->GetMessagesByNameDict().at(message_name); - body_pytype = parent_db->GetMessagesByNameDict().at(message_body_name); - } catch (const std::out_of_range& e) + try { - message_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); - body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN_Body"); + body_pytype = parent_db->GetMessagesByNameDict().at(message_name); + } + catch (const std::out_of_range& e) + { + body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); } - nb::object body_pyinst = nb::inst_alloc(body_pytype); PyMessageBody* body_cinst = nb::inst_ptr(body_pyinst); - new (body_cinst) PyMessageBody(std::move(fields), parent_db, message_body_name); + new (body_cinst) PyMessageBody(std::move(fields), parent_db, message_name); nb::inst_mark_ready(body_pyinst); - nb::object message_pyinst = nb::inst_alloc(message_pytype); - PyMessage* message_cinst = nb::inst_ptr(message_pyinst); - new (message_cinst) PyMessage(body_pyinst, header, message_name); - nb::inst_mark_ready(message_pyinst); + auto message_pyinst = std::make_shared(body_pyinst, header, message_name); return nb::make_tuple(status, message_pyinst); }, @@ -292,18 +292,18 @@ void init_novatel_message_decoder(nb::module_& m) std::string body_str(message_body.c_str(), message_body.size()); const char* data_ptr = body_str.c_str(); STATUS status = static_cast(&decoder)->TestDecodeAscii(msg_def_fields, &data_ptr, fields); - return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN_Body")); - }, - "msg_def_fields"_a, "message_body"_a) + return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN")); + }, + "msg_def_fields"_a, "message_body"_a) .def( "_decode_binary", [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body, - uint32_t message_length) { + uint32_t message_length) { std::vector fields; const char* data_ptr = message_body.c_str(); STATUS status = static_cast(&decoder)->TestDecodeBinary(msg_def_fields, reinterpret_cast(&data_ptr), fields, message_length); - return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN_Body")); - }, - "msg_def_fields"_a, "message_body"_a, "message_length"_a); - } + return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN")); + }, + "msg_def_fields"_a, "message_body"_a, "message_length"_a); +} diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index a785e3d22..dbb512b00 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -46,15 +46,14 @@ struct PyMessage nb::object header; std::string name; - PyMessage(nb::object message_body_, nb::object header_, std::string name_) - : message_body(message_body_), header(header_), name(name_) {} + PyMessage(nb::object message_body_, nb::object header_, std::string name_) : message_body(message_body_), header(header_), name(name_) {} std::string repr() const { - return "<" + name + " Message>"; + std::stringstream repr; + repr << ""; + return repr.str(); } }; - - } // namespace novatel::edie::oem diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py index 127fcb196..0d031528a 100644 --- a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py @@ -196,22 +196,22 @@ def body(self) -> {name}_Body: ... """) # Create the MessageBody type hint - body_name = f'{message_def["name"]}_Body' - body_hint = f'class {body_name}(MessageBody):\n' + name = message_def["name"] + body_hint = f'class {name}(MessageBody):\n' fields = message_def['fields'][message_def['latestMsgDefCrc']] if not fields: body_hint += ' pass\n\n' for field in fields: - python_type = self._get_field_pytype(field, body_name) + python_type = self._get_field_pytype(field, name) body_hint += ' @property\n' body_hint += f' def {field["name"]}(self) -> {python_type}: ...\n\n' # Create hints for any subfields if field['type'] == 'FIELD_ARRAY': - subfield_hints.append(self._convert_field_array_def(field, body_name)) + subfield_hints.append(self._convert_field_array_def(field, name)) # Combine all hints - hints = subfield_hints + [body_hint, message_hint] + hints = subfield_hints + [body_hint] hint_str = '\n'.join(hints) return hint_str diff --git a/python/test/test_decode_encode.py b/python/test/test_decode_encode.py index 230424a0b..4c8e00267 100644 --- a/python/test/test_decode_encode.py +++ b/python/test/test_decode_encode.py @@ -28,12 +28,14 @@ # Encoder and Filter. ################################################################################ import enum +from collections import namedtuple import novatel_edie as ne import pytest from novatel_edie import STATUS, ENCODE_FORMAT +from novatel_edie.messages import * from pytest import approx -from collections import namedtuple + # ------------------------------------------------------------------------------------------------------- # Decode/Encode Unit Tests @@ -289,38 +291,40 @@ def test_ascii_log_roundtrip_gloalmanac(helper): def test_ascii_log_roundtrip_gloephem(helper): log = b"#GLOEPHEMERISA,COM1,11,67.0,SATTIME,2168,160218.000,02000820,8d29,32768;51,0,1,80,2168,161118000,10782,573,0,0,95,0,-2.3917966796875000e+07,4.8163881835937500e+06,7.4258510742187500e+06,-1.0062713623046875e+03,1.8321990966796875e+02,-3.3695755004882813e+03,1.86264514923095700e-06,-9.31322574615478510e-07,-0.00000000000000000,-6.69313594698905940e-05,5.587935448e-09,0.00000000000000000,84600,3,2,0,13*ad20fc5f\r\n" - ret_code, message_data, glo_ephemeris = helper.DecodeEncode(ENCODE_FORMAT.FLATTENED_BINARY, log, return_message=True) + ret_code, message_data, message = helper.DecodeEncode(ENCODE_FORMAT.FLATTENED_BINARY, log, return_message=True) assert ret_code == Result.SUCCESS - assert glo_ephemeris.body.sloto == 51 - assert glo_ephemeris.body.freqo == 0 - assert glo_ephemeris.body.sat_type == 1 - assert glo_ephemeris.body.false_iod == 80 - assert glo_ephemeris.body.ephem_week == 2168 - assert glo_ephemeris.body.ephem_time == 161118000 - assert glo_ephemeris.body.time_offset == 10782 - assert glo_ephemeris.body.nt == 573 - assert glo_ephemeris.body.GLOEPHEMERIS_reserved == 0 - assert glo_ephemeris.body.GLOEPHEMERIS_reserved_9 == 0 - assert glo_ephemeris.body.issue == 95 - assert glo_ephemeris.body.broadcast_health == 0 - assert glo_ephemeris.body.pos_x == -2.3917966796875000e+07 - assert glo_ephemeris.body.pos_y == 4.8163881835937500e+06 - assert glo_ephemeris.body.pos_z == 7.4258510742187500e+06 - assert glo_ephemeris.body.vel_x == -1.0062713623046875e+03 - assert glo_ephemeris.body.vel_y == 1.8321990966796875e+02 - assert glo_ephemeris.body.vel_z == -3.3695755004882813e+03 - assert glo_ephemeris.body.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) - assert glo_ephemeris.body.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) - assert glo_ephemeris.body.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) - assert glo_ephemeris.body.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) - assert glo_ephemeris.body.delta_tau == 5.587935448e-09 - assert glo_ephemeris.body.gamma == approx(0.00000000000000000, abs=0.0000000000000001) - assert glo_ephemeris.body.tk == 84600 - assert glo_ephemeris.body.p == 3 - assert glo_ephemeris.body.ft == 2 - assert glo_ephemeris.body.age == 0 - assert glo_ephemeris.body.flags == 13 + glo_ephemeris = message.body + assert isinstance(glo_ephemeris, GLOEPHEMERIS) + assert glo_ephemeris.sloto == 51 + assert glo_ephemeris.freqo == 0 + assert glo_ephemeris.sat_type == 1 + assert glo_ephemeris.false_iod == 80 + assert glo_ephemeris.ephem_week == 2168 + assert glo_ephemeris.ephem_time == 161118000 + assert glo_ephemeris.time_offset == 10782 + assert glo_ephemeris.nt == 573 + assert glo_ephemeris.GLOEPHEMERIS_reserved == 0 + assert glo_ephemeris.GLOEPHEMERIS_reserved_9 == 0 + assert glo_ephemeris.issue == 95 + assert glo_ephemeris.broadcast_health == 0 + assert glo_ephemeris.pos_x == -2.3917966796875000e+07 + assert glo_ephemeris.pos_y == 4.8163881835937500e+06 + assert glo_ephemeris.pos_z == 7.4258510742187500e+06 + assert glo_ephemeris.vel_x == -1.0062713623046875e+03 + assert glo_ephemeris.vel_y == 1.8321990966796875e+02 + assert glo_ephemeris.vel_z == -3.3695755004882813e+03 + assert glo_ephemeris.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) + assert glo_ephemeris.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) + assert glo_ephemeris.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) + assert glo_ephemeris.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) + assert glo_ephemeris.delta_tau == 5.587935448e-09 + assert glo_ephemeris.gamma == approx(0.00000000000000000, abs=0.0000000000000001) + assert glo_ephemeris.tk == 84600 + assert glo_ephemeris.p == 3 + assert glo_ephemeris.ft == 2 + assert glo_ephemeris.age == 0 + assert glo_ephemeris.flags == 13 def test_ascii_log_roundtrip_loglist(helper): @@ -480,7 +484,7 @@ def test_short_binary_log_roundtrip_rawimu(helper): def test_flat_binary_log_decode_bestpos(helper): log = b" Date: Thu, 6 Feb 2025 15:12:38 -0700 Subject: [PATCH 47/67] get substruct working --- python/bindings/encoder.cpp | 5 +- python/bindings/message_database.cpp | 2 +- python/bindings/message_decoder.cpp | 84 ++++++++++++++------------ python/bindings/py_decoded_message.hpp | 35 +++++++---- 4 files changed, 69 insertions(+), 57 deletions(-) diff --git a/python/bindings/encoder.cpp b/python/bindings/encoder.cpp index 775d369b1..3011b74aa 100644 --- a/python/bindings/encoder.cpp +++ b/python/bindings/encoder.cpp @@ -24,7 +24,6 @@ void init_novatel_encoder(nb::module_& m) const oem::MetaDataStruct& metadata, ENCODE_FORMAT format) { MessageDataStruct message_data; oem::IntermediateHeader* header_cinst = nb::inst_ptr(py_message.header); - oem::PyMessageBody* body_cinst = nb::inst_ptr(py_message.message_body); if (format == ENCODE_FORMAT::JSON) { // Allocate more space for JSON messages. @@ -33,7 +32,7 @@ void init_novatel_encoder(nb::module_& m) uint8_t buffer[MESSAGE_SIZE_MAX * 3]; auto* buf_ptr = reinterpret_cast(&buffer); uint32_t buf_size = MESSAGE_SIZE_MAX * 3; - STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, body_cinst->fields, message_data, metadata, format); + STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, py_message.fields, message_data, metadata, format); return nb::make_tuple(status, oem::PyMessageData(message_data)); } else @@ -41,7 +40,7 @@ void init_novatel_encoder(nb::module_& m) uint8_t buffer[MESSAGE_SIZE_MAX]; auto buf_ptr = reinterpret_cast(&buffer); uint32_t buf_size = MESSAGE_SIZE_MAX; - STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, body_cinst->fields, message_data, metadata, format); + STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, py_message.fields, message_data, metadata, format); return nb::make_tuple(status, oem::PyMessageData(message_data)); } }, diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index 46e6cb321..e9903c399 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -275,7 +275,7 @@ void PyMessageDatabase::UpdatePythonMessageTypes() // get type constructor nb::object type_constructor = nb::module_::import_("builtins").attr("type"); // specify the python superclasses for the new message and message body types - nb::tuple body_type_tuple = nb::make_tuple(nb::type()); + nb::tuple body_type_tuple = nb::make_tuple(nb::type()); // provide no additional attributes via `__dict__` nb::dict type_dict = nb::dict(); diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 60aad0b4e..533be0e19 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -58,9 +58,9 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C for (const auto& subfield : message_field) { nb::object pyinst = nb::inst_alloc(field_ptype); - PyMessageBody* cinst = nb::inst_ptr(pyinst); + PyField* cinst = nb::inst_ptr(pyinst); const auto& message_subfield = std::get>(subfield.fieldValue); - new (cinst) PyMessageBody(message_subfield, parent_db, field_name); + new (cinst) PyField(message_subfield, parent_db, field_name); nb::inst_mark_ready(pyinst); sub_values.push_back(pyinst); } @@ -104,12 +104,12 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } } -PyMessageBody::PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) +PyField::PyField(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) : name(std::move(name_)), fields(std::move(message_)), parent_db_(std::move(parent_db_)) { } -nb::dict& PyMessageBody::get_values() const +nb::dict& PyField::get_values() const { if (cached_values_.size() == 0) { @@ -118,7 +118,7 @@ nb::dict& PyMessageBody::get_values() const return cached_values_; } -nb::dict& PyMessageBody::get_fields() const +nb::dict& PyField::get_fields() const { if (cached_fields_.size() == 0) { @@ -127,18 +127,18 @@ nb::dict& PyMessageBody::get_fields() const return cached_fields_; } -nb::dict PyMessageBody::to_dict() const +nb::dict PyField::to_dict() const { nb::dict dict; for (const auto& [field_name, value] : get_values()) { - if (nb::isinstance(value)) { dict[field_name] = nb::cast(value).to_dict(); } + if (nb::isinstance(value)) { dict[field_name] = nb::cast(value).to_dict(); } else if (nb::isinstance>(value)) { nb::list list; for (const auto& sub_item : nb::cast>(value)) { - if (nb::isinstance(sub_item)) { list.append(nb::cast(sub_item).to_dict()); } + if (nb::isinstance(sub_item)) { list.append(nb::cast(sub_item).to_dict()); } else { list.append(sub_item); } } dict[field_name] = list; @@ -148,19 +148,19 @@ nb::dict PyMessageBody::to_dict() const return dict; } -nb::object PyMessageBody::getattr(nb::str field_name) const +nb::object PyField::getattr(nb::str field_name) const { if (!contains(field_name)) { throw nb::attribute_error(field_name.c_str()); } return get_values()[std::move(field_name)]; } -nb::object PyMessageBody::getitem(nb::str field_name) const { return get_values()[std::move(field_name)]; } +nb::object PyField::getitem(nb::str field_name) const { return get_values()[std::move(field_name)]; } -bool PyMessageBody::contains(nb::str field_name) const { return get_values().contains(std::move(field_name)); } +bool PyField::contains(nb::str field_name) const { return get_values().contains(std::move(field_name)); } -size_t PyMessageBody::len() const { return fields.size(); } +size_t PyField::len() const { return fields.size(); } -std::string PyMessageBody::repr() const +std::string PyField::repr() const { std::stringstream repr; repr << name << "("; @@ -206,13 +206,13 @@ void init_novatel_message_decoder(nb::module_& m) .def_rw("milliseconds", &PyGpsTime::milliseconds) .def_rw("status", &PyGpsTime::time_status); - nb::class_(m, "MessageBody") - .def_prop_ro("_values", &PyMessageBody::get_values) - .def_prop_ro("_fields", &PyMessageBody::get_fields) - .def("to_dict", &PyMessageBody::to_dict, "Convert the message and its sub-messages into a dict") - .def("__getattr__", &PyMessageBody::getattr, "field_name"_a) - .def("__repr__", &PyMessageBody::repr) - .def("__str__", &PyMessageBody::repr) + nb::class_(m, "Field") + .def_prop_ro("_values", &PyField::get_values) + .def_prop_ro("_fields", &PyField::get_fields) + .def("to_dict", &PyField::to_dict, "Convert the message and its sub-messages into a dict") + .def("__getattr__", &PyField::getattr, "field_name"_a) + .def("__repr__", &PyField::repr) + .def("__str__", &PyField::repr) .def("__dir__", [](nb::object self) { // get required Python builtin functions nb::module_ builtins = nb::module_::import_("builtins"); @@ -224,25 +224,29 @@ void init_novatel_message_decoder(nb::module_& m) nb::object super_obj = super(body_type, self); nb::list base_list = nb::cast(super_obj.attr("__dir__")()); // add dynamic fields to the list - PyMessageBody* body = nb::inst_ptr(self); + PyField* body = nb::inst_ptr(self); for (const auto& [field_name, _] : body->get_fields()) { base_list.append(field_name); } return base_list; }); - nb::class_(m, "Message") - .def_ro("body", &PyMessage::message_body) - .def_ro("header", &PyMessage::header) - .def_ro("name", &PyMessage::name) - .def("to_dict", - [](const PyMessage& self) { - nb::dict message_dict; - message_dict["header"] = self.header.attr("to_dict")(); - message_dict["body"] = self.message_body.attr("to_dict")(); - return message_dict; - }) - .def("__repr__", &PyMessage::repr) - .def("__str__", &PyMessage::repr); + nb::class_(m, "Message") + .def_ro("header", &PyMessage::header); + + + //nb::class_(m, "Message") + // .def_ro("body", &PyMessage::message_body) + // .def_ro("header", &PyMessage::header) + // .def_ro("name", &PyMessage::name) + // .def("to_dict", + // [](const PyMessage& self) { + // nb::dict message_dict; + // message_dict["header"] = self.header.attr("to_dict")(); + // message_dict["body"] = self.message_body.attr("to_dict")(); + // return message_dict; + // }) + // .def("__repr__", &PyMessage::repr) + // .def("__str__", &PyMessage::repr); nb::class_(m, "FieldContainer") .def_rw("value", &FieldContainer::fieldValue) @@ -275,13 +279,13 @@ void init_novatel_message_decoder(nb::module_& m) } nb::object body_pyinst = nb::inst_alloc(body_pytype); - PyMessageBody* body_cinst = nb::inst_ptr(body_pyinst); - new (body_cinst) PyMessageBody(std::move(fields), parent_db, message_name); + PyField* body_cinst = nb::inst_ptr(body_pyinst); + new (body_cinst) PyField(std::move(fields), parent_db, message_name); nb::inst_mark_ready(body_pyinst); - auto message_pyinst = std::make_shared(body_pyinst, header, message_name); + //auto message_pyinst = std::make_shared(body_pyinst, header, message_name); - return nb::make_tuple(status, message_pyinst); + return nb::make_tuple(status, body_pyinst); }, "message_body"_a, "decoded_header"_a, "metadata"_a) .def( @@ -292,7 +296,7 @@ void init_novatel_message_decoder(nb::module_& m) std::string body_str(message_body.c_str(), message_body.size()); const char* data_ptr = body_str.c_str(); STATUS status = static_cast(&decoder)->TestDecodeAscii(msg_def_fields, &data_ptr, fields); - return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN")); + return nb::make_tuple(status, PyField(std::move(fields), get_parent_db(decoder), "UNKNOWN")); }, "msg_def_fields"_a, "message_body"_a) .def( @@ -303,7 +307,7 @@ void init_novatel_message_decoder(nb::module_& m) const char* data_ptr = message_body.c_str(); STATUS status = static_cast(&decoder)->TestDecodeBinary(msg_def_fields, reinterpret_cast(&data_ptr), fields, message_length); - return nb::make_tuple(status, PyMessageBody(std::move(fields), get_parent_db(decoder), "UNKNOWN")); + return nb::make_tuple(status, PyField(std::move(fields), get_parent_db(decoder), "UNKNOWN")); }, "msg_def_fields"_a, "message_body"_a, "message_length"_a); } diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index dbb512b00..95c4b980d 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -18,10 +18,10 @@ struct PyGpsTime TIME_STATUS time_status{TIME_STATUS::UNKNOWN}; }; -struct PyMessageBody +struct PyField { std::string name; - explicit PyMessageBody(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name); + explicit PyField(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_); nb::dict& get_values() const; nb::dict& get_fields() const; nb::dict to_dict() const; @@ -40,20 +40,29 @@ struct PyMessageBody PyMessageDatabase::ConstPtr parent_db_; }; -struct PyMessage +struct PyMessage : public PyField { - nb::object message_body; + public: nb::object header; - std::string name; - - PyMessage(nb::object message_body_, nb::object header_, std::string name_) : message_body(message_body_), header(header_), name(name_) {} - std::string repr() const - { - std::stringstream repr; - repr << ""; - return repr.str(); - } + PyMessage(nb::object header_, std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) + : PyField(std::move(fields_), std::move(parent_db_), std::move(name_)), header(std::move(header_)) {} }; +//struct PyMessage +//{ +// nb::object message_body; +// nb::object header; +// std::string name; +// +// PyMessage(nb::object message_body_, nb::object header_, std::string name_) : message_body(message_body_), header(header_), name(name_) {} +// +// std::string repr() const +// { +// std::stringstream repr; +// repr << ""; +// return repr.str(); +// } +//}; + } // namespace novatel::edie::oem From 0d356167f558861a1da531f93c7144cbe0809f82 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 6 Feb 2025 16:29:30 -0700 Subject: [PATCH 48/67] Integrate body into message --- python/bindings/message_database.cpp | 9 +- python/bindings/message_decoder.cpp | 42 +-- python/bindings/py_decoded_message.hpp | 2 +- python/novatel_edie/__init__.py | 1 + .../novatel_edie_customizer/stubgen.py | 8 +- python/test/test_decode_encode.py | 265 +++++++++--------- 6 files changed, 159 insertions(+), 168 deletions(-) diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index e9903c399..dfa8342c4 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -275,19 +275,20 @@ void PyMessageDatabase::UpdatePythonMessageTypes() // get type constructor nb::object type_constructor = nb::module_::import_("builtins").attr("type"); // specify the python superclasses for the new message and message body types - nb::tuple body_type_tuple = nb::make_tuple(nb::type()); + nb::tuple message_type_tuple = nb::make_tuple(nb::type()); + nb::tuple field_type_tuple = nb::make_tuple(nb::type()); // provide no additional attributes via `__dict__` nb::dict type_dict = nb::dict(); // add message and message body types for each message definition for (const auto& message_def : MessageDefinitions()) { - nb::object msg_body_def = type_constructor(message_def->name, body_type_tuple, type_dict); + nb::object msg_body_def = type_constructor(message_def->name, message_type_tuple, type_dict); messages_by_name[message_def->name] = msg_body_def; // add additional MessageBody types for each field array element within the message definition - AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name, type_constructor, body_type_tuple, type_dict); + AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name, type_constructor, field_type_tuple, type_dict); } // provide UNKNOWN types for undecodable messages - nb::object default_msg_body_def = type_constructor("UNKNOWN", body_type_tuple, type_dict); + nb::object default_msg_body_def = type_constructor("UNKNOWN", message_type_tuple, type_dict); messages_by_name["UNKNOWN"] = default_msg_body_def; } diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 533be0e19..e72cbef8a 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -218,10 +218,17 @@ void init_novatel_message_decoder(nb::module_& m) nb::module_ builtins = nb::module_::import_("builtins"); nb::handle super = builtins.attr("super"); nb::handle type = builtins.attr("type"); - // get base MessageBody type from concrete instance - nb::handle body_type = (type(self).attr("__bases__"))[0]; + + nb::handle current_type = type(self); + std::string current_type_name = nb::cast(current_type.attr("__name__")); + while (current_type_name != "Field") + { + current_type = (current_type.attr("__bases__"))[0]; + current_type_name = nb::cast(current_type.attr("__name__")); + } + // retrieve base list based on superclass method - nb::object super_obj = super(body_type, self); + nb::object super_obj = super(current_type, self); nb::list base_list = nb::cast(super_obj.attr("__dir__")()); // add dynamic fields to the list PyField* body = nb::inst_ptr(self); @@ -231,22 +238,15 @@ void init_novatel_message_decoder(nb::module_& m) }); nb::class_(m, "Message") - .def_ro("header", &PyMessage::header); - - - //nb::class_(m, "Message") - // .def_ro("body", &PyMessage::message_body) - // .def_ro("header", &PyMessage::header) - // .def_ro("name", &PyMessage::name) - // .def("to_dict", - // [](const PyMessage& self) { - // nb::dict message_dict; - // message_dict["header"] = self.header.attr("to_dict")(); - // message_dict["body"] = self.message_body.attr("to_dict")(); - // return message_dict; - // }) - // .def("__repr__", &PyMessage::repr) - // .def("__str__", &PyMessage::repr); + .def_ro("header", &PyMessage::header) + .def( + "to_dict", [](const PyMessage& self, bool include_header) { + nb::dict dict = self.to_dict(); + if (include_header) { dict["header"] = self.header.attr("to_dict")(); } + return dict; + }, + "include_header"_a = true, + "Convert the message and its sub-messages into a dict"); nb::class_(m, "FieldContainer") .def_rw("value", &FieldContainer::fieldValue) @@ -279,8 +279,8 @@ void init_novatel_message_decoder(nb::module_& m) } nb::object body_pyinst = nb::inst_alloc(body_pytype); - PyField* body_cinst = nb::inst_ptr(body_pyinst); - new (body_cinst) PyField(std::move(fields), parent_db, message_name); + PyMessage* body_cinst = nb::inst_ptr(body_pyinst); + new (body_cinst) PyMessage(fields, parent_db, message_name, header); nb::inst_mark_ready(body_pyinst); //auto message_pyinst = std::make_shared(body_pyinst, header, message_name); diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 95c4b980d..dcaefea9a 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -45,7 +45,7 @@ struct PyMessage : public PyField public: nb::object header; - PyMessage(nb::object header_, std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) + PyMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_, nb::object header_) : PyField(std::move(fields_), std::move(parent_db_), std::move(name_)), header(std::move(header_)) {} }; diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 4dd99afa0..687835eee 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -10,6 +10,7 @@ OEM4_BINARY_SYNC_LENGTH, OEM4_SHORT_ASCII_SYNC_LENGTH, OEM4_ASCII_SYNC_LENGTH, OEM4_SHORT_BINARY_SYNC_LENGTH, OEM4_BINARY_SYNC1, OEM4_BINARY_SYNC2, OEM4_BINARY_SYNC3, string_to_encode_format, pretty_version, + Header, Field, Message, JsonDbReader, MessageDatabase, get_default_database, Oem4BinaryHeader, Oem4BinaryShortHeader, MetaData, MessageData, MessageDefinition, BaseField, Framer, Filter, HeaderDecoder, MessageDecoder, DecoderException, Encoder, Commander, Parser, FileParser, diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py index 0d031528a..2e87a84ca 100644 --- a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py +++ b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py @@ -160,7 +160,7 @@ class will be based on this. # Create MessageBodyField type hint name = f'{parent}_{field_array_def["name"]}_Field' - type_hint = f'class {name}(MessageBody):\n' + type_hint = f'class {name}(Field):\n' for field in field_array_def['fields']: python_type = self._get_field_pytype(field, parent) type_hint += (' @property\n' @@ -191,13 +191,11 @@ def _convert_message_def(self, message_def: dict) -> str: class {name}(Message): @property def header(self) -> Header: ... - @property - def body(self) -> {name}_Body: ... """) # Create the MessageBody type hint name = message_def["name"] - body_hint = f'class {name}(MessageBody):\n' + body_hint = f'class {name}(Message):\n' fields = message_def['fields'][message_def['latestMsgDefCrc']] if not fields: body_hint += ' pass\n\n' @@ -223,7 +221,7 @@ def get_message_stubs(self) -> str: A string containing type hint stubs for all messages in the database. """ stub_str = 'from typing import Any\n\n' - stub_str += 'from novatel_edie import Header, Message, MessageBody, SatelliteId\n' + stub_str += 'from novatel_edie import Header, Field, Message, SatelliteId\n' stub_str += 'from novatel_edie.enums import *\n\n' messages = self.database['messages'] diff --git a/python/test/test_decode_encode.py b/python/test/test_decode_encode.py index 4c8e00267..b2e05c4ed 100644 --- a/python/test/test_decode_encode.py +++ b/python/test/test_decode_encode.py @@ -294,37 +294,36 @@ def test_ascii_log_roundtrip_gloephem(helper): ret_code, message_data, message = helper.DecodeEncode(ENCODE_FORMAT.FLATTENED_BINARY, log, return_message=True) assert ret_code == Result.SUCCESS - glo_ephemeris = message.body - assert isinstance(glo_ephemeris, GLOEPHEMERIS) - assert glo_ephemeris.sloto == 51 - assert glo_ephemeris.freqo == 0 - assert glo_ephemeris.sat_type == 1 - assert glo_ephemeris.false_iod == 80 - assert glo_ephemeris.ephem_week == 2168 - assert glo_ephemeris.ephem_time == 161118000 - assert glo_ephemeris.time_offset == 10782 - assert glo_ephemeris.nt == 573 - assert glo_ephemeris.GLOEPHEMERIS_reserved == 0 - assert glo_ephemeris.GLOEPHEMERIS_reserved_9 == 0 - assert glo_ephemeris.issue == 95 - assert glo_ephemeris.broadcast_health == 0 - assert glo_ephemeris.pos_x == -2.3917966796875000e+07 - assert glo_ephemeris.pos_y == 4.8163881835937500e+06 - assert glo_ephemeris.pos_z == 7.4258510742187500e+06 - assert glo_ephemeris.vel_x == -1.0062713623046875e+03 - assert glo_ephemeris.vel_y == 1.8321990966796875e+02 - assert glo_ephemeris.vel_z == -3.3695755004882813e+03 - assert glo_ephemeris.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) - assert glo_ephemeris.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) - assert glo_ephemeris.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) - assert glo_ephemeris.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) - assert glo_ephemeris.delta_tau == 5.587935448e-09 - assert glo_ephemeris.gamma == approx(0.00000000000000000, abs=0.0000000000000001) - assert glo_ephemeris.tk == 84600 - assert glo_ephemeris.p == 3 - assert glo_ephemeris.ft == 2 - assert glo_ephemeris.age == 0 - assert glo_ephemeris.flags == 13 + assert isinstance(message, GLOEPHEMERIS) + assert message.sloto == 51 + assert message.freqo == 0 + assert message.sat_type == 1 + assert message.false_iod == 80 + assert message.ephem_week == 2168 + assert message.ephem_time == 161118000 + assert message.time_offset == 10782 + assert message.nt == 573 + assert message.GLOEPHEMERIS_reserved == 0 + assert message.GLOEPHEMERIS_reserved_9 == 0 + assert message.issue == 95 + assert message.broadcast_health == 0 + assert message.pos_x == -2.3917966796875000e+07 + assert message.pos_y == 4.8163881835937500e+06 + assert message.pos_z == 7.4258510742187500e+06 + assert message.vel_x == -1.0062713623046875e+03 + assert message.vel_y == 1.8321990966796875e+02 + assert message.vel_z == -3.3695755004882813e+03 + assert message.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) + assert message.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) + assert message.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) + assert message.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) + assert message.delta_tau == 5.587935448e-09 + assert message.gamma == approx(0.00000000000000000, abs=0.0000000000000001) + assert message.tk == 84600 + assert message.p == 3 + assert message.ft == 2 + assert message.age == 0 + assert message.flags == 13 def test_ascii_log_roundtrip_loglist(helper): @@ -506,29 +505,28 @@ def test_flat_binary_log_decode_bestpos(helper): assert log_header.receiver_sw_version == 32768 # SOL_COMPUTED SINGLE 51.15043711386 -114.03067767000 1097.2099 -17.0000 WGS84 0.9038 0.8534 1.7480 \"\" 0.000 0.000 35 30 30 30 00 06 39 33\r\n" - bestpos = message.body - assert isinstance(bestpos, BESTPOS) - assert bestpos.solution_status == ne.enums.SolStatus.SOL_COMPUTED - assert bestpos.position_type == ne.enums.SolType.SINGLE - assert bestpos.latitude == 51.15043711386 - assert bestpos.longitude == -114.03067767000 - assert bestpos.orthometric_height == 1097.2099 - assert bestpos.undulation == -17.0000 - assert bestpos.datum_id == ne.enums.Datum.WGS84 - assert bestpos.latitude_std_dev == approx(0.9038, abs=1e-5) - assert bestpos.longitude_std_dev == approx(0.8534, abs=1e-5) - assert bestpos.height_std_dev == approx(1.7480, abs=1e-5) - # assert bestpos.base_id == "" - assert bestpos.diff_age == 0.000 - assert bestpos.solution_age == 0.000 - assert bestpos.num_svs == 35 - assert bestpos.num_soln_svs == 30 - assert bestpos.num_soln_L1_svs == 30 - assert bestpos.num_soln_multi_svs == 30 - assert bestpos.extended_solution_status2 == 0x00 - assert bestpos.ext_sol_stat == 0x06 - assert bestpos.gal_and_bds_mask == 0x39 - assert bestpos.gps_and_glo_mask == 0x33 + assert isinstance(message, BESTPOS) + assert message.solution_status == ne.enums.SolStatus.SOL_COMPUTED + assert message.position_type == ne.enums.SolType.SINGLE + assert message.latitude == 51.15043711386 + assert message.longitude == -114.03067767000 + assert message.orthometric_height == 1097.2099 + assert message.undulation == -17.0000 + assert message.datum_id == ne.enums.Datum.WGS84 + assert message.latitude_std_dev == approx(0.9038, abs=1e-5) + assert message.longitude_std_dev == approx(0.8534, abs=1e-5) + assert message.height_std_dev == approx(1.7480, abs=1e-5) + # assert message.base_id == "" + assert message.diff_age == 0.000 + assert message.solution_age == 0.000 + assert message.num_svs == 35 + assert message.num_soln_svs == 30 + assert message.num_soln_L1_svs == 30 + assert message.num_soln_multi_svs == 30 + assert message.extended_solution_status2 == 0x00 + assert message.ext_sol_stat == 0x06 + assert message.gal_and_bds_mask == 0x39 + assert message.gps_and_glo_mask == 0x33 def test_flat_binary_log_decode_gloephema(helper): @@ -554,37 +552,36 @@ def test_flat_binary_log_decode_gloephema(helper): assert log_header.msg_def_crc == 0x8d29 assert log_header.receiver_sw_version == 32768 - gloephemeris = message.body - assert isinstance(gloephemeris, GLOEPHEMERIS) - assert gloephemeris.sloto == 51 - assert gloephemeris.freqo == 0 - assert gloephemeris.sat_type == 1 - assert gloephemeris.false_iod == 80 - assert gloephemeris.ephem_week == 2168 - assert gloephemeris.ephem_time == 161118000 - assert gloephemeris.time_offset == 10782 - assert gloephemeris.nt == 573 - assert gloephemeris.GLOEPHEMERIS_reserved == 0 - assert gloephemeris.GLOEPHEMERIS_reserved_9 == 0 - assert gloephemeris.issue == 95 - assert gloephemeris.broadcast_health == 0 - assert gloephemeris.pos_x == -2.3917966796875000e+07 - assert gloephemeris.pos_y == 4.8163881835937500e+06 - assert gloephemeris.pos_z == 7.4258510742187500e+06 - assert gloephemeris.vel_x == -1.0062713623046875e+03 - assert gloephemeris.vel_y == 1.8321990966796875e+02 - assert gloephemeris.vel_z == -3.3695755004882813e+03 - assert gloephemeris.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) - assert gloephemeris.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) - assert gloephemeris.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) - assert gloephemeris.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) - assert gloephemeris.delta_tau == 5.587935448e-09 - assert gloephemeris.gamma == approx(0.00000000000000000, abs=0.0000000000000001) - assert gloephemeris.tk == 84600 - assert gloephemeris.p == 3 - assert gloephemeris.ft == 2 - assert gloephemeris.age == 0 - assert gloephemeris.flags == 13 + assert isinstance(message, GLOEPHEMERIS) + assert message.sloto == 51 + assert message.freqo == 0 + assert message.sat_type == 1 + assert message.false_iod == 80 + assert message.ephem_week == 2168 + assert message.ephem_time == 161118000 + assert message.time_offset == 10782 + assert message.nt == 573 + assert message.GLOEPHEMERIS_reserved == 0 + assert message.GLOEPHEMERIS_reserved_9 == 0 + assert message.issue == 95 + assert message.broadcast_health == 0 + assert message.pos_x == -2.3917966796875000e+07 + assert message.pos_y == 4.8163881835937500e+06 + assert message.pos_z == 7.4258510742187500e+06 + assert message.vel_x == -1.0062713623046875e+03 + assert message.vel_y == 1.8321990966796875e+02 + assert message.vel_z == -3.3695755004882813e+03 + assert message.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) + assert message.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) + assert message.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) + assert message.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) + assert message.delta_tau == 5.587935448e-09 + assert message.gamma == approx(0.00000000000000000, abs=0.0000000000000001) + assert message.tk == 84600 + assert message.p == 3 + assert message.ft == 2 + assert message.age == 0 + assert message.flags == 13 def test_flat_binary_log_decode_portstatsb(helper): @@ -601,8 +598,8 @@ def test_flat_binary_log_decode_portstatsb(helper): assert compare_binary_headers(test_log_header, log_header) # Check the populated parts of the log - assert len(portstats.body.port_statistics) == 23 - for port_statistics, expected_port in zip(portstats.body.port_statistics, portstats_port_fields): + assert len(portstats.port_statistics) == 23 + for port_statistics, expected_port in zip(portstats.port_statistics, portstats_port_fields): assert port_statistics.port == expected_port @@ -620,8 +617,8 @@ def test_flat_binary_log_decode_psrdopb(helper): assert compare_binary_headers(test_log_header, log_header) # Check the populated parts of the log - assert len(psrdop.body.sats) == 35 - for sat, expected in zip(psrdop.body.sats, psrdop_sat_fields): + assert len(psrdop.sats) == 35 + for sat, expected in zip(psrdop.sats, psrdop_sat_fields): assert sat == expected @@ -637,8 +634,8 @@ def test_flat_binary_log_decode_validmodelsb(helper): assert compare_binary_headers(test_log_header, log_header) # Check the populated parts of the log - assert len(validmodels.body.models) == 1 - for models in validmodels.body.models: + assert len(validmodels.models) == 1 + for models in validmodels.models: assert models.model == "FFNRNNCBN" @@ -672,7 +669,7 @@ def test_flat_binary_log_decode_version(helper): assert log_header.receiver_sw_version == 32768 # Check the populated parts of the log - versions = version.body.versions + versions = version.versions assert len(versions) == 4 # Check GPSCARD fields @@ -741,7 +738,7 @@ def test_flat_binary_log_decode_versiona(helper): assert log_header.receiver_sw_version == 16248 # Check the populated parts of the log - versions = version.body.versions + versions = version.versions assert len(versions) == 8 # Check GPSCARD fields @@ -1035,8 +1032,8 @@ def ASSERT_SHORT_HEADER_EQ(short_header_, header_): def ASSERT_BESTSATS_EQ(message1, message2): - assert len(message1.body.satellite_entries) == len(message2.body.satellite_entries) - for satellite_entries1, satellite_entries2 in zip(message1.body.satellite_entries, message2.body.satellite_entries): + assert len(message1.satellite_entries) == len(message2.satellite_entries) + for satellite_entries1, satellite_entries2 in zip(message1.satellite_entries, message2.satellite_entries): assert satellite_entries1.system_type == satellite_entries2.system_type assert satellite_entries1.id.prn_or_slot == satellite_entries2.id.prn_or_slot assert satellite_entries1.id.frequency_channel == satellite_entries2.id.frequency_channel @@ -1045,38 +1042,36 @@ def ASSERT_BESTSATS_EQ(message1, message2): def ASSERT_BESTPOS_EQ(message1, message2): - body1 = message1.body - body2 = message2.body - assert body1.solution_status == body2.solution_status - assert body1.position_type == body2.position_type - assert body1.latitude == approx(body2.latitude, abs=1e-11) - assert body1.longitude == approx(body2.longitude, abs=1e-11) - if hasattr(body1, "orthometric_height") and hasattr(body2, "orthometric_height"): - assert body1.orthometric_height == approx(body2.orthometric_height, abs=1e-4) + assert message1.solution_status == message2.solution_status + assert message1.position_type == message2.position_type + assert message1.latitude == approx(message2.latitude, abs=1e-11) + assert message1.longitude == approx(message2.longitude, abs=1e-11) + if hasattr(message1, "orthometric_height") and hasattr(message2, "orthometric_height"): + assert message1.orthometric_height == approx(message2.orthometric_height, abs=1e-4) else: - assert body1.height == approx(body2.height, abs=1e-4) - assert body1.undulation == approx(body2.undulation, rel=1e-6) - assert body1.datum_id == body2.datum_id - assert body1.latitude_std_dev == approx(body2.latitude_std_dev, abs=1e-4) - assert body1.longitude_std_dev == approx(body2.longitude_std_dev, abs=1e-4) - assert body1.height_std_dev == approx(body2.height_std_dev, abs=1e-4) - assert body1.base_id == body2.base_id - assert body1.diff_age == approx(body2.diff_age, rel=1e-6) - assert body1.solution_age == approx(body2.solution_age, rel=1e-6) - assert body1.num_svs == body2.num_svs - assert body1.num_soln_svs == body2.num_soln_svs - assert body1.num_soln_L1_svs == body2.num_soln_L1_svs - assert body1.num_soln_multi_svs == body2.num_soln_multi_svs - if hasattr(body1, "extended_solution_status2") and hasattr(body2, "extended_solution_status2"): - assert body1.extended_solution_status2 == body2.extended_solution_status2 - assert body1.ext_sol_stat == body2.ext_sol_stat - assert body1.gal_and_bds_mask == body2.gal_and_bds_mask - assert body1.gps_and_glo_mask == body2.gps_and_glo_mask + assert message1.height == approx(message2.height, abs=1e-4) + assert message1.undulation == approx(message2.undulation, rel=1e-6) + assert message1.datum_id == message2.datum_id + assert message1.latitude_std_dev == approx(message2.latitude_std_dev, abs=1e-4) + assert message1.longitude_std_dev == approx(message2.longitude_std_dev, abs=1e-4) + assert message1.height_std_dev == approx(message2.height_std_dev, abs=1e-4) + assert message1.base_id == message2.base_id + assert message1.diff_age == approx(message2.diff_age, rel=1e-6) + assert message1.solution_age == approx(message2.solution_age, rel=1e-6) + assert message1.num_svs == message2.num_svs + assert message1.num_soln_svs == message2.num_soln_svs + assert message1.num_soln_L1_svs == message2.num_soln_L1_svs + assert message1.num_soln_multi_svs == message2.num_soln_multi_svs + if hasattr(message1, "extended_solution_status2") and hasattr(message2, "extended_solution_status2"): + assert message1.extended_solution_status2 == message2.extended_solution_status2 + assert message1.ext_sol_stat == message2.ext_sol_stat + assert message1.gal_and_bds_mask == message2.gal_and_bds_mask + assert message1.gps_and_glo_mask == message2.gps_and_glo_mask def ASSERT_LOGLIST_EQ(message1, message2): - assert len(message1.body.log_list) == len(message2.body.log_list) - for log_list1, log_list2 in zip(message1.body.log_list, message2.body.log_list): + assert len(message1.log_list) == len(message2.log_list) + for log_list1, log_list2 in zip(message1.log_list, message2.log_list): assert log_list1.log_port_address == log_list2.log_port_address assert log_list1.message_id == log_list2.message_id assert log_list1.trigger == log_list2.trigger @@ -1086,23 +1081,19 @@ def ASSERT_LOGLIST_EQ(message1, message2): def ASSERT_RAWGPSSUBFRAME_EQ(message1, message2): - body1 = message1.body - body2 = message2.body - assert body1.frame_decoder_number == body2.frame_decoder_number - assert body1.satellite_id == body2.satellite_id - assert body1.sub_frame_id == body2.sub_frame_id - assert body1.raw_sub_frame_data == body2.raw_sub_frame_data - assert body1.signal_channel_number == body2.signal_channel_number + assert message1.frame_decoder_number == message2.frame_decoder_number + assert message1.satellite_id == message2.satellite_id + assert message1.sub_frame_id == message2.sub_frame_id + assert message1.raw_sub_frame_data == message2.raw_sub_frame_data + assert message1.signal_channel_number == message2.signal_channel_number def ASSERT_TRACKSTAT_EQ(message1, message2): - body1 = message1.body - body2 = message2.body - assert body1.position_status == body2.position_status - assert body1.position_type == body2.position_type - assert body1.tracking_elevation_cutoff == body2.tracking_elevation_cutoff - assert len(body1.chan_status) == len(body2.chan_status) - for chan_status1, chan_status2 in zip(body1.chan_status, body2.chan_status): + assert message1.position_status == message2.position_status + assert message1.position_type == message2.position_type + assert message1.tracking_elevation_cutoff == message2.tracking_elevation_cutoff + assert len(message1.chan_status) == len(message2.chan_status) + for chan_status1, chan_status2 in zip(message1.chan_status, message2.chan_status): assert chan_status1.prn == chan_status2.prn assert chan_status1.freq == chan_status2.freq assert chan_status1.channel_status == chan_status2.channel_status From 15c7940343cdc7346d277f8fbbf73e1818a2ab76 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Thu, 6 Feb 2025 16:33:23 -0700 Subject: [PATCH 49/67] Fix example --- python/examples/converter_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index 243006c89..604d8b132 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -116,7 +116,7 @@ def main(): # Get info from the log. if isinstance(message, RANGE): - obs = message.body.obs + obs = message.obs for ob in obs: value = ob.psr pass From 61a53298a4024181ac23fa7074ef6b1eed1fb172 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 08:50:27 -0700 Subject: [PATCH 50/67] Delete dead code --- python/bindings/py_decoded_message.hpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index dcaefea9a..8bd9ca67c 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -49,20 +49,4 @@ struct PyMessage : public PyField : PyField(std::move(fields_), std::move(parent_db_), std::move(name_)), header(std::move(header_)) {} }; -//struct PyMessage -//{ -// nb::object message_body; -// nb::object header; -// std::string name; -// -// PyMessage(nb::object message_body_, nb::object header_, std::string name_) : message_body(message_body_), header(header_), name(name_) {} -// -// std::string repr() const -// { -// std::stringstream repr; -// repr << ""; -// return repr.str(); -// } -//}; - } // namespace novatel::edie::oem From 67687d147408b6f938a7735c8adb55307add9848 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 11:18:33 -0700 Subject: [PATCH 51/67] Update CLI --- pyproject.toml | 3 +++ python/novatel_edie/__init__.py | 24 ++++++++++++++++++++++++ python/novatel_edie/enums.py | 24 ++++++++++++++++++++++++ python/novatel_edie/messages.py | 24 ++++++++++++++++++++++++ 4 files changed, 75 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 614024bb1..7a2b98b59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,3 +58,6 @@ archs = "auto64" # Needed for full C++17 support [tool.cibuildwheel.macos.environment] MACOSX_DEPLOYMENT_TARGET = "10.15" + +[project.scripts] +novatel_edie = "novatel_edie.cli:app" diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 687835eee..8fab79bf6 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -1,3 +1,27 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + import importlib_resources from .bindings import ( diff --git a/python/novatel_edie/enums.py b/python/novatel_edie/enums.py index 50ec26f0a..e8fee6e6a 100644 --- a/python/novatel_edie/enums.py +++ b/python/novatel_edie/enums.py @@ -1 +1,25 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + from .bindings.enums import * diff --git a/python/novatel_edie/messages.py b/python/novatel_edie/messages.py index cc657ab9c..74f63b195 100644 --- a/python/novatel_edie/messages.py +++ b/python/novatel_edie/messages.py @@ -1 +1,25 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + from .bindings.messages import * From 9507222a610b2b488417acc49216747541207d76 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 11:26:22 -0700 Subject: [PATCH 52/67] Delete old code --- pyproject.toml | 1 + python/CMakeLists.txt | 4 +- .../novatel_edie_customizer/__init__.py | 23 -- .../novatel_edie_customizer/__main__.py | 27 -- .../novatel_edie_customizer/cli.py | 39 --- .../novatel_edie_customizer/installer.py | 139 ---------- .../novatel_edie_customizer/stubgen.py | 255 ------------------ python/novatel_edie_customizer/pyproject.toml | 13 - 8 files changed, 3 insertions(+), 498 deletions(-) delete mode 100644 python/novatel_edie_customizer/novatel_edie_customizer/__init__.py delete mode 100644 python/novatel_edie_customizer/novatel_edie_customizer/__main__.py delete mode 100644 python/novatel_edie_customizer/novatel_edie_customizer/cli.py delete mode 100644 python/novatel_edie_customizer/novatel_edie_customizer/installer.py delete mode 100644 python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py delete mode 100644 python/novatel_edie_customizer/pyproject.toml diff --git a/pyproject.toml b/pyproject.toml index 7a2b98b59..dfb1396e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ license = {text = "MIT"} requires-python = ">=3.8" dependencies = [ "importlib_resources", + "typer >=0.15" ] [build-system] diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 2f128b15b..f4de3ecef 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -64,10 +64,10 @@ nanobind_add_stub( add_custom_target( dynamic_stubs ALL - COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie_customizer/novatel_edie_customizer/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs + COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs COMMENT "Generating dynamic stubs" DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie_customizer/novatel_edie_customizer/stubgen.py + ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ) diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/__init__.py b/python/novatel_edie_customizer/novatel_edie_customizer/__init__.py deleted file mode 100644 index a104f7c5a..000000000 --- a/python/novatel_edie_customizer/novatel_edie_customizer/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -MIT - -Copyright (c) 2023 NovAtel Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py b/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py deleted file mode 100644 index b2ccb37ed..000000000 --- a/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT - -Copyright (c) 2023 NovAtel Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -if __name__ == "__main__": - from .cli import app - app() diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/cli.py b/python/novatel_edie_customizer/novatel_edie_customizer/cli.py deleted file mode 100644 index 541bf9578..000000000 --- a/python/novatel_edie_customizer/novatel_edie_customizer/cli.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -MIT - -Copyright (c) 2023 NovAtel Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import typer - -from .stubgen import generate_stubs -from .installer import install_custom - -app = typer.Typer() - -# sub_app = typer.Typer() -# sub_app.command()(generate_stubs) - -app.command()(generate_stubs) -app.command()(install_custom) - -if __name__ == "__main__": - app() diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/installer.py b/python/novatel_edie_customizer/novatel_edie_customizer/installer.py deleted file mode 100644 index 3e8e3a937..000000000 --- a/python/novatel_edie_customizer/novatel_edie_customizer/installer.py +++ /dev/null @@ -1,139 +0,0 @@ -""" -MIT - -Copyright (c) 2023 NovAtel Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -A module concerning the installation of a custom package. -""" - -import os -import shutil -import sys -from contextlib import contextmanager -import subprocess - -import typer -from typing_extensions import Annotated - -from novatel_edie_customizer.stubgen import StubGenerator - -@contextmanager -def open_library_clone(library: str): - """Creates a clone of a specified library within a temporary directory. - - Args: - library: The name of the library to clone. Must be within - the current Python environment. - """ - packages_path = os.path.join(sys.exec_prefix, 'Lib', 'site-packages') - library_path = os.path.join(packages_path, library) - if not os.path.exists(library_path): - raise FileNotFoundError(f"Source library {library} not found.") - library_dist_info = [ - d for d in os.listdir(packages_path) - if d.startswith(library) and d.endswith('.dist-info')] - if not library_dist_info: - raise FileNotFoundError( - f"No .dist-info directory found for library {library}.") - library_dist_info = library_dist_info[0] - library_dist_info_path = os.path.join(packages_path, library_dist_info) - - temp_dir = os.path.join(os.getcwd(), 'temp_dir') - destination_dir = os.path.join(temp_dir, 'wheel') - - cwd = os.getcwd() - - try: - if not os.path.exists(destination_dir): - os.makedirs(destination_dir) - shutil.copytree( - library_path, - os.path.join(destination_dir, library), - dirs_exist_ok=True) - shutil.copytree( - library_dist_info_path, - os.path.join(destination_dir, library_dist_info), - dirs_exist_ok=True) - - os.chdir(temp_dir) - yield - finally: - os.chdir(cwd) - shutil.rmtree(temp_dir) - -def copy_file(file_path: str, destination_path: str = None): - """Copies a file to the current working directory. - - Args: - file_path: The path of the file to copy. - destination_path: A relative path within current working directory - to copy the file to. - """ - if not os.path.exists(file_path): - raise FileNotFoundError(f"File {file_path} not found.") - - destination_path = (os.path.basename(file_path) - if destination_path is None - else destination_path) - - destination_path = os.path.join(os.getcwd(), destination_path) - shutil.copy2(file_path, destination_path) - -def install_package(): - """Installs a package in the current working directory.""" - try: - subprocess.check_call([ - sys.executable, '-m', 'wheel', 'pack', './wheel']) - pass - except subprocess.CalledProcessError as e: - print(f"Failed to pack package: {e}") - wheel_files = [f for f in os.listdir('.') if f.endswith('.whl')] - for wheel_file in wheel_files: - try: - subprocess.check_call([ - sys.executable, '-m', 'pip', - 'install', wheel_file, '--force-reinstall']) - except subprocess.CalledProcessError as e: - print(f"Failed to install {wheel_file}: {e}") - -def install_custom( - database: Annotated[ - str, - typer.Argument(help='A path to a database file.') - ]): - """Create a custom installation of novatel_edie based on the provided DB. - - The custom installation will have all messages and enums of the provided - database be directly importable from the 'enums' and 'messages' submodules. - - Args: - database: A path to a database file. - """ - database = os.path.abspath(database) - library_name = 'novatel_edie' - with open_library_clone(library_name): - copy_file( - database, os.path.join('wheel', library_name, 'database.json')) - - database = StubGenerator(database) - database.write_stub_files(os.path.join('wheel', library_name)) - - install_package() diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py deleted file mode 100644 index 2e87a84ca..000000000 --- a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py +++ /dev/null @@ -1,255 +0,0 @@ -""" -MIT - -Copyright (c) 2023 NovAtel Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -A module concerning the generation of type hint stub files for the novatel_edie package. -""" -import os -import json -import re -import textwrap -from typing import Union - -import typer -from typing_extensions import Annotated - -class StubGenerator: - """Generator of type hint stub files for the novatel_edie package.""" - data_type_to_pytype = { - 'INT': 'int', - 'UNIT': 'int', - 'BOOL': 'bool', - 'CHAR': 'int', - 'UCHAR': 'int', - 'SHORT': 'int', - 'USHORT': 'int', - 'LONG': 'int', - 'ULONG': 'int', - 'LONGLONG': 'int', - 'ULONGLONG': 'int', - 'FLOAT': 'float', - 'DOUBLE': 'float', - 'HEXBYTE': 'int', - 'SATELLITEID': 'SatelliteId', - 'UNKNOWN': 'bytes' - } - def __init__(self, database: Union[str, dict]): - """Initializes a StubGenerator. - - Args: - database: A path to a message database json file or - a dictionary representation of a database. - """ - if isinstance(database, str): - with open(database, 'r') as f: - database = json.load(f) - if not isinstance(database, dict): - raise TypeError( - 'database must be a dict or a path to a JSON database file.') - self.database = database - self.database['enums_by_id'] = {enum['_id']: enum for enum in database['enums']} - - def write_stub_files(self, file_path: str): - """Writes package stub files to the specified directory. - - Args: - file_path: The directory to write the stub files to. - """ - file_specs = { - 'enums.pyi': self.get_enum_stubs(), - 'messages.pyi': self.get_message_stubs()} - - if not os.path.exists(file_path): - os.makedirs(file_path) - for file_name, file_contents in file_specs.items(): - with open(os.path.join(file_path, file_name), 'w') as f: - f.write(file_contents) - - def _convert_enum_def(self, enum_def) -> str: - """Create a type hint string for an enum definition. - - Args: - enum_def: A dictionary containing an enum definition. - Returns: - A string containing a type hint for the enum definition. - """ - type_hint = f'class {enum_def["name"]}(Enum):\n' - for enumerator in enum_def['enumerators']: - name = enumerator['name'] - name = re.sub(r'[)(\-+./\\]', '_', name) - if name[0].isdigit(): - name = f'_{name}' - type_hint += f' {name} = {enumerator["value"]}\n' - return type_hint - - def get_enum_stubs(self) -> str: - """Get a stub string for all enums in the database. - - Returns: - A string containing type hint stubs for all enums in the database. - """ - stub_str = ('from enum import Enum\n' - 'from typing import Any\n\n') - enums = self.database['enums'] - type_hints = [self._convert_enum_def(enum_def) for enum_def in enums] - type_hints_str = '\n'.join(type_hints) - stub_str += type_hints_str - return stub_str - - def _get_field_pytype(self, field: dict, parent: str) -> str: - """Get a type hint string for a field definition. - - Args: - field: A dictionary containing a field definition. - parent: The name of the parent class. May be used for naming - subfield classes. - Returns: - A string containing a type hint for the field definition. - """ - python_type = None - if field['type'] == 'SIMPLE': - python_type = self.data_type_to_pytype.get(field['dataType']['name']) - if field['type'] == 'ENUM': - enum_def = self.database['enums_by_id'].get(field['enumID']) - python_type = enum_def['name'] if enum_def else 'Any' - if field['type'] == 'STRING': - python_type = 'str' - if (field['type'] in { - 'FIXED_LENGTH_ARRAY', - 'VARIABLE_LENGTH_ARRAY'}): - if field['conversionString'] == r'%s': - python_type = 'str' - else: - python_type = f'list[{self.data_type_to_pytype.get(field["dataType"]["name"])}]' - if field['type'] == 'FIELD_ARRAY': - python_type = f'list[{parent}_{field["name"]}_Field]' - if not python_type: - python_type = 'Any' - return python_type - - def _convert_field_array_def(self, field_array_def: dict, parent: str) -> str: - """Convert a field array definition to a type hint string. - - Args: - field_array_def: A dictionary containing a field array definition. - parent: The name of the parent class. The name for the field array - class will be based on this. - Returns: - A string containing type hint stubs for the field array definition. - """ - subfield_hints = [] - - # Create MessageBodyField type hint - name = f'{parent}_{field_array_def["name"]}_Field' - type_hint = f'class {name}(Field):\n' - for field in field_array_def['fields']: - python_type = self._get_field_pytype(field, parent) - type_hint += (' @property\n' - f' def {field["name"]}(self) -> {python_type}: ...\n\n') - # Create hints for any subfields - if field['type'] == 'FIELD_ARRAY': - subfield_hints.append(self._convert_field_array_def(field, name)) - - # Combine all hints - hints = subfield_hints + [type_hint] - hint_str = '\n'.join(hints) - - return hint_str - - def _convert_message_def(self, message_def: dict) -> str: - """Create a type hint string for a message definition. - - Args: - message_def: A dictionary containing a message definition. - Returns: - A string containing type hint stubs for the message definition. - """ - subfield_hints = [] - - # Create the Message type hint - name = message_def['name'] - message_hint = textwrap.dedent(f"""\ - class {name}(Message): - @property - def header(self) -> Header: ... - """) - - # Create the MessageBody type hint - name = message_def["name"] - body_hint = f'class {name}(Message):\n' - fields = message_def['fields'][message_def['latestMsgDefCrc']] - if not fields: - body_hint += ' pass\n\n' - for field in fields: - python_type = self._get_field_pytype(field, name) - body_hint += ' @property\n' - body_hint += f' def {field["name"]}(self) -> {python_type}: ...\n\n' - - # Create hints for any subfields - if field['type'] == 'FIELD_ARRAY': - subfield_hints.append(self._convert_field_array_def(field, name)) - - # Combine all hints - hints = subfield_hints + [body_hint] - hint_str = '\n'.join(hints) - - return hint_str - - def get_message_stubs(self) -> str: - """Get a stub string for all messages in the database. - - Returns: - A string containing type hint stubs for all messages in the database. - """ - stub_str = 'from typing import Any\n\n' - stub_str += 'from novatel_edie import Header, Field, Message, SatelliteId\n' - stub_str += 'from novatel_edie.enums import *\n\n' - - messages = self.database['messages'] - type_hints = [self._convert_message_def(msg_def) - for msg_def in messages] - type_hints_str = '\n'.join(type_hints) - stub_str += type_hints_str - - return stub_str - - -def generate_stubs( - database: Annotated[ - str, - typer.Argument(help='A path to a database file.') - ], - output_dir: Annotated[ - str, - typer.Argument(help='The directory to write stub files to.') - ] = './stubs' - ): - """Generate type hint stub files for a provided database. - - Args: - database: A path to a database file. - output_dir: The directory to write stub files to. - """ - StubGenerator(database).write_stub_files(output_dir) - -if __name__ == '__main__': - typer.run(generate_stubs) diff --git a/python/novatel_edie_customizer/pyproject.toml b/python/novatel_edie_customizer/pyproject.toml deleted file mode 100644 index d10ecd1b3..000000000 --- a/python/novatel_edie_customizer/pyproject.toml +++ /dev/null @@ -1,13 +0,0 @@ -[project] -name = "novatel_edie_customizer" -description = "Tool for creating a custom installation of novatel_edie package" -version = "0.0.1" -license = {text = "MIT"} -requires-python = ">=3.8" -dependencies = [ - "typer>=0.15" -] - - -[project.scripts] -novatel_edie_customizer = "novatel_edie_customizer.cli:app" From 326a2f3ce95d0ced520d93276a3bbe1187993205 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 11:35:07 -0700 Subject: [PATCH 53/67] Update docs --- python/bindings/py_database.hpp | 5 ++--- python/bindings/py_decoded_message.hpp | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index 1e7a687a5..b073cd5c7 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -34,9 +34,8 @@ class PyMessageDatabase final : public MessageDatabase //! \brief Creates Python types for each component of all message definitions in the database. //! //! A message named "MESSAGE" will be mapped to a Python class named "MESSAGE". - //! The message's body will be mapped to a class named "MESSAGE_Body". - //! A field of that body named "FIELD" will be mapped to a class named "MESSAGE_Body_FIELD_Field". - //! A subfield of that field named "SUBFIELD" will be mapped to a class named "MESSAGE_Body_FIELD_Field_SUBFIELD_Field". + //! A field of that body named "FIELD" will be mapped to a class named "MESSAGE_FIELD_Field". + //! A subfield of that field named "SUBFIELD" will be mapped to a class named "MESSAGE_FIELD_Field_SUBFIELD_Field". //! //! These classes are stored by name in the messages_by_name map. //----------------------------------------------------------------------- diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 8bd9ca67c..03d1b2a69 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -18,6 +18,13 @@ struct PyGpsTime TIME_STATUS time_status{TIME_STATUS::UNKNOWN}; }; +//============================================================================ +//! \class PyField +//! \brief A python representation for a single log message or message field. +//! +//! Contains a vector of FieldContainer objects, which behave like attributes +//! within the Python API. +//============================================================================ struct PyField { std::string name; @@ -40,6 +47,13 @@ struct PyField PyMessageDatabase::ConstPtr parent_db_; }; + +//============================================================================ +//! \class PyMessage +//! \brief A python representation for a single log message. +//! +//! Extends PyField with reference to the Python represenation of a Header. +//============================================================================ struct PyMessage : public PyField { public: From 184dcc4c188276fb580d2bd45e37403dd6e82446 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 14:22:19 -0700 Subject: [PATCH 54/67] expose name for message obs --- python/bindings/message_decoder.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index e72cbef8a..98674c58d 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -207,8 +207,8 @@ void init_novatel_message_decoder(nb::module_& m) .def_rw("status", &PyGpsTime::time_status); nb::class_(m, "Field") - .def_prop_ro("_values", &PyField::get_values) - .def_prop_ro("_fields", &PyField::get_fields) + //.def_prop_ro("_values", &PyField::get_values) + //.def_prop_ro("_fields", &PyField::get_fields) .def("to_dict", &PyField::to_dict, "Convert the message and its sub-messages into a dict") .def("__getattr__", &PyField::getattr, "field_name"_a) .def("__repr__", &PyField::repr) @@ -239,6 +239,7 @@ void init_novatel_message_decoder(nb::module_& m) nb::class_(m, "Message") .def_ro("header", &PyMessage::header) + .def_ro("name", &PyMessage::name) .def( "to_dict", [](const PyMessage& self, bool include_header) { nb::dict dict = self.to_dict(); From 4ae4960fa6f5f758dceabbfddecc11c5d4945b3c Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 15:39:59 -0700 Subject: [PATCH 55/67] Remove indirection --- python/bindings/encoder.cpp | 5 ++- python/bindings/header_decoder.cpp | 46 +++++++++++++++++++++++++- python/bindings/message_decoder.cpp | 4 +-- python/bindings/oem_common.cpp | 40 ---------------------- python/bindings/py_decoded_message.hpp | 13 ++++++-- 5 files changed, 60 insertions(+), 48 deletions(-) diff --git a/python/bindings/encoder.cpp b/python/bindings/encoder.cpp index 3011b74aa..cd2d5494c 100644 --- a/python/bindings/encoder.cpp +++ b/python/bindings/encoder.cpp @@ -23,7 +23,6 @@ void init_novatel_encoder(nb::module_& m) [](oem::Encoder& encoder, const oem::PyMessage& py_message, const oem::MetaDataStruct& metadata, ENCODE_FORMAT format) { MessageDataStruct message_data; - oem::IntermediateHeader* header_cinst = nb::inst_ptr(py_message.header); if (format == ENCODE_FORMAT::JSON) { // Allocate more space for JSON messages. @@ -32,7 +31,7 @@ void init_novatel_encoder(nb::module_& m) uint8_t buffer[MESSAGE_SIZE_MAX * 3]; auto* buf_ptr = reinterpret_cast(&buffer); uint32_t buf_size = MESSAGE_SIZE_MAX * 3; - STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, py_message.fields, message_data, metadata, format); + STATUS status = encoder.Encode(&buf_ptr, buf_size, py_message.header, py_message.fields, message_data, metadata, format); return nb::make_tuple(status, oem::PyMessageData(message_data)); } else @@ -40,7 +39,7 @@ void init_novatel_encoder(nb::module_& m) uint8_t buffer[MESSAGE_SIZE_MAX]; auto buf_ptr = reinterpret_cast(&buffer); uint32_t buf_size = MESSAGE_SIZE_MAX; - STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, py_message.fields, message_data, metadata, format); + STATUS status = encoder.Encode(&buf_ptr, buf_size, py_message.header, py_message.fields, message_data, metadata, format); return nb::make_tuple(status, oem::PyMessageData(message_data)); } }, diff --git a/python/bindings/header_decoder.cpp b/python/bindings/header_decoder.cpp index fa1366573..42586ab9d 100644 --- a/python/bindings/header_decoder.cpp +++ b/python/bindings/header_decoder.cpp @@ -2,13 +2,57 @@ #include "bindings_core.hpp" #include "message_db_singleton.hpp" +#include "py_decoded_message.hpp" namespace nb = nanobind; using namespace nb::literals; using namespace novatel::edie; +using namespace novatel::edie::oem; + +nb::dict PyHeader::to_dict() const +{ + nb::dict header_dict; + header_dict["message_id"] = usMessageId; + header_dict["message_type"] = ucMessageType; + header_dict["port_address"] = uiPortAddress; + header_dict["length"] = usLength; + header_dict["sequence"] = usSequence; + header_dict["idle_time"] = ucIdleTime; + header_dict["time_status"] = uiTimeStatus; + header_dict["week"] = usWeek; + header_dict["milliseconds"] = dMilliseconds; + header_dict["receiver_status"] = uiReceiverStatus; + header_dict["message_definition_crc"] = uiMessageDefinitionCrc; + header_dict["receiver_sw_version"] = usReceiverSwVersion; + return header_dict; +} void init_novatel_header_decoder(nb::module_& m) { + nb::class_(m, "Header") + .def_ro("message_id", &oem::PyHeader::usMessageId) + .def_ro("message_type", &oem::PyHeader::ucMessageType) + .def_ro("port_address", &oem::PyHeader::uiPortAddress) + .def_ro("length", &oem::PyHeader::usLength) + .def_ro("sequence", &oem::PyHeader::usSequence) + .def_ro("idle_time", &oem::PyHeader::ucIdleTime) + .def_ro("time_status", &oem::PyHeader::uiTimeStatus) + .def_ro("week", &oem::PyHeader::usWeek) + .def_ro("milliseconds", &oem::PyHeader::dMilliseconds) + .def_ro("receiver_status", &oem::PyHeader::uiReceiverStatus) + .def_ro("message_definition_crc", &oem::PyHeader::uiMessageDefinitionCrc) + .def_ro("receiver_sw_version", &oem::PyHeader::usReceiverSwVersion) + .def("to_dict", [](const oem::PyHeader& self) { return self.to_dict(); }) + .def("__repr__", [](const nb::handle self) { + auto& header = nb::cast(self); + return nb::str("Header(message_id={!r}, message_type={!r}, port_address={!r}, length={!r}, sequence={!r}, " + "idle_time={!r}, time_status={!r}, week={!r}, milliseconds={!r}, receiver_status={!r}, " + "message_definition_crc={!r}, receiver_sw_version={!r})") + .format(header.usMessageId, header.ucMessageType, header.uiPortAddress, header.usLength, header.usSequence, header.ucIdleTime, + header.uiTimeStatus, header.usWeek, header.dMilliseconds, header.uiReceiverStatus, header.uiMessageDefinitionCrc, + header.usReceiverSwVersion); + }); + nb::class_(m, "HeaderDecoder") .def("__init__", [](oem::HeaderDecoder* t) { new (t) oem::HeaderDecoder(MessageDbSingleton::get()); }) // NOLINT(*.NewDeleteLeaks) .def(nb::init(), "json_db"_a) @@ -17,7 +61,7 @@ void init_novatel_header_decoder(nb::module_& m) .def( "decode", [](const oem::HeaderDecoder& decoder, const nb::bytes& raw_header, oem::MetaDataStruct& metadata) { - oem::IntermediateHeader header; + oem::PyHeader header; STATUS status = decoder.Decode(reinterpret_cast(raw_header.c_str()), header, metadata); return nb::make_tuple(status, header); }, diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 98674c58d..e69147258 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -243,7 +243,7 @@ void init_novatel_message_decoder(nb::module_& m) .def( "to_dict", [](const PyMessage& self, bool include_header) { nb::dict dict = self.to_dict(); - if (include_header) { dict["header"] = self.header.attr("to_dict")(); } + if (include_header) { dict["header"] = self.header.to_dict(); } return dict; }, "include_header"_a = true, @@ -263,7 +263,7 @@ void init_novatel_message_decoder(nb::module_& m) .def_prop_ro("logger", [](oem::MessageDecoder& decoder) { return decoder.GetLogger(); }) .def( "decode", - [](const oem::MessageDecoder& decoder, const nb::bytes& message_body, nb::object header, oem::MetaDataStruct& metadata) { + [](const oem::MessageDecoder& decoder, const nb::bytes& message_body, PyHeader header, oem::MetaDataStruct& metadata) { std::vector fields; STATUS status = decoder.Decode(reinterpret_cast(message_body.c_str()), fields, metadata); PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); diff --git a/python/bindings/oem_common.cpp b/python/bindings/oem_common.cpp index d416769f5..f93d95b5a 100644 --- a/python/bindings/oem_common.cpp +++ b/python/bindings/oem_common.cpp @@ -107,46 +107,6 @@ void init_novatel_common(nb::module_& m) metadata.usMessageId, metadata.uiMessageCrc); }); - nb::class_(m, "Header") - .def(nb::init()) - .def_rw("message_id", &oem::IntermediateHeader::usMessageId) - .def_rw("message_type", &oem::IntermediateHeader::ucMessageType) - .def_rw("port_address", &oem::IntermediateHeader::uiPortAddress) - .def_rw("length", &oem::IntermediateHeader::usLength) - .def_rw("sequence", &oem::IntermediateHeader::usSequence) - .def_rw("idle_time", &oem::IntermediateHeader::ucIdleTime) - .def_rw("time_status", &oem::IntermediateHeader::uiTimeStatus) - .def_rw("week", &oem::IntermediateHeader::usWeek) - .def_rw("milliseconds", &oem::IntermediateHeader::dMilliseconds) - .def_rw("receiver_status", &oem::IntermediateHeader::uiReceiverStatus) - .def_rw("message_definition_crc", &oem::IntermediateHeader::uiMessageDefinitionCrc) - .def_rw("receiver_sw_version", &oem::IntermediateHeader::usReceiverSwVersion) - .def("to_dict", [](const oem::IntermediateHeader& self) { - nb::dict header_dict; - header_dict["message_id"] = self.usMessageId; - header_dict["message_type"] = self.ucMessageType; - header_dict["port_address"] = self.uiPortAddress; - header_dict["length"] = self.usLength; - header_dict["sequence"] = self.usSequence; - header_dict["idle_time"] = self.ucIdleTime; - header_dict["time_status"] = self.uiTimeStatus; - header_dict["week"] = self.usWeek; - header_dict["milliseconds"] = self.dMilliseconds; - header_dict["receiver_status"] = self.uiReceiverStatus; - header_dict["message_definition_crc"] = self.uiMessageDefinitionCrc; - header_dict["receiver_sw_version"] = self.usReceiverSwVersion; - return header_dict; - }) - .def("__repr__", [](const nb::handle self) { - auto& header = nb::cast(self); - return nb::str("Header(message_id={!r}, message_type={!r}, port_address={!r}, length={!r}, sequence={!r}, " - "idle_time={!r}, time_status={!r}, week={!r}, milliseconds={!r}, receiver_status={!r}, " - "message_definition_crc={!r}, receiver_sw_version={!r})") - .format(header.usMessageId, header.ucMessageType, header.uiPortAddress, header.usLength, header.usSequence, header.ucIdleTime, - header.uiTimeStatus, header.usWeek, header.dMilliseconds, header.uiReceiverStatus, header.uiMessageDefinitionCrc, - header.usReceiverSwVersion); - }); - nb::class_(m, "Oem4BinaryHeader") .def(nb::init()) .def("__init__", diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 03d1b2a69..87c690c41 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -18,6 +18,15 @@ struct PyGpsTime TIME_STATUS time_status{TIME_STATUS::UNKNOWN}; }; +//============================================================================ +//! \class PyField +//! \brief A python representation for a log header. +//============================================================================ +struct PyHeader : public IntermediateHeader +{ + nb::dict to_dict() const; +}; + //============================================================================ //! \class PyField //! \brief A python representation for a single log message or message field. @@ -57,9 +66,9 @@ struct PyField struct PyMessage : public PyField { public: - nb::object header; + PyHeader header; - PyMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_, nb::object header_) + PyMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_, PyHeader header_) : PyField(std::move(fields_), std::move(parent_db_), std::move(name_)), header(std::move(header_)) {} }; From bb072e4017e16024f2887c7da256504a4d7c068f Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 15:48:10 -0700 Subject: [PATCH 56/67] Update test enum refs --- python/test/test_decode_encode.py | 7 ++++--- python/test/test_message_database.py | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/test/test_decode_encode.py b/python/test/test_decode_encode.py index b2e05c4ed..f7809435b 100644 --- a/python/test/test_decode_encode.py +++ b/python/test/test_decode_encode.py @@ -31,6 +31,7 @@ from collections import namedtuple import novatel_edie as ne +from novatel_edie.enums import SolStatus, SolType, Datum import pytest from novatel_edie import STATUS, ENCODE_FORMAT from novatel_edie.messages import * @@ -506,13 +507,13 @@ def test_flat_binary_log_decode_bestpos(helper): # SOL_COMPUTED SINGLE 51.15043711386 -114.03067767000 1097.2099 -17.0000 WGS84 0.9038 0.8534 1.7480 \"\" 0.000 0.000 35 30 30 30 00 06 39 33\r\n" assert isinstance(message, BESTPOS) - assert message.solution_status == ne.enums.SolStatus.SOL_COMPUTED - assert message.position_type == ne.enums.SolType.SINGLE + assert message.solution_status == SolStatus.SOL_COMPUTED + assert message.position_type == SolType.SINGLE assert message.latitude == 51.15043711386 assert message.longitude == -114.03067767000 assert message.orthometric_height == 1097.2099 assert message.undulation == -17.0000 - assert message.datum_id == ne.enums.Datum.WGS84 + assert message.datum_id == Datum.WGS84 assert message.latitude_std_dev == approx(0.9038, abs=1e-5) assert message.longitude_std_dev == approx(0.8534, abs=1e-5) assert message.height_std_dev == approx(1.7480, abs=1e-5) diff --git a/python/test/test_message_database.py b/python/test/test_message_database.py index f1a6ede27..0085d8034 100644 --- a/python/test/test_message_database.py +++ b/python/test/test_message_database.py @@ -22,10 +22,9 @@ # ################################################################################= -import novatel_edie as ne - +from novatel_edie.enums import Datum def test_message_db_enums(json_db): assert json_db.enums["Datum"].WGS84 == 61 assert json_db.enums["Datum"].WGS84.name == "WGS84" - assert json_db.enums["Datum"].WGS84 == ne.enums.Datum.WGS84 + assert json_db.enums["Datum"].WGS84 == Datum.WGS84 From 9c6f4f8f6658920960c0a61e199f44b67d82aafe Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 15:51:56 -0700 Subject: [PATCH 57/67] Name and comment cleanup --- python/bindings/message_database.cpp | 8 ++++---- python/bindings/message_decoder.cpp | 16 +++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index dfa8342c4..85ff45284 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -283,12 +283,12 @@ void PyMessageDatabase::UpdatePythonMessageTypes() // add message and message body types for each message definition for (const auto& message_def : MessageDefinitions()) { - nb::object msg_body_def = type_constructor(message_def->name, message_type_tuple, type_dict); - messages_by_name[message_def->name] = msg_body_def; + nb::object msg_type_def = type_constructor(message_def->name, message_type_tuple, type_dict); + messages_by_name[message_def->name] = msg_type_def; // add additional MessageBody types for each field array element within the message definition AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name, type_constructor, field_type_tuple, type_dict); } // provide UNKNOWN types for undecodable messages - nb::object default_msg_body_def = type_constructor("UNKNOWN", message_type_tuple, type_dict); - messages_by_name["UNKNOWN"] = default_msg_body_def; + nb::object default_msg_type_def = type_constructor("UNKNOWN", message_type_tuple, type_dict); + messages_by_name["UNKNOWN"] = default_msg_type_def; } diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index e69147258..9040be878 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -267,26 +267,24 @@ void init_novatel_message_decoder(nb::module_& m) std::vector fields; STATUS status = decoder.Decode(reinterpret_cast(message_body.c_str()), fields, metadata); PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); - nb::handle body_pytype; + nb::handle message_pytype; const std::string message_name = metadata.MessageName(); try { - body_pytype = parent_db->GetMessagesByNameDict().at(message_name); + message_pytype = parent_db->GetMessagesByNameDict().at(message_name); } catch (const std::out_of_range& e) { - body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + message_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); } - nb::object body_pyinst = nb::inst_alloc(body_pytype); - PyMessage* body_cinst = nb::inst_ptr(body_pyinst); + nb::object message_pyinst = nb::inst_alloc(message_pytype); + PyMessage* body_cinst = nb::inst_ptr(message_pyinst); new (body_cinst) PyMessage(fields, parent_db, message_name, header); - nb::inst_mark_ready(body_pyinst); + nb::inst_mark_ready(message_pyinst); - //auto message_pyinst = std::make_shared(body_pyinst, header, message_name); - - return nb::make_tuple(status, body_pyinst); + return nb::make_tuple(status, message_pyinst); }, "message_body"_a, "decoded_header"_a, "metadata"_a) .def( From ca6f6b95bd4acd09112a7653b859039ba6c8dfae Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 15:59:15 -0700 Subject: [PATCH 58/67] More renaming --- python/bindings/message_decoder.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 9040be878..01433bb44 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -267,24 +267,24 @@ void init_novatel_message_decoder(nb::module_& m) std::vector fields; STATUS status = decoder.Decode(reinterpret_cast(message_body.c_str()), fields, metadata); PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); - nb::handle message_pytype; + nb::handle body_pytype; const std::string message_name = metadata.MessageName(); try { - message_pytype = parent_db->GetMessagesByNameDict().at(message_name); + body_pytype = parent_db->GetMessagesByNameDict().at(message_name); } catch (const std::out_of_range& e) { - message_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); } - nb::object message_pyinst = nb::inst_alloc(message_pytype); - PyMessage* body_cinst = nb::inst_ptr(message_pyinst); + nb::object body_pyinst = nb::inst_alloc(body_pytype); + PyMessage* body_cinst = nb::inst_ptr(body_pyinst); new (body_cinst) PyMessage(fields, parent_db, message_name, header); - nb::inst_mark_ready(message_pyinst); + nb::inst_mark_ready(body_pyinst); - return nb::make_tuple(status, message_pyinst); + return nb::make_tuple(status, body_pyinst); }, "message_body"_a, "decoded_header"_a, "metadata"_a) .def( From dc392d3bc3bf35c5781fa235102dc14629fa5485 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 16:18:40 -0700 Subject: [PATCH 59/67] Fix CMake path --- python/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f4de3ecef..913be38cf 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -67,7 +67,7 @@ add_custom_target( COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs COMMENT "Generating dynamic stubs" DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie/stubgen.py + ${CMAKE_SOURCE_DIR}/python/novatel_edie/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ) From 8d5186a8cb30ee82a5bce884b8026bf6ebb6acad Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Fri, 7 Feb 2025 16:25:48 -0700 Subject: [PATCH 60/67] Add missing files --- python/CMakeLists.txt | 2 +- python/novatel_edie/__main__.py | 27 ++++ python/novatel_edie/cli.py | 51 +++++++ python/novatel_edie/installer.py | 71 +++++++++ python/novatel_edie/stubgen.py | 253 +++++++++++++++++++++++++++++++ 5 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 python/novatel_edie/__main__.py create mode 100644 python/novatel_edie/cli.py create mode 100644 python/novatel_edie/installer.py create mode 100644 python/novatel_edie/stubgen.py diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 913be38cf..f4de3ecef 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -67,7 +67,7 @@ add_custom_target( COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs COMMENT "Generating dynamic stubs" DEPENDS - ${CMAKE_SOURCE_DIR}/python/novatel_edie/stubgen.py + ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ) diff --git a/python/novatel_edie/__main__.py b/python/novatel_edie/__main__.py new file mode 100644 index 000000000..b2ccb37ed --- /dev/null +++ b/python/novatel_edie/__main__.py @@ -0,0 +1,27 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +if __name__ == "__main__": + from .cli import app + app() diff --git a/python/novatel_edie/cli.py b/python/novatel_edie/cli.py new file mode 100644 index 000000000..9e3f99e86 --- /dev/null +++ b/python/novatel_edie/cli.py @@ -0,0 +1,51 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import logging + +import typer + +from .stubgen import generate_stubs +from .installer import install_custom + +# Create CLI app +app = typer.Typer() + +# Register commands +app.command()(generate_stubs) +app.command()(install_custom) + +def setup_cli_logging(): + """Sets up logging configuartion used when executing code via the CLI.""" + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + root_logger = logging.getLogger() + root_logger.addHandler(console_handler) + root_logger.setLevel(logging.INFO) + +# Enable logging config whenever any app command is executed +app.callback()(setup_cli_logging) + +if __name__ == "__main__": + app() diff --git a/python/novatel_edie/installer.py b/python/novatel_edie/installer.py new file mode 100644 index 000000000..8d9ee23cf --- /dev/null +++ b/python/novatel_edie/installer.py @@ -0,0 +1,71 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +A module concerning the installation of a custom package. +""" + +import os +import shutil +import importlib +import logging + +import typer +from typing_extensions import Annotated + +from novatel_edie.stubgen import StubGenerator + +def install_custom( + database: Annotated[ + str, + typer.Argument(help='A path to a database file.') + ]): + """Create a custom installation of novatel_edie based on the provided DB. + + The custom installation will have all messages and enums of the provided + database be directly importable from the 'enums' and 'messages' submodules. + + Args: + database: A path to a database file. + """ + # Determine directory information + database = os.path.abspath(database) + library_name = 'novatel_edie' + lib_spec = importlib.util.find_spec(library_name) + lib_dir = os.path.dirname(lib_spec.origin) + database_path = os.path.join(lib_dir, 'database.json') + + # Update static type information + logging.info('Updating static type information...') + StubGenerator(database).write_stub_files(lib_dir) + # Install new database + logging.info('Copying database info...') + try: + shutil.copy2(database, database_path) + except Exception as e: + raise IOError('Failed to copy database info') from e + logging.info(f'Database copied to {database_path}') + + logging.info(f'\nCustom installation of {library_name} created!\n') + +if __name__ == '__main__': + typer.run(install_custom) diff --git a/python/novatel_edie/stubgen.py b/python/novatel_edie/stubgen.py new file mode 100644 index 000000000..4ddc8465a --- /dev/null +++ b/python/novatel_edie/stubgen.py @@ -0,0 +1,253 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +A module concerning the generation of type hint stub files for the novatel_edie package. +""" +import os +import json +import re +import textwrap +import logging +from typing import Union + +import typer +from typing_extensions import Annotated + +class StubGenerator: + """Generator of type hint stub files for the novatel_edie package.""" + data_type_to_pytype = { + 'INT': 'int', + 'UNIT': 'int', + 'BOOL': 'bool', + 'CHAR': 'int', + 'UCHAR': 'int', + 'SHORT': 'int', + 'USHORT': 'int', + 'LONG': 'int', + 'ULONG': 'int', + 'LONGLONG': 'int', + 'ULONGLONG': 'int', + 'FLOAT': 'float', + 'DOUBLE': 'float', + 'HEXBYTE': 'int', + 'SATELLITEID': 'SatelliteId', + 'UNKNOWN': 'bytes' + } + def __init__(self, database: Union[str, dict]): + """Initializes a StubGenerator. + + Args: + database: A path to a message database json file or + a dictionary representation of a database. + """ + if isinstance(database, str): + try: + with open(database, 'r') as f: + database = json.load(f) + except Exception as e: + raise Exception(f'Failed to load database from {database}') from e + if not isinstance(database, dict): + raise TypeError( + 'database must be a dict or a path to a JSON database file.') + self.database = database + self.database['enums_by_id'] = {enum['_id']: enum for enum in database['enums']} + logging.info('Database loaded successfully') + + def write_stub_files(self, file_path: str): + """Writes package stub files to the specified directory. + + Args: + file_path: The directory to write the stub files to. + """ + file_specs = { + 'enums.pyi': self.get_enum_stubs(), + 'messages.pyi': self.get_message_stubs()} + + if not os.path.exists(file_path): + os.makedirs(file_path) + for file_name, file_contents in file_specs.items(): + with open(os.path.join(file_path, file_name), 'w') as f: + f.write(file_contents) + logging.info(f'Stub files written to {file_path}') + + def _convert_enum_def(self, enum_def) -> str: + """Create a type hint string for an enum definition. + + Args: + enum_def: A dictionary containing an enum definition. + Returns: + A string containing a type hint for the enum definition. + """ + type_hint = f'class {enum_def["name"]}(Enum):\n' + for enumerator in enum_def['enumerators']: + name = enumerator['name'] + name = re.sub(r'[)(\-+./\\]', '_', name) + if name[0].isdigit(): + name = f'_{name}' + type_hint += f' {name} = {enumerator["value"]}\n' + return type_hint + + def get_enum_stubs(self) -> str: + """Get a stub string for all enums in the database. + + Returns: + A string containing type hint stubs for all enums in the database. + """ + stub_str = ('from enum import Enum\n' + 'from typing import Any\n\n') + enums = self.database['enums'] + type_hints = [self._convert_enum_def(enum_def) for enum_def in enums] + type_hints_str = '\n'.join(type_hints) + stub_str += type_hints_str + return stub_str + + def _get_field_pytype(self, field: dict, parent: str) -> str: + """Get a type hint string for a field definition. + + Args: + field: A dictionary containing a field definition. + parent: The name of the parent class. May be used for naming + subfield classes. + Returns: + A string containing a type hint for the field definition. + """ + python_type = None + if field['type'] == 'SIMPLE': + python_type = self.data_type_to_pytype.get(field['dataType']['name']) + if field['type'] == 'ENUM': + enum_def = self.database['enums_by_id'].get(field['enumID']) + python_type = enum_def['name'] if enum_def else 'Any' + if field['type'] == 'STRING': + python_type = 'str' + if (field['type'] in { + 'FIXED_LENGTH_ARRAY', + 'VARIABLE_LENGTH_ARRAY'}): + if field['conversionString'] == r'%s': + python_type = 'str' + else: + python_type = f'list[{self.data_type_to_pytype.get(field["dataType"]["name"])}]' + if field['type'] == 'FIELD_ARRAY': + python_type = f'list[{parent}_{field["name"]}_Field]' + if not python_type: + python_type = 'Any' + return python_type + + def _convert_field_array_def(self, field_array_def: dict, parent: str) -> str: + """Convert a field array definition to a type hint string. + + Args: + field_array_def: A dictionary containing a field array definition. + parent: The name of the parent class. The name for the field array + class will be based on this. + Returns: + A string containing type hint stubs for the field array definition. + """ + subfield_hints = [] + + # Create MessageBodyField type hint + name = f'{parent}_{field_array_def["name"]}_Field' + type_hint = f'class {name}(Field):\n' + for field in field_array_def['fields']: + python_type = self._get_field_pytype(field, name) + type_hint += (' @property\n' + f' def {field["name"]}(self) -> {python_type}: ...\n\n') + # Create hints for any subfields + if field['type'] == 'FIELD_ARRAY': + subfield_hints.append(self._convert_field_array_def(field, name)) + + # Combine all hints + hints = subfield_hints + [type_hint] + hint_str = '\n'.join(hints) + + return hint_str + + def _convert_message_def(self, message_def: dict) -> str: + """Create a type hint string for a message definition. + + Args: + message_def: A dictionary containing a message definition. + Returns: + A string containing type hint stubs for the message definition. + """ + subfield_hints = [] + + # Create the Message type hint + name = message_def["name"] + body_hint = f'class {name}(Message):\n' + fields = message_def['fields'][message_def['latestMsgDefCrc']] + if not fields: + body_hint += ' pass\n\n' + for field in fields: + python_type = self._get_field_pytype(field, name) + body_hint += ' @property\n' + body_hint += f' def {field["name"]}(self) -> {python_type}: ...\n\n' + + # Create hints for any subfields + if field['type'] == 'FIELD_ARRAY': + subfield_hints.append(self._convert_field_array_def(field, name)) + + # Combine all hints + hints = subfield_hints + [body_hint] + hint_str = '\n'.join(hints) + + return hint_str + + def get_message_stubs(self) -> str: + """Get a stub string for all messages in the database. + + Returns: + A string containing type hint stubs for all messages in the database. + """ + stub_str = 'from typing import Any\n\n' + stub_str += 'from novatel_edie import Header, Field, Message, SatelliteId\n' + stub_str += 'from novatel_edie.enums import *\n\n' + + messages = self.database['messages'] + type_hints = [self._convert_message_def(msg_def) + for msg_def in messages] + type_hints_str = '\n'.join(type_hints) + stub_str += type_hints_str + + return stub_str + + +def generate_stubs( + database: Annotated[ + str, + typer.Argument(help='A path to a database file.') + ], + output_dir: Annotated[ + str, + typer.Argument(help='The directory to write stub files to.') + ] = './stubs' + ): + """Generate type hint stub files for a provided database. + + Args: + database: A path to a database file. + output_dir: The directory to write stub files to. + """ + StubGenerator(database).write_stub_files(output_dir) + +if __name__ == '__main__': + typer.run(generate_stubs) From 3d51f15e7dec70dc9699d2bd2a6e8b21c6a0903a Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 10 Feb 2025 11:36:02 -0700 Subject: [PATCH 61/67] CRC fix --- python/bindings/framer.cpp | 1 - python/bindings/init_modules.cpp | 8 +++-- python/bindings/message_database.cpp | 10 +++--- python/bindings/message_decoder.cpp | 49 ++++++++++++++++++++------ python/bindings/py_database.hpp | 14 ++++++-- python/bindings/py_decoded_message.hpp | 18 +++++++++- 6 files changed, 77 insertions(+), 23 deletions(-) diff --git a/python/bindings/framer.cpp b/python/bindings/framer.cpp index cbfd945c7..9f7c12b5c 100644 --- a/python/bindings/framer.cpp +++ b/python/bindings/framer.cpp @@ -1,5 +1,4 @@ #include "novatel_edie/decoders/oem/framer.hpp" - #include "bindings_core.hpp" namespace nb = nanobind; diff --git a/python/bindings/init_modules.cpp b/python/bindings/init_modules.cpp index cdb2499f2..60ea1b616 100644 --- a/python/bindings/init_modules.cpp +++ b/python/bindings/init_modules.cpp @@ -15,8 +15,12 @@ void init_novatel_oem_enums(nb::module_& m) void init_novatel_oem_messages(nb::module_& m) { - for (const auto& [name, message_type] : MessageDbSingleton::get()->GetMessagesByNameDict()) + for (const auto& [name, message_type_struct] : MessageDbSingleton::get()->GetMessagesByNameDict()) { - m.attr(name.c_str()) = message_type; + m.attr(name.c_str()) = message_type_struct->python_type; + } + for (const auto& [name, field_type] : MessageDbSingleton::get()->GetFieldsByNameDict()) + { + m.attr(name.c_str()) = field_type; } } diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index 85ff45284..96939a637 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -261,7 +261,7 @@ void PyMessageDatabase::AddFieldType(std::vector> fie auto* field_array_field = dynamic_cast(field.get()); std::string field_name = base_name + "_" + field_array_field->name + "_Field"; nb::object field_type = type_constructor(field_name, type_tuple, type_dict); - messages_by_name[field_name] = field_type; + fields_by_name[field_name] = field_type; AddFieldType(field_array_field->fields, field_name, type_constructor, type_tuple, type_dict); } } @@ -283,12 +283,10 @@ void PyMessageDatabase::UpdatePythonMessageTypes() // add message and message body types for each message definition for (const auto& message_def : MessageDefinitions()) { + uint32_t crc = message_def->latestMessageCrc; nb::object msg_type_def = type_constructor(message_def->name, message_type_tuple, type_dict); - messages_by_name[message_def->name] = msg_type_def; + messages_by_name[message_def->name] = new PyMessageType(msg_type_def, crc); // add additional MessageBody types for each field array element within the message definition - AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name, type_constructor, field_type_tuple, type_dict); + AddFieldType(message_def->fields.at(crc), message_def->name, type_constructor, field_type_tuple, type_dict); } - // provide UNKNOWN types for undecodable messages - nb::object default_msg_type_def = type_constructor("UNKNOWN", message_type_tuple, type_dict); - messages_by_name["UNKNOWN"] = default_msg_type_def; } diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 01433bb44..215fe3654 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -49,11 +49,11 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C std::string field_name = parent + "_" + field.fieldDef->name + "_Field"; try { - field_ptype = parent_db->GetMessagesByNameDict().at(field_name); + field_ptype = parent_db->GetFieldsByNameDict().at(field_name); } catch (const std::out_of_range& e) { - field_ptype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + field_ptype = parent_db->GetFieldsByNameDict().at("UNKNOWN"); } for (const auto& subfield : message_field) { @@ -207,8 +207,6 @@ void init_novatel_message_decoder(nb::module_& m) .def_rw("status", &PyGpsTime::time_status); nb::class_(m, "Field") - //.def_prop_ro("_values", &PyField::get_values) - //.def_prop_ro("_fields", &PyField::get_fields) .def("to_dict", &PyField::to_dict, "Convert the message and its sub-messages into a dict") .def("__getattr__", &PyField::getattr, "field_name"_a) .def("__repr__", &PyField::repr) @@ -219,6 +217,7 @@ void init_novatel_message_decoder(nb::module_& m) nb::handle super = builtins.attr("super"); nb::handle type = builtins.attr("type"); + // start from the 'Field' class instead of a specific subclass nb::handle current_type = type(self); std::string current_type_name = nb::cast(current_type.attr("__name__")); while (current_type_name != "Field") @@ -227,7 +226,7 @@ void init_novatel_message_decoder(nb::module_& m) current_type_name = nb::cast(current_type.attr("__name__")); } - // retrieve base list based on superclass method + // retrieve base list based on 'Field' superclass method nb::object super_obj = super(current_type, self); nb::list base_list = nb::cast(super_obj.attr("__dir__")()); // add dynamic fields to the list @@ -249,6 +248,8 @@ void init_novatel_message_decoder(nb::module_& m) "include_header"_a = true, "Convert the message and its sub-messages into a dict"); + nb::class_(m, "UNKNOWN").def_ro("bytes", &UnknownMessage::bytes); + nb::class_(m, "FieldContainer") .def_rw("value", &FieldContainer::fieldValue) .def_rw("field_def", &FieldContainer::fieldDef, nb::rv_policy::reference_internal) @@ -268,18 +269,44 @@ void init_novatel_message_decoder(nb::module_& m) STATUS status = decoder.Decode(reinterpret_cast(message_body.c_str()), fields, metadata); PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); nb::handle body_pytype; + nb::object body_pyinst; const std::string message_name = metadata.MessageName(); - try + if (message_name == "UNKNOWN") { + body_pytype = nb::type(); + } + else { + try + { + PyMessageType* message_type_struct = parent_db->GetMessagesByNameDict().at(message_name); + if (message_type_struct->crc == metadata.uiMessageCrc) { + // If the CRCs match, use the specific message type + body_pytype = message_type_struct->python_type; + } + else { + // If the CRCs don't match, use the generic "MESSAGE" type + body_pytype = nb::type(); + } + } + catch (const std::out_of_range& e) + { + // This case should be impossible, if this occurs it indicates a bug in the code + throw std::runtime_error("Message name '" + message_name + "' not found in the JSON database"); + } + } + + + body_pyinst = nb::inst_alloc(body_pytype); + if (message_name == "UNKNOWN") { - body_pytype = parent_db->GetMessagesByNameDict().at(message_name); + UnknownMessage* body_cinst = nb::inst_ptr(body_pyinst); + new (body_cinst) UnknownMessage(fields, parent_db, message_name, header, message_body); } - catch (const std::out_of_range& e) + else { - body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + PyMessage* body_cinst = nb::inst_ptr(body_pyinst); + new (body_cinst) PyMessage(fields, parent_db, message_name, header); } - - nb::object body_pyinst = nb::inst_alloc(body_pytype); PyMessage* body_cinst = nb::inst_ptr(body_pyinst); new (body_cinst) PyMessage(fields, parent_db, message_name, header); nb::inst_mark_ready(body_pyinst); diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index b073cd5c7..088b5c337 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -7,6 +7,14 @@ namespace nb = nanobind; namespace novatel::edie { +struct PyMessageType +{ + nb::object python_type; + uint32_t crc; + + PyMessageType(nb::object python_type_, uint32_t crc_) : python_type(std::move(python_type_)), crc(crc_) {} +}; + class PyMessageDatabase final : public MessageDatabase { public: @@ -15,7 +23,8 @@ class PyMessageDatabase final : public MessageDatabase explicit PyMessageDatabase(const MessageDatabase& message_db) noexcept; explicit PyMessageDatabase(const MessageDatabase&& message_db) noexcept; - [[nodiscard]] const std::unordered_map& GetMessagesByNameDict() const { return messages_by_name; } + [[nodiscard]] const std::unordered_map& GetMessagesByNameDict() const { return messages_by_name; } + [[nodiscard]] const std::unordered_map& GetFieldsByNameDict() const { return fields_by_name; } [[nodiscard]] const std::unordered_map& GetEnumsByIdDict() const { return enums_by_id; } [[nodiscard]] const std::unordered_map& GetEnumsByNameDict() const { return enums_by_name; } @@ -42,7 +51,8 @@ class PyMessageDatabase final : public MessageDatabase void UpdatePythonMessageTypes(); void AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, nb::handle type_dict); - std::unordered_map messages_by_name{}; + std::unordered_map messages_by_name{}; + std::unordered_map fields_by_name{}; std::unordered_map enums_by_id{}; std::unordered_map enums_by_name{}; diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 87c690c41..703e043dd 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -1,7 +1,7 @@ #pragma once #include "bindings_core.hpp" -#include "novatel_edie/decoders/oem/message_decoder.hpp" +#include "novatel_edie/decoders/oem/filter.hpp" namespace nb = nanobind; @@ -72,4 +72,20 @@ struct PyMessage : public PyField : PyField(std::move(fields_), std::move(parent_db_), std::move(name_)), header(std::move(header_)) {} }; +//============================================================================ +//! \class UnknownMessage +//! \brief A python representation for an unknown log message. +//! +//! Contains the raw bytes of the message. +//============================================================================ +struct UnknownMessage : public PyMessage +{ + nb::bytes bytes; + explicit UnknownMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_, PyHeader header_, + nb::bytes bytes_) + : PyMessage(std::move(fields_), std::move(parent_db_), std::move(name_), std::move(header_)), bytes(std::move(bytes_)) + { + } +}; + } // namespace novatel::edie::oem From 21aa923bbb09550cc82bece9b66dc7f59c044b5f Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 10 Feb 2025 16:12:34 -0700 Subject: [PATCH 62/67] Fix unknown handling --- python/bindings/message_decoder.cpp | 56 +++++++++++--------- python/bindings/py_decoded_message.hpp | 16 +++--- python/novatel_edie/messages.py | 1 + src/decoders/common/src/message_database.cpp | 2 +- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 215fe3654..a858bba6c 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -17,7 +17,7 @@ using namespace novatel::edie::oem; NB_MAKE_OPAQUE(std::vector); -nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::ConstPtr& parent_db, std::string parent) +nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::ConstPtr& parent_db, std::string parent, bool has_ptype) { if (field.fieldDef->type == FIELD_TYPE::ENUM) { @@ -43,24 +43,37 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C else if (field.fieldDef->type == FIELD_TYPE::FIELD_ARRAY) { // Handle Field Arrays - std::vector sub_values; - sub_values.reserve(message_field.size()); nb::handle field_ptype; - std::string field_name = parent + "_" + field.fieldDef->name + "_Field"; - try + std::string field_name; + if (has_ptype) { - field_ptype = parent_db->GetFieldsByNameDict().at(field_name); + // If a parent type name is provided, get a field type based on the parent and field name + field_name = parent + "_" + field.fieldDef->name + "_Field"; + try + { + field_ptype = parent_db->GetFieldsByNameDict().at(field_name); + } + catch (const std::out_of_range& e) + { + // This case should never happen, if it does there is a bug + throw std::runtime_error("Field type not found for " + field_name); + } } - catch (const std::out_of_range& e) - { - field_ptype = parent_db->GetFieldsByNameDict().at("UNKNOWN"); + else { + // If field has no ptype, use the generic "Field" type + field_name = std::move(parent); + field_ptype = nb::type(); } + + // Create an appropriate PyField instance for each subfield in the array + std::vector sub_values; + sub_values.reserve(message_field.size()); for (const auto& subfield : message_field) { nb::object pyinst = nb::inst_alloc(field_ptype); PyField* cinst = nb::inst_ptr(pyinst); const auto& message_subfield = std::get>(subfield.fieldValue); - new (cinst) PyField(message_subfield, parent_db, field_name); + new (cinst) PyField(field_name, has_ptype, message_subfield, parent_db); nb::inst_mark_ready(pyinst); sub_values.push_back(pyinst); } @@ -84,7 +97,7 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } std::vector sub_values; sub_values.reserve(message_field.size()); - for (const auto& f : message_field) { sub_values.push_back(convert_field(f, parent_db, parent)); } + for (const auto& f : message_field) { sub_values.push_back(convert_field(f, parent_db, parent, has_ptype)); } return nb::cast(sub_values); } } @@ -104,16 +117,11 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } } -PyField::PyField(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) - : name(std::move(name_)), fields(std::move(message_)), parent_db_(std::move(parent_db_)) -{ -} - nb::dict& PyField::get_values() const { if (cached_values_.size() == 0) { - for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_, this->name); } + for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_, this->name, this->has_ptype); } } return cached_values_; } @@ -271,6 +279,7 @@ void init_novatel_message_decoder(nb::module_& m) nb::handle body_pytype; nb::object body_pyinst; const std::string message_name = metadata.MessageName(); + bool has_ptype = true; if (message_name == "UNKNOWN") { body_pytype = nb::type(); @@ -286,11 +295,12 @@ void init_novatel_message_decoder(nb::module_& m) else { // If the CRCs don't match, use the generic "MESSAGE" type body_pytype = nb::type(); + has_ptype = false; } } catch (const std::out_of_range& e) { - // This case should be impossible, if this occurs it indicates a bug in the code + // This case should never happen, if it does there is a bug throw std::runtime_error("Message name '" + message_name + "' not found in the JSON database"); } } @@ -300,15 +310,13 @@ void init_novatel_message_decoder(nb::module_& m) if (message_name == "UNKNOWN") { UnknownMessage* body_cinst = nb::inst_ptr(body_pyinst); - new (body_cinst) UnknownMessage(fields, parent_db, message_name, header, message_body); + new (body_cinst) UnknownMessage(fields, parent_db, header, message_body); } else { PyMessage* body_cinst = nb::inst_ptr(body_pyinst); - new (body_cinst) PyMessage(fields, parent_db, message_name, header); + new (body_cinst) PyMessage(message_name, has_ptype, fields, parent_db, header); } - PyMessage* body_cinst = nb::inst_ptr(body_pyinst); - new (body_cinst) PyMessage(fields, parent_db, message_name, header); nb::inst_mark_ready(body_pyinst); return nb::make_tuple(status, body_pyinst); @@ -322,7 +330,7 @@ void init_novatel_message_decoder(nb::module_& m) std::string body_str(message_body.c_str(), message_body.size()); const char* data_ptr = body_str.c_str(); STATUS status = static_cast(&decoder)->TestDecodeAscii(msg_def_fields, &data_ptr, fields); - return nb::make_tuple(status, PyField(std::move(fields), get_parent_db(decoder), "UNKNOWN")); + return nb::make_tuple(status, PyField("", false, std::move(fields), get_parent_db(decoder))); }, "msg_def_fields"_a, "message_body"_a) .def( @@ -333,7 +341,7 @@ void init_novatel_message_decoder(nb::module_& m) const char* data_ptr = message_body.c_str(); STATUS status = static_cast(&decoder)->TestDecodeBinary(msg_def_fields, reinterpret_cast(&data_ptr), fields, message_length); - return nb::make_tuple(status, PyField(std::move(fields), get_parent_db(decoder), "UNKNOWN")); + return nb::make_tuple(status, PyField("", false, std::move(fields), get_parent_db(decoder))); }, "msg_def_fields"_a, "message_body"_a, "message_length"_a); } diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 703e043dd..3a8176750 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -37,7 +37,10 @@ struct PyHeader : public IntermediateHeader struct PyField { std::string name; - explicit PyField(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_); + bool has_ptype; // Whether the field has a specific Python type associated with it + + explicit PyField(std::string name_, bool has_ptype_, std::vector message_, PyMessageDatabase::ConstPtr parent_db_) + : name(std::move(name_)), has_ptype(has_ptype_), fields(std::move(message_)), parent_db_(std::move(parent_db_)) {}; nb::dict& get_values() const; nb::dict& get_fields() const; nb::dict to_dict() const; @@ -68,8 +71,10 @@ struct PyMessage : public PyField public: PyHeader header; - PyMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_, PyHeader header_) - : PyField(std::move(fields_), std::move(parent_db_), std::move(name_)), header(std::move(header_)) {} + PyMessage(std::string name_, bool has_ptype_, std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, PyHeader header_) + : PyField(std::move(name_), has_ptype_, std::move(fields_), std::move(parent_db_)), header(std::move(header_)) + { + } }; //============================================================================ @@ -81,9 +86,8 @@ struct PyMessage : public PyField struct UnknownMessage : public PyMessage { nb::bytes bytes; - explicit UnknownMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_, PyHeader header_, - nb::bytes bytes_) - : PyMessage(std::move(fields_), std::move(parent_db_), std::move(name_), std::move(header_)), bytes(std::move(bytes_)) + explicit UnknownMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, PyHeader header_, nb::bytes bytes_) + : PyMessage("UNKNOWN", true, std::move(fields_), std::move(parent_db_), std::move(header_)), bytes(std::move(bytes_)) { } }; diff --git a/python/novatel_edie/messages.py b/python/novatel_edie/messages.py index 74f63b195..2a44cd2a0 100644 --- a/python/novatel_edie/messages.py +++ b/python/novatel_edie/messages.py @@ -23,3 +23,4 @@ """ from .bindings.messages import * +from .bindings import UNKNOWN diff --git a/src/decoders/common/src/message_database.cpp b/src/decoders/common/src/message_database.cpp index 02457547d..a20e08b3a 100644 --- a/src/decoders/common/src/message_database.cpp +++ b/src/decoders/common/src/message_database.cpp @@ -121,7 +121,7 @@ std::string MessageDatabase::MsgIdToMsgName(const uint32_t uiMessageId_) const UnpackMsgId(uiMessageId_, usLogId, uiSiblingId, uiMessageFormat, uiResponse); MessageDefinition::ConstPtr pstMessageDefinition = GetMsgDef(usLogId); - std::string strMessageName = pstMessageDefinition != nullptr ? pstMessageDefinition->name : GetEnumString(vEnumDefinitions[0], usLogId); + std::string strMessageName = pstMessageDefinition != nullptr ? pstMessageDefinition->name : "UNKNOWN"; std::string strMessageFormatSuffix; if (uiResponse != 0U) { strMessageFormatSuffix = "R"; } From 7d0df499e61d4d56d6e6eff159479ce6aa9b8ac2 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 10 Feb 2025 16:19:05 -0700 Subject: [PATCH 63/67] Update unknown message behavior --- python/examples/converter_components.py | 31 ++++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index 604d8b132..b702b6bb0 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -109,25 +109,28 @@ def main(): if not filter.do_filtering(meta): continue - # Decode the log body. - body = frame[meta.header_length:] - status, message = message_decoder.decode(body, header, meta) - status.raise_on_error("MessageDecoder.decode() failed") + if meta.message_id == 39: + pass + + # Decode the log body. + body = frame[meta.header_length:] + status, message = message_decoder.decode(body, header, meta) + status.raise_on_error("MessageDecoder.decode() failed") # Get info from the log. - if isinstance(message, RANGE): - obs = message.obs - for ob in obs: - value = ob.psr - pass + # if isinstance(message, RANGE): + # obs = message.obs + # for ob in obs: + # value = ob.psr + # pass - # Re-encode the log and write it to the output file. - status, encoded_message = encoder.encode(message, meta, encode_format) - status.raise_on_error("Encoder.encode() failed") + # # Re-encode the log and write it to the output file. + # status, encoded_message = encoder.encode(message, meta, encode_format) + # status.raise_on_error("Encoder.encode() failed") - converted_logs_stream.write(encoded_message.message) - logger.info( f"Encoded ({len(encoded_message.message)}): {format_frame(encoded_message.message, encode_format)}") + # converted_logs_stream.write(encoded_message.message) + # logger.info( f"Encoded ({len(encoded_message.message)}): {format_frame(encoded_message.message, encode_format)}") except ne.DecoderException as e: logger.warn(str(e)) From 376e59c69363861be515001f11281222edf63bbf Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 10 Feb 2025 16:58:07 -0700 Subject: [PATCH 64/67] Fix unknown repr --- python/bindings/message_decoder.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index a858bba6c..21487d8f4 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -256,7 +256,12 @@ void init_novatel_message_decoder(nb::module_& m) "include_header"_a = true, "Convert the message and its sub-messages into a dict"); - nb::class_(m, "UNKNOWN").def_ro("bytes", &UnknownMessage::bytes); + nb::class_(m, "UNKNOWN") + .def("__repr__", [](const UnknownMessage self) { + std::string name = nb::cast(nb::str(self.bytes)); + return "UNKNOWN(bytes=" + name + ")"; + }) + .def_ro("bytes", &UnknownMessage::bytes); nb::class_(m, "FieldContainer") .def_rw("value", &FieldContainer::fieldValue) From da8eb462c96787b13f3f636ba8aa8f8b9dd58e92 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 10 Feb 2025 17:04:10 -0700 Subject: [PATCH 65/67] Small fix --- python/bindings/py_decoded_message.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 3a8176750..b196360c7 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -87,7 +87,7 @@ struct UnknownMessage : public PyMessage { nb::bytes bytes; explicit UnknownMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, PyHeader header_, nb::bytes bytes_) - : PyMessage("UNKNOWN", true, std::move(fields_), std::move(parent_db_), std::move(header_)), bytes(std::move(bytes_)) + : PyMessage("UNKNOWN", false, std::move(fields_), std::move(parent_db_), std::move(header_)), bytes(std::move(bytes_)) { } }; From e4224be00b107c79853a53bba4f653e8bbe38170 Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Mon, 10 Feb 2025 17:43:48 -0700 Subject: [PATCH 66/67] remove blank comment --- python/bindings/init_modules.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bindings/init_modules.cpp b/python/bindings/init_modules.cpp index 60ea1b616..fc4915c1e 100644 --- a/python/bindings/init_modules.cpp +++ b/python/bindings/init_modules.cpp @@ -7,7 +7,7 @@ using namespace novatel::edie; void init_novatel_oem_enums(nb::module_& m) { - for (const auto& [name, enum_type] : MessageDbSingleton::get()->GetEnumsByNameDict()) // + for (const auto& [name, enum_type] : MessageDbSingleton::get()->GetEnumsByNameDict()) { m.attr(name.c_str()) = enum_type; } From a9522eb51bd5c84f382d2879a5dc32ceb95c5e1d Mon Sep 17 00:00:00 2001 From: Riley Kinahan Date: Tue, 11 Feb 2025 08:08:46 -0700 Subject: [PATCH 67/67] clang-format --- python/bindings/framer.cpp | 1 + python/bindings/init_modules.cpp | 10 ++----- python/bindings/json_db_reader.cpp | 3 +- python/bindings/message_decoder.cpp | 39 ++++++++++++++----------- python/bindings/py_database.hpp | 11 ++++--- python/bindings/py_decoded_message.hpp | 11 ++++--- python/examples/converter_components.py | 31 +++++++++----------- 7 files changed, 51 insertions(+), 55 deletions(-) diff --git a/python/bindings/framer.cpp b/python/bindings/framer.cpp index 9f7c12b5c..cbfd945c7 100644 --- a/python/bindings/framer.cpp +++ b/python/bindings/framer.cpp @@ -1,4 +1,5 @@ #include "novatel_edie/decoders/oem/framer.hpp" + #include "bindings_core.hpp" namespace nb = nanobind; diff --git a/python/bindings/init_modules.cpp b/python/bindings/init_modules.cpp index fc4915c1e..96240f7e6 100644 --- a/python/bindings/init_modules.cpp +++ b/python/bindings/init_modules.cpp @@ -7,10 +7,7 @@ using namespace novatel::edie; void init_novatel_oem_enums(nb::module_& m) { - for (const auto& [name, enum_type] : MessageDbSingleton::get()->GetEnumsByNameDict()) - { - m.attr(name.c_str()) = enum_type; - } + for (const auto& [name, enum_type] : MessageDbSingleton::get()->GetEnumsByNameDict()) { m.attr(name.c_str()) = enum_type; } } void init_novatel_oem_messages(nb::module_& m) @@ -19,8 +16,5 @@ void init_novatel_oem_messages(nb::module_& m) { m.attr(name.c_str()) = message_type_struct->python_type; } - for (const auto& [name, field_type] : MessageDbSingleton::get()->GetFieldsByNameDict()) - { - m.attr(name.c_str()) = field_type; - } + for (const auto& [name, field_type] : MessageDbSingleton::get()->GetFieldsByNameDict()) { m.attr(name.c_str()) = field_type; } } diff --git a/python/bindings/json_db_reader.cpp b/python/bindings/json_db_reader.cpp index 264da4e08..103e32b74 100644 --- a/python/bindings/json_db_reader.cpp +++ b/python/bindings/json_db_reader.cpp @@ -24,7 +24,8 @@ PyMessageDatabase::Ptr& MessageDbSingleton::get() nb::object module_spec = import_lib_util.attr("find_spec")("novatel_edie"); bool module_exists = nb::cast(builtins.attr("bool")(import_lib_util.attr("find_spec")("novatel_edie"))); // If the package does not exist, return an empty database - if (!module_exists) { + if (!module_exists) + { json_db = std::make_shared(MessageDatabase()); return json_db; } diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 21487d8f4..97693220c 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -59,7 +59,8 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C throw std::runtime_error("Field type not found for " + field_name); } } - else { + else + { // If field has no ptype, use the generic "Field" type field_name = std::move(parent); field_ptype = nb::type(); @@ -121,7 +122,10 @@ nb::dict& PyField::get_values() const { if (cached_values_.size() == 0) { - for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_, this->name, this->has_ptype); } + for (const auto& field : fields) + { + cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_, this->name, this->has_ptype); + } } return cached_values_; } @@ -248,19 +252,20 @@ void init_novatel_message_decoder(nb::module_& m) .def_ro("header", &PyMessage::header) .def_ro("name", &PyMessage::name) .def( - "to_dict", [](const PyMessage& self, bool include_header) { + "to_dict", + [](const PyMessage& self, bool include_header) { nb::dict dict = self.to_dict(); if (include_header) { dict["header"] = self.header.to_dict(); } return dict; - }, - "include_header"_a = true, - "Convert the message and its sub-messages into a dict"); + }, + "include_header"_a = true, "Convert the message and its sub-messages into a dict"); nb::class_(m, "UNKNOWN") - .def("__repr__", [](const UnknownMessage self) { - std::string name = nb::cast(nb::str(self.bytes)); - return "UNKNOWN(bytes=" + name + ")"; - }) + .def("__repr__", + [](const UnknownMessage self) { + std::string name = nb::cast(nb::str(self.bytes)); + return "UNKNOWN(bytes=" + name + ")"; + }) .def_ro("bytes", &UnknownMessage::bytes); nb::class_(m, "FieldContainer") @@ -286,18 +291,19 @@ void init_novatel_message_decoder(nb::module_& m) const std::string message_name = metadata.MessageName(); bool has_ptype = true; - if (message_name == "UNKNOWN") { - body_pytype = nb::type(); - } - else { + if (message_name == "UNKNOWN") { body_pytype = nb::type(); } + else + { try { PyMessageType* message_type_struct = parent_db->GetMessagesByNameDict().at(message_name); - if (message_type_struct->crc == metadata.uiMessageCrc) { + if (message_type_struct->crc == metadata.uiMessageCrc) + { // If the CRCs match, use the specific message type body_pytype = message_type_struct->python_type; } - else { + else + { // If the CRCs don't match, use the generic "MESSAGE" type body_pytype = nb::type(); has_ptype = false; @@ -309,7 +315,6 @@ void init_novatel_message_decoder(nb::module_& m) throw std::runtime_error("Message name '" + message_name + "' not found in the JSON database"); } } - body_pyinst = nb::inst_alloc(body_pytype); if (message_name == "UNKNOWN") diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index 088b5c337..473a2b693 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -29,8 +29,6 @@ class PyMessageDatabase final : public MessageDatabase [[nodiscard]] const std::unordered_map& GetEnumsByIdDict() const { return enums_by_id; } [[nodiscard]] const std::unordered_map& GetEnumsByNameDict() const { return enums_by_name; } - - private: void GenerateMappings() override; //----------------------------------------------------------------------- @@ -41,17 +39,18 @@ class PyMessageDatabase final : public MessageDatabase void UpdatePythonEnums(); //----------------------------------------------------------------------- //! \brief Creates Python types for each component of all message definitions in the database. - //! + //! //! A message named "MESSAGE" will be mapped to a Python class named "MESSAGE". //! A field of that body named "FIELD" will be mapped to a class named "MESSAGE_FIELD_Field". //! A subfield of that field named "SUBFIELD" will be mapped to a class named "MESSAGE_FIELD_Field_SUBFIELD_Field". - //! + //! //! These classes are stored by name in the messages_by_name map. //----------------------------------------------------------------------- void UpdatePythonMessageTypes(); - void AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, nb::handle type_dict); + void AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, + nb::handle type_dict); - std::unordered_map messages_by_name{}; + std::unordered_map messages_by_name{}; std::unordered_map fields_by_name{}; std::unordered_map enums_by_id{}; diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index b196360c7..b561cff20 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -30,8 +30,8 @@ struct PyHeader : public IntermediateHeader //============================================================================ //! \class PyField //! \brief A python representation for a single log message or message field. -//! -//! Contains a vector of FieldContainer objects, which behave like attributes +//! +//! Contains a vector of FieldContainer objects, which behave like attributes //! within the Python API. //============================================================================ struct PyField @@ -59,11 +59,10 @@ struct PyField PyMessageDatabase::ConstPtr parent_db_; }; - //============================================================================ //! \class PyMessage //! \brief A python representation for a single log message. -//! +//! //! Extends PyField with reference to the Python represenation of a Header. //============================================================================ struct PyMessage : public PyField @@ -72,7 +71,7 @@ struct PyMessage : public PyField PyHeader header; PyMessage(std::string name_, bool has_ptype_, std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, PyHeader header_) - : PyField(std::move(name_), has_ptype_, std::move(fields_), std::move(parent_db_)), header(std::move(header_)) + : PyField(std::move(name_), has_ptype_, std::move(fields_), std::move(parent_db_)), header(std::move(header_)) { } }; @@ -80,7 +79,7 @@ struct PyMessage : public PyField //============================================================================ //! \class UnknownMessage //! \brief A python representation for an unknown log message. -//! +//! //! Contains the raw bytes of the message. //============================================================================ struct UnknownMessage : public PyMessage diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index b702b6bb0..604d8b132 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -109,28 +109,25 @@ def main(): if not filter.do_filtering(meta): continue - if meta.message_id == 39: - pass - - # Decode the log body. - body = frame[meta.header_length:] - status, message = message_decoder.decode(body, header, meta) - status.raise_on_error("MessageDecoder.decode() failed") + # Decode the log body. + body = frame[meta.header_length:] + status, message = message_decoder.decode(body, header, meta) + status.raise_on_error("MessageDecoder.decode() failed") # Get info from the log. - # if isinstance(message, RANGE): - # obs = message.obs - # for ob in obs: - # value = ob.psr - # pass + if isinstance(message, RANGE): + obs = message.obs + for ob in obs: + value = ob.psr + pass - # # Re-encode the log and write it to the output file. - # status, encoded_message = encoder.encode(message, meta, encode_format) - # status.raise_on_error("Encoder.encode() failed") + # Re-encode the log and write it to the output file. + status, encoded_message = encoder.encode(message, meta, encode_format) + status.raise_on_error("Encoder.encode() failed") - # converted_logs_stream.write(encoded_message.message) - # logger.info( f"Encoded ({len(encoded_message.message)}): {format_frame(encoded_message.message, encode_format)}") + converted_logs_stream.write(encoded_message.message) + logger.info( f"Encoded ({len(encoded_message.message)}): {format_frame(encoded_message.message, encode_format)}") except ne.DecoderException as e: logger.warn(str(e))