diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 7ed2bc303..123dd1d7d 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -228,7 +228,7 @@ jobs: - name: Generate documentation for EDIE run: | sphinx-build docs docs/build - python3 scripts/database_to_html.py database/messages_public.json + python3 scripts/database_to_html.py database/database.json - name: Archive documentation artifacts uses: actions/upload-artifact@v4.3.0 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index fbc32c4d2..117024301 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -23,8 +23,8 @@ jobs: # - [Linux, ubuntu-latest, i686] - [Windows, windows-latest, AMD64] # - [Windows, windows-latest, x86] - - [Windows, windows-latest, ARM64] - - [MacOS, macos-latest, x86_64] + # - [Windows, windows-latest, ARM64] + # - [MacOS, macos-latest, x86_64] - [MacOS, macos-latest, arm64] # spdlog fails to build with the correct architecture on universal2 # - [MacOS, macos-latest, universal2] diff --git a/.gitignore b/.gitignore index 18357e305..4c54cfaf9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,12 @@ build/ docs/build bin/ +out/ +CMakeFiles/ +**.cmake +**.dir/ +Debug/ +conan_host_profile CMakeSettings.json CMakeUserPresets.json /include/novatel_edie/version.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d151bca2..b949f6b92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,5 +82,5 @@ if(BUILD_BENCHMARKS OR BUILD_EXAMPLES OR BUILD_TESTS) endif() install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -install(FILES database/messages_public.json DESTINATION ${CMAKE_INSTALL_DATADIR}/novatel_edie) +install(FILES database/database.json DESTINATION ${CMAKE_INSTALL_DATADIR}/novatel_edie) install_novatel_edie_cmake_config() diff --git a/benchmarks/benchmark.cpp b/benchmarks/benchmark.cpp index 1f718d116..644e3da28 100644 --- a/benchmarks/benchmark.cpp +++ b/benchmarks/benchmark.cpp @@ -212,7 +212,7 @@ int main(int argc, char** argv) { if (argc < 2) { throw std::invalid_argument("1 argument required.\nUsage: [benchmark options]"); } - std::string strDatabaseVar = std::string(argv[1]) + "/database/messages_public.json"; + std::string strDatabaseVar = std::string(argv[1]) + "/database/database.json"; #ifdef _WIN32 if (_putenv_s("TEST_DATABASE_PATH", strDatabaseVar.c_str()) != 0) { throw std::runtime_error("Failed to set db path."); } diff --git a/conanfile.py b/conanfile.py index a753a6f55..cb2d79827 100644 --- a/conanfile.py +++ b/conanfile.py @@ -104,5 +104,5 @@ def package_info(self): "edie_decoders_common", "edie_common", ] - db_path = os.path.join(self.package_folder, "res", "novatel_edie", "messages_public.json") + db_path = os.path.join(self.package_folder, "res", "novatel_edie", "database.json") self.runenv_info.define_path("EDIE_DATABASE_FILE", db_path) diff --git a/database/messages_public.json b/database/database.json similarity index 99% rename from database/messages_public.json rename to database/database.json index eaff6fea4..e55c0fdcc 100644 --- a/database/messages_public.json +++ b/database/database.json @@ -166879,4 +166879,4 @@ "subset": "all", "version": "0.3.0" } -} \ No newline at end of file +} diff --git a/examples/novatel/command_encoding/command_encoding.cpp b/examples/novatel/command_encoding/command_encoding.cpp index 2867f6ce0..3e325aa17 100644 --- a/examples/novatel/command_encoding/command_encoding.cpp +++ b/examples/novatel/command_encoding/command_encoding.cpp @@ -52,7 +52,7 @@ int main(int argc, char* argv[]) if (argc < 3) { logger->error("Format: command_encoding \n"); - logger->error("Example: command_encoding database/messages_public.json ASCII \"RTKTIMEOUT 30\"\n"); + logger->error("Example: command_encoding database/database.json ASCII \"RTKTIMEOUT 30\"\n"); return 1; } diff --git a/pyproject.toml b/pyproject.toml index 00f321000..614024bb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ requires = [ "scikit-build-core >=0.4.3", "nanobind >=2.0.0", "conan >=2.4.0", + "typer >=0.15" ] build-backend = "scikit_build_core.build" @@ -34,10 +35,10 @@ build-backend = "scikit_build_core.build" [tool.scikit-build] cmake.minimum-version = "3.24" -cmake.targets = ["python_bindings"] +cmake.targets = ["python_bindings", "bindings_stub", "dynamic_stubs"] install.components = ["python"] wheel.license-files = ["LICENSE"] -build-dir = "build/{wheel_tag}" +build-dir = "out/build/{wheel_tag}" # Build stable ABI wheels for CPython 3.12+ wheel.py-api = "cp312" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 14383593c..2f128b15b 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -38,7 +38,7 @@ nanobind_add_module(python_bindings ${CMAKE_CURRENT_SOURCE_DIR}/bindings/message_decoder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/nexcept.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/oem_common.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/bindings/oem_enums.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/bindings/init_modules.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/parser.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/range_decompressor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bindings/rxconfig_handler.cpp @@ -54,6 +54,23 @@ target_compile_definitions(python_bindings PRIVATE HAVE_SNPRINTF ) +nanobind_add_stub( + bindings_stub + MODULE bindings + OUTPUT stubs/bindings.pyi + PYTHON_PATH $ + DEPENDS bindings +) + +add_custom_target( + dynamic_stubs ALL + COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie_customizer/novatel_edie_customizer/stubgen.py ${CMAKE_SOURCE_DIR}/database/database.json ${CMAKE_CURRENT_BINARY_DIR}/stubs + COMMENT "Generating dynamic stubs" + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/novatel_edie_customizer/novatel_edie_customizer/stubgen.py + ${CMAKE_SOURCE_DIR}/database/database.json +) + if(DEFINED SKBUILD_PROJECT_NAME) set(PYTHON_INSTALL_DIR ${SKBUILD_PROJECT_NAME}) elseif(NOT DEFINED PYTHON_INSTALL_DIR) @@ -66,6 +83,9 @@ install(TARGETS python_bindings install(DIRECTORY novatel_edie/ DESTINATION ${PYTHON_INSTALL_DIR} COMPONENT python) -install(FILES ${CMAKE_SOURCE_DIR}/database/messages_public.json +install(FILES ${CMAKE_SOURCE_DIR}/database/database.json + DESTINATION ${PYTHON_INSTALL_DIR} + COMPONENT python) +install (DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/stubs/ DESTINATION ${PYTHON_INSTALL_DIR} COMPONENT python) diff --git a/python/bindings/bindings.cpp b/python/bindings/bindings.cpp index a4e73c4b5..639b7651c 100644 --- a/python/bindings/bindings.cpp +++ b/python/bindings/bindings.cpp @@ -16,6 +16,7 @@ void init_novatel_framer(nb::module_&); void init_novatel_header_decoder(nb::module_&); void init_novatel_message_decoder(nb::module_&); void init_novatel_oem_enums(nb::module_&); +void init_novatel_oem_messages(nb::module_&); void init_novatel_parser(nb::module_&); void init_novatel_range_decompressor(nb::module_&); void init_novatel_rxconfig_handler(nb::module_&); @@ -25,7 +26,6 @@ NB_MODULE(bindings, m) init_common_common(m); init_common_logger(m); init_common_json_db_reader(m); - init_common_message_database(m); init_common_nexcept(m); init_novatel_commander(m); init_novatel_common(m); @@ -35,8 +35,12 @@ NB_MODULE(bindings, m) init_novatel_framer(m); init_novatel_header_decoder(m); init_novatel_message_decoder(m); - init_novatel_oem_enums(m); + init_common_message_database(m); init_novatel_parser(m); - init_novatel_range_decompressor(m); init_novatel_rxconfig_handler(m); + init_novatel_range_decompressor(m); + nb::module_ messages_mod = m.def_submodule("messages", "NovAtel OEM message definitions."); + init_novatel_oem_messages(messages_mod); + nb::module_ enums_mod = m.def_submodule("enums", "Enumerations used by NovAtel OEM message fields."); + init_novatel_oem_enums(enums_mod); } diff --git a/python/bindings/encoder.cpp b/python/bindings/encoder.cpp index d7e796172..3011b74aa 100644 --- a/python/bindings/encoder.cpp +++ b/python/bindings/encoder.cpp @@ -20,9 +20,10 @@ void init_novatel_encoder(nb::module_& m) .def_prop_ro("logger", [](const oem::Encoder& encoder) { return encoder.GetLogger(); }) .def( "encode", - [](oem::Encoder& encoder, const oem::IntermediateHeader& header, const oem::PyDecodedMessage& py_message, + [](oem::Encoder& encoder, const oem::PyMessage& py_message, const oem::MetaDataStruct& metadata, ENCODE_FORMAT format) { MessageDataStruct message_data; + oem::IntermediateHeader* header_cinst = nb::inst_ptr(py_message.header); if (format == ENCODE_FORMAT::JSON) { // Allocate more space for JSON messages. @@ -31,7 +32,7 @@ void init_novatel_encoder(nb::module_& m) uint8_t buffer[MESSAGE_SIZE_MAX * 3]; auto* buf_ptr = reinterpret_cast(&buffer); uint32_t buf_size = MESSAGE_SIZE_MAX * 3; - STATUS status = encoder.Encode(&buf_ptr, buf_size, header, py_message.fields, message_data, metadata, format); + STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, py_message.fields, message_data, metadata, format); return nb::make_tuple(status, oem::PyMessageData(message_data)); } else @@ -39,9 +40,9 @@ void init_novatel_encoder(nb::module_& m) uint8_t buffer[MESSAGE_SIZE_MAX]; auto buf_ptr = reinterpret_cast(&buffer); uint32_t buf_size = MESSAGE_SIZE_MAX; - STATUS status = encoder.Encode(&buf_ptr, buf_size, header, py_message.fields, message_data, metadata, format); + STATUS status = encoder.Encode(&buf_ptr, buf_size, *header_cinst, py_message.fields, message_data, metadata, format); return nb::make_tuple(status, oem::PyMessageData(message_data)); } }, - "header"_a, "message"_a, "metadata"_a, "encode_format"_a); + "message"_a, "metadata"_a, "encode_format"_a); } diff --git a/python/bindings/oem_enums.cpp b/python/bindings/init_modules.cpp similarity index 55% rename from python/bindings/oem_enums.cpp rename to python/bindings/init_modules.cpp index 183f325f1..cdb2499f2 100644 --- a/python/bindings/oem_enums.cpp +++ b/python/bindings/init_modules.cpp @@ -7,9 +7,16 @@ using namespace novatel::edie; void init_novatel_oem_enums(nb::module_& m) { - nb::module_ enums_mod = m.def_submodule("enums", "Enumerations used by NovAtel OEM message fields."); for (const auto& [name, enum_type] : MessageDbSingleton::get()->GetEnumsByNameDict()) // { - enums_mod.attr(name.c_str()) = enum_type; + m.attr(name.c_str()) = enum_type; + } +} + +void init_novatel_oem_messages(nb::module_& m) +{ + for (const auto& [name, message_type] : MessageDbSingleton::get()->GetMessagesByNameDict()) + { + m.attr(name.c_str()) = message_type; } } diff --git a/python/bindings/json_db_reader.cpp b/python/bindings/json_db_reader.cpp index fbeb7c8e7..264da4e08 100644 --- a/python/bindings/json_db_reader.cpp +++ b/python/bindings/json_db_reader.cpp @@ -8,31 +8,40 @@ namespace nb = nanobind; using namespace nb::literals; using namespace novatel::edie; -namespace { -std::string default_json_db_path() -{ - // Does the following, but using nanobind: - // import importlib_resources - // path_ctx = importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("messages_public.json")) - // with path_ctx as path: - // return path - nb::object ir = nb::module_::import_("importlib_resources"); - nb::object path_ctx = ir.attr("as_file")(ir.attr("files")("novatel_edie").attr("joinpath")("messages_public.json")); - auto py_path = path_ctx.attr("__enter__")(); - if (!nb::cast(py_path.attr("is_file")())) - { - throw NExcept((std::string("Could not find the default JSON DB file at ") + nb::str(py_path).c_str()).c_str()); - } - auto path = nb::cast(nb::str(py_path)); - path_ctx.attr("__exit__")(nb::none(), nb::none(), nb::none()); - return path; -} -} // namespace - PyMessageDatabase::Ptr& MessageDbSingleton::get() { static PyMessageDatabase::Ptr json_db = nullptr; - if (!json_db) { json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path())); } + + // If the database has already been loaded, return it + if (json_db) { return json_db; } + + // Import necessary modules for locating the default database + nb::object builtins = nb::module_::import_("builtins"); + nb::object os_path = nb::module_::import_("os.path"); + nb::object import_lib_util = nb::module_::import_("importlib.util"); + + // Determine if the novatel_edie package exists within the current Python environment + nb::object module_spec = import_lib_util.attr("find_spec")("novatel_edie"); + bool module_exists = nb::cast(builtins.attr("bool")(import_lib_util.attr("find_spec")("novatel_edie"))); + // If the package does not exist, return an empty database + if (!module_exists) { + json_db = std::make_shared(MessageDatabase()); + return json_db; + } + + // Determine the path to the database file within the novatel_edie package + nb::object novatel_edie_path = os_path.attr("dirname")(module_spec.attr("origin")); + nb::object db_path = os_path.attr("join")(novatel_edie_path, "database.json"); + // If the database file does not exist, return an empty database + bool db_exists = nb::cast(os_path.attr("isfile")(db_path)); + if (!db_exists) + { + json_db = std::make_shared(MessageDatabase()); + return json_db; + } + // If the database file exists, load it + std::string default_json_db_path = nb::cast(db_path); + json_db = std::make_shared(*JsonDbReader::LoadFile(default_json_db_path)); return json_db; } diff --git a/python/bindings/message_database.cpp b/python/bindings/message_database.cpp index fa8f04810..314f78fc9 100644 --- a/python/bindings/message_database.cpp +++ b/python/bindings/message_database.cpp @@ -4,6 +4,7 @@ #include "bindings_core.hpp" #include "py_database.hpp" +#include "py_decoded_message.hpp" namespace nb = nanobind; using namespace nb::literals; @@ -183,26 +184,51 @@ void init_common_message_database(nb::module_& m) .def("get_msg_def", nb::overload_cast(&PyMessageDatabase::GetMsgDef, nb::const_), "msg_id"_a) .def("get_enum_def", &PyMessageDatabase::GetEnumDefId, "enum_id"_a) .def("get_enum_def", &PyMessageDatabase::GetEnumDefName, "enum_name"_a) - .def_prop_ro("enums", &PyMessageDatabase::GetEnumsByNameDict); + .def_prop_ro("enums", &PyMessageDatabase::GetEnumsByNameDict) + .def_prop_ro("messages", &PyMessageDatabase::GetMessagesByNameDict); } -PyMessageDatabase::PyMessageDatabase() { UpdatePythonEnums(); } +PyMessageDatabase::PyMessageDatabase() +{ + UpdatePythonEnums(); + UpdatePythonMessageTypes(); +} PyMessageDatabase::PyMessageDatabase(std::vector vMessageDefinitions_, std::vector vEnumDefinitions_) : MessageDatabase(std::move(vMessageDefinitions_), std::move(vEnumDefinitions_)) { UpdatePythonEnums(); + UpdatePythonMessageTypes(); } -PyMessageDatabase::PyMessageDatabase(const MessageDatabase& message_db) noexcept : MessageDatabase(message_db) { UpdatePythonEnums(); } +PyMessageDatabase::PyMessageDatabase(const MessageDatabase& message_db) noexcept : MessageDatabase(message_db) +{ + UpdatePythonEnums(); + UpdatePythonMessageTypes(); +} -PyMessageDatabase::PyMessageDatabase(const MessageDatabase&& message_db) noexcept : MessageDatabase(message_db) { UpdatePythonEnums(); } +PyMessageDatabase::PyMessageDatabase(const MessageDatabase&& message_db) noexcept : MessageDatabase(message_db) +{ + UpdatePythonEnums(); + UpdatePythonMessageTypes(); +} void PyMessageDatabase::GenerateMappings() { MessageDatabase::GenerateMappings(); UpdatePythonEnums(); + UpdatePythonMessageTypes(); +} + +void cleanString(std::string& str) +{ + // Remove special characters from the string to make it a valid python attribute name + for (char& c : str) + { + if (!isalnum(c)) { c = '_'; } + } + if (isdigit(str[0])) { str = "_" + str; } } inline void PyMessageDatabase::UpdatePythonEnums() @@ -214,7 +240,12 @@ inline void PyMessageDatabase::UpdatePythonEnums() { nb::dict values; const char* enum_name = enum_def->name.c_str(); - for (const auto& enumerator : enum_def->enumerators) { values[enumerator.name.c_str()] = enumerator.value; } + for (const auto& enumerator : enum_def->enumerators) + { + std::string enumerator_name = enumerator.name; + cleanString(enumerator_name); + values[enumerator_name.c_str()] = enumerator.value; + } nb::object enum_type = IntEnum(enum_name, values); enum_type.attr("_name") = enum_name; enum_type.attr("_id") = enum_def->_id; @@ -222,3 +253,46 @@ inline void PyMessageDatabase::UpdatePythonEnums() enums_by_name[enum_name] = enum_type; } } + +void PyMessageDatabase::AddFieldType(std::vector> fields, std::string base_name, nb::handle type_constructor, + nb::handle type_tuple, nb::handle type_dict) +{ + // rescursively add field types for each field array element within the provided vector + for (const auto& field : fields) + { + if (field->type == FIELD_TYPE::FIELD_ARRAY) + { + auto* field_array_field = dynamic_cast(field.get()); + std::string field_name = base_name + "_" + field_array_field->name + "_Field"; + nb::object field_type = type_constructor(field_name, type_tuple, type_dict); + messages_by_name[field_name] = field_type; + AddFieldType(field_array_field->fields, field_name, type_constructor, type_tuple, type_dict); + } + } +} + +void PyMessageDatabase::UpdatePythonMessageTypes() +{ + // clear existing definitions + messages_by_name.clear(); + + // get type constructor + nb::object type_constructor = nb::module_::import_("builtins").attr("type"); + // specify the python superclasses for the new message and message body types + nb::tuple message_type_tuple = nb::make_tuple(nb::type()); + nb::tuple field_type_tuple = nb::make_tuple(nb::type()); + // provide no additional attributes via `__dict__` + nb::dict type_dict = nb::dict(); + + // add message and message body types for each message definition + for (const auto& message_def : MessageDefinitions()) + { + nb::object msg_body_def = type_constructor(message_def->name, message_type_tuple, type_dict); + messages_by_name[message_def->name] = msg_body_def; + // add additional MessageBody types for each field array element within the message definition + AddFieldType(message_def->fields.at(message_def->latestMessageCrc), message_def->name, type_constructor, field_type_tuple, type_dict); + } + // provide UNKNOWN types for undecodable messages + nb::object default_msg_body_def = type_constructor("UNKNOWN", message_type_tuple, type_dict); + messages_by_name["UNKNOWN"] = default_msg_body_def; +} diff --git a/python/bindings/message_db_singleton.hpp b/python/bindings/message_db_singleton.hpp index 017cbef60..becc77d5c 100644 --- a/python/bindings/message_db_singleton.hpp +++ b/python/bindings/message_db_singleton.hpp @@ -5,11 +5,27 @@ namespace novatel::edie { +//============================================================================ +//! \class MessageDbSingleton +//! \brief The singular default MessageDatabase based on the JSON database +//! file included in the novatel_edie package. +//! +//! If the package does not contain a JSON database file or the C++ bindings +//! submodule is imported outside of the package, the default database will +//! be empty. +//============================================================================ class MessageDbSingleton { public: MessageDbSingleton() = delete; + + + //---------------------------------------------------------------------------- + //! \brief Method to get the MessageDbSingleton. + //! + //! If the instance does not yet exist, it will be created and returned. + //---------------------------------------------------------------------------- [[nodiscard]] static PyMessageDatabase::Ptr& get(); }; -} // namespace novatel::edie \ No newline at end of file +} // namespace novatel::edie diff --git a/python/bindings/message_decoder.cpp b/python/bindings/message_decoder.cpp index 863f171eb..e72cbef8a 100644 --- a/python/bindings/message_decoder.cpp +++ b/python/bindings/message_decoder.cpp @@ -1,6 +1,8 @@ #include "novatel_edie/decoders/oem/message_decoder.hpp" #include +#include +#include #include #include "bindings_core.hpp" @@ -15,10 +17,11 @@ using namespace novatel::edie::oem; NB_MAKE_OPAQUE(std::vector); -nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::ConstPtr& parent_db) +nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::ConstPtr& parent_db, std::string parent) { if (field.fieldDef->type == FIELD_TYPE::ENUM) { + // Handle Enums const std::string& enumId = static_cast(field.fieldDef.get())->enumId; auto it = parent_db->GetEnumsByIdDict().find(enumId); if (it == parent_db->GetEnumsByIdDict().end()) @@ -34,14 +37,38 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C const auto& message_field = std::get>(field.fieldValue); if (message_field.empty()) { - // Empty array + // Handle Empty Arrays return nb::list(); } - else if (message_field[0].fieldDef->type == field.fieldDef->type && message_field[0].fieldDef->name == field.fieldDef->name) + else if (field.fieldDef->type == FIELD_TYPE::FIELD_ARRAY) { - // Fixed-length, variable-length and field arrays are stored as a field - // with a list of sub-fields of the same type and name. - // This needs to be un-nested for the translated Python structure. + // Handle Field Arrays + std::vector sub_values; + sub_values.reserve(message_field.size()); + nb::handle field_ptype; + std::string field_name = parent + "_" + field.fieldDef->name + "_Field"; + try + { + field_ptype = parent_db->GetMessagesByNameDict().at(field_name); + } + catch (const std::out_of_range& e) + { + field_ptype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + } + for (const auto& subfield : message_field) + { + nb::object pyinst = nb::inst_alloc(field_ptype); + PyField* cinst = nb::inst_ptr(pyinst); + const auto& message_subfield = std::get>(subfield.fieldValue); + new (cinst) PyField(message_subfield, parent_db, field_name); + nb::inst_mark_ready(pyinst); + sub_values.push_back(pyinst); + } + return nb::cast(sub_values); + } + else + { + // Handle Fixed or Variable-Length Arrays if (field.fieldDef->conversion == "%s") { // The array is actually a string @@ -57,17 +84,13 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } std::vector sub_values; sub_values.reserve(message_field.size()); - for (const auto& f : message_field) { sub_values.push_back(convert_field(f, parent_db)); } + for (const auto& f : message_field) { sub_values.push_back(convert_field(f, parent_db, parent)); } return nb::cast(sub_values); } - else - { - // This is an array element of a field array. - return nb::cast(PyDecodedMessage(message_field, {}, parent_db)); - } } else if (field.fieldDef->conversion == "%id") { + // Handle Satellite IDs const uint32_t temp_id = std::get(field.fieldValue); SatelliteId sat_id; sat_id.usPrnOrSlot = temp_id & 0x0000FFFF; @@ -81,22 +104,21 @@ nb::object convert_field(const FieldContainer& field, const PyMessageDatabase::C } } -PyDecodedMessage::PyDecodedMessage(std::vector message_, const oem::MetaDataStruct& meta_, PyMessageDatabase::ConstPtr parent_db_) - : fields(std::move(message_)), message_id(meta_.usMessageId), message_crc(meta_.uiMessageCrc), message_name(meta_.acMessageName), time(meta_), - measurement_source(meta_.eMeasurementSource), constellation(meta_.constellation), parent_db_(std::move(parent_db_)) +PyField::PyField(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_) + : name(std::move(name_)), fields(std::move(message_)), parent_db_(std::move(parent_db_)) { } -nb::dict& PyDecodedMessage::get_values() const +nb::dict& PyField::get_values() const { if (cached_values_.size() == 0) { - for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_); } + for (const auto& field : fields) { cached_values_[nb::cast(field.fieldDef->name)] = convert_field(field, parent_db_, this->name); } } return cached_values_; } -nb::dict& PyDecodedMessage::get_fields() const +nb::dict& PyField::get_fields() const { if (cached_fields_.size() == 0) { @@ -105,18 +127,18 @@ nb::dict& PyDecodedMessage::get_fields() const return cached_fields_; } -nb::dict PyDecodedMessage::to_dict() const +nb::dict PyField::to_dict() const { nb::dict dict; for (const auto& [field_name, value] : get_values()) { - if (nb::isinstance(value)) { dict[field_name] = nb::cast(value).to_dict(); } + if (nb::isinstance(value)) { dict[field_name] = nb::cast(value).to_dict(); } else if (nb::isinstance>(value)) { nb::list list; for (const auto& sub_item : nb::cast>(value)) { - if (nb::isinstance(sub_item)) { list.append(nb::cast(sub_item).to_dict()); } + if (nb::isinstance(sub_item)) { list.append(nb::cast(sub_item).to_dict()); } else { list.append(sub_item); } } dict[field_name] = list; @@ -126,23 +148,22 @@ nb::dict PyDecodedMessage::to_dict() const return dict; } -nb::object PyDecodedMessage::getattr(nb::str field_name) const +nb::object PyField::getattr(nb::str field_name) const { if (!contains(field_name)) { throw nb::attribute_error(field_name.c_str()); } return get_values()[std::move(field_name)]; } -nb::object PyDecodedMessage::getitem(nb::str field_name) const { return get_values()[std::move(field_name)]; } +nb::object PyField::getitem(nb::str field_name) const { return get_values()[std::move(field_name)]; } -bool PyDecodedMessage::contains(nb::str field_name) const { return get_values().contains(std::move(field_name)); } +bool PyField::contains(nb::str field_name) const { return get_values().contains(std::move(field_name)); } -size_t PyDecodedMessage::len() const { return fields.size(); } +size_t PyField::len() const { return fields.size(); } -std::string PyDecodedMessage::repr() const +std::string PyField::repr() const { std::stringstream repr; - repr << "<"; - if (!message_name.empty()) { repr << message_name << " "; } + repr << name << "("; bool first = true; for (const auto& [field_name, value] : get_values()) { @@ -150,7 +171,7 @@ std::string PyDecodedMessage::repr() const first = false; repr << nb::str("{}={!r}").format(field_name, value).c_str(); } - repr << ">"; + repr << ")"; return repr.str(); } @@ -185,22 +206,47 @@ void init_novatel_message_decoder(nb::module_& m) .def_rw("milliseconds", &PyGpsTime::milliseconds) .def_rw("status", &PyGpsTime::time_status); - nb::class_(m, "Message") - .def_rw("message_id", &PyDecodedMessage::message_id) - .def_rw("message_crc", &PyDecodedMessage::message_crc) - .def_rw("message_name", &PyDecodedMessage::message_name) - .def_rw("message_time", &PyDecodedMessage::time) - .def_rw("message_measurement_source", &PyDecodedMessage::measurement_source) - .def_rw("message_constellation", &PyDecodedMessage::constellation) - .def_prop_ro("_values", &PyDecodedMessage::get_values) - .def_prop_ro("_fields", &PyDecodedMessage::get_fields) - .def("to_dict", &PyDecodedMessage::to_dict, "Convert the message and its sub-messages into a dict") - .def("__getattr__", &PyDecodedMessage::getattr, "field_name"_a) - .def("__getitem__", &PyDecodedMessage::getitem, "field_name"_a) - .def("__contains__", &PyDecodedMessage::contains, "field_name"_a) - .def("__len__", &PyDecodedMessage::len) - .def("__repr__", &PyDecodedMessage::repr) - .def("__str__", &PyDecodedMessage::repr); + nb::class_(m, "Field") + .def_prop_ro("_values", &PyField::get_values) + .def_prop_ro("_fields", &PyField::get_fields) + .def("to_dict", &PyField::to_dict, "Convert the message and its sub-messages into a dict") + .def("__getattr__", &PyField::getattr, "field_name"_a) + .def("__repr__", &PyField::repr) + .def("__str__", &PyField::repr) + .def("__dir__", [](nb::object self) { + // get required Python builtin functions + nb::module_ builtins = nb::module_::import_("builtins"); + nb::handle super = builtins.attr("super"); + nb::handle type = builtins.attr("type"); + + nb::handle current_type = type(self); + std::string current_type_name = nb::cast(current_type.attr("__name__")); + while (current_type_name != "Field") + { + current_type = (current_type.attr("__bases__"))[0]; + current_type_name = nb::cast(current_type.attr("__name__")); + } + + // retrieve base list based on superclass method + nb::object super_obj = super(current_type, self); + nb::list base_list = nb::cast(super_obj.attr("__dir__")()); + // add dynamic fields to the list + PyField* body = nb::inst_ptr(self); + for (const auto& [field_name, _] : body->get_fields()) { base_list.append(field_name); } + + return base_list; + }); + + nb::class_(m, "Message") + .def_ro("header", &PyMessage::header) + .def( + "to_dict", [](const PyMessage& self, bool include_header) { + nb::dict dict = self.to_dict(); + if (include_header) { dict["header"] = self.header.attr("to_dict")(); } + return dict; + }, + "include_header"_a = true, + "Convert the message and its sub-messages into a dict"); nb::class_(m, "FieldContainer") .def_rw("value", &FieldContainer::fieldValue) @@ -216,13 +262,32 @@ void init_novatel_message_decoder(nb::module_& m) .def_prop_ro("logger", [](oem::MessageDecoder& decoder) { return decoder.GetLogger(); }) .def( "decode", - [](const oem::MessageDecoder& decoder, const nb::bytes& message_body, oem::MetaDataStruct& metadata) { + [](const oem::MessageDecoder& decoder, const nb::bytes& message_body, nb::object header, oem::MetaDataStruct& metadata) { std::vector fields; STATUS status = decoder.Decode(reinterpret_cast(message_body.c_str()), fields, metadata); - return nb::make_tuple(status, PyDecodedMessage(std::move(fields), metadata, get_parent_db(decoder))); + PyMessageDatabase::ConstPtr parent_db = get_parent_db(decoder); + nb::handle body_pytype; + const std::string message_name = metadata.MessageName(); + + try + { + body_pytype = parent_db->GetMessagesByNameDict().at(message_name); + } + catch (const std::out_of_range& e) + { + body_pytype = parent_db->GetMessagesByNameDict().at("UNKNOWN"); + } + + nb::object body_pyinst = nb::inst_alloc(body_pytype); + PyMessage* body_cinst = nb::inst_ptr(body_pyinst); + new (body_cinst) PyMessage(fields, parent_db, message_name, header); + nb::inst_mark_ready(body_pyinst); + + //auto message_pyinst = std::make_shared(body_pyinst, header, message_name); + + return nb::make_tuple(status, body_pyinst); }, - "message_body"_a, "metadata"_a) - // For internal testing purposes only + "message_body"_a, "decoded_header"_a, "metadata"_a) .def( "_decode_ascii", [](oem::MessageDecoder& decoder, const std::vector& msg_def_fields, const nb::bytes& message_body) { @@ -231,7 +296,7 @@ void init_novatel_message_decoder(nb::module_& m) std::string body_str(message_body.c_str(), message_body.size()); const char* data_ptr = body_str.c_str(); STATUS status = static_cast(&decoder)->TestDecodeAscii(msg_def_fields, &data_ptr, fields); - return nb::make_tuple(status, PyDecodedMessage(std::move(fields), {}, get_parent_db(decoder))); + return nb::make_tuple(status, PyField(std::move(fields), get_parent_db(decoder), "UNKNOWN")); }, "msg_def_fields"_a, "message_body"_a) .def( @@ -242,7 +307,7 @@ void init_novatel_message_decoder(nb::module_& m) const char* data_ptr = message_body.c_str(); STATUS status = static_cast(&decoder)->TestDecodeBinary(msg_def_fields, reinterpret_cast(&data_ptr), fields, message_length); - return nb::make_tuple(status, PyDecodedMessage(std::move(fields), {}, get_parent_db(decoder))); + return nb::make_tuple(status, PyField(std::move(fields), get_parent_db(decoder), "UNKNOWN")); }, "msg_def_fields"_a, "message_body"_a, "message_length"_a); } diff --git a/python/bindings/oem_common.cpp b/python/bindings/oem_common.cpp index c30b54e38..d416769f5 100644 --- a/python/bindings/oem_common.cpp +++ b/python/bindings/oem_common.cpp @@ -121,6 +121,22 @@ void init_novatel_common(nb::module_& m) .def_rw("receiver_status", &oem::IntermediateHeader::uiReceiverStatus) .def_rw("message_definition_crc", &oem::IntermediateHeader::uiMessageDefinitionCrc) .def_rw("receiver_sw_version", &oem::IntermediateHeader::usReceiverSwVersion) + .def("to_dict", [](const oem::IntermediateHeader& self) { + nb::dict header_dict; + header_dict["message_id"] = self.usMessageId; + header_dict["message_type"] = self.ucMessageType; + header_dict["port_address"] = self.uiPortAddress; + header_dict["length"] = self.usLength; + header_dict["sequence"] = self.usSequence; + header_dict["idle_time"] = self.ucIdleTime; + header_dict["time_status"] = self.uiTimeStatus; + header_dict["week"] = self.usWeek; + header_dict["milliseconds"] = self.dMilliseconds; + header_dict["receiver_status"] = self.uiReceiverStatus; + header_dict["message_definition_crc"] = self.uiMessageDefinitionCrc; + header_dict["receiver_sw_version"] = self.usReceiverSwVersion; + return header_dict; + }) .def("__repr__", [](const nb::handle self) { auto& header = nb::cast(self); return nb::str("Header(message_id={!r}, message_type={!r}, port_address={!r}, length={!r}, sequence={!r}, " diff --git a/python/bindings/py_database.hpp b/python/bindings/py_database.hpp index 4a2a5e8bd..1e7a687a5 100644 --- a/python/bindings/py_database.hpp +++ b/python/bindings/py_database.hpp @@ -15,13 +15,35 @@ class PyMessageDatabase final : public MessageDatabase explicit PyMessageDatabase(const MessageDatabase& message_db) noexcept; explicit PyMessageDatabase(const MessageDatabase&& message_db) noexcept; - [[nodiscard]] const std::unordered_map& GetEnumsByIdDict() const { return enums_by_id; } + [[nodiscard]] const std::unordered_map& GetMessagesByNameDict() const { return messages_by_name; } + [[nodiscard]] const std::unordered_map& GetEnumsByIdDict() const { return enums_by_id; } [[nodiscard]] const std::unordered_map& GetEnumsByNameDict() const { return enums_by_name; } + + private: void GenerateMappings() override; + //----------------------------------------------------------------------- + //! \brief Creates Python Enums for each enum definition in the database. + //! + //! These classes are stored by ID in the enums_by_id map and by name in the enums_by_name map. + //----------------------------------------------------------------------- void UpdatePythonEnums(); + //----------------------------------------------------------------------- + //! \brief Creates Python types for each component of all message definitions in the database. + //! + //! A message named "MESSAGE" will be mapped to a Python class named "MESSAGE". + //! The message's body will be mapped to a class named "MESSAGE_Body". + //! A field of that body named "FIELD" will be mapped to a class named "MESSAGE_Body_FIELD_Field". + //! A subfield of that field named "SUBFIELD" will be mapped to a class named "MESSAGE_Body_FIELD_Field_SUBFIELD_Field". + //! + //! These classes are stored by name in the messages_by_name map. + //----------------------------------------------------------------------- + void UpdatePythonMessageTypes(); + void AddFieldType(std::vector> fields, std::string base_name, nb::handle type_cons, nb::handle type_tuple, nb::handle type_dict); + + std::unordered_map messages_by_name{}; std::unordered_map enums_by_id{}; std::unordered_map enums_by_name{}; @@ -31,4 +53,4 @@ class PyMessageDatabase final : public MessageDatabase using ConstPtr = std::shared_ptr; }; -} // namespace novatel::edie \ No newline at end of file +} // namespace novatel::edie diff --git a/python/bindings/py_decoded_message.hpp b/python/bindings/py_decoded_message.hpp index 705036b35..dcaefea9a 100644 --- a/python/bindings/py_decoded_message.hpp +++ b/python/bindings/py_decoded_message.hpp @@ -18,9 +18,10 @@ struct PyGpsTime TIME_STATUS time_status{TIME_STATUS::UNKNOWN}; }; -struct PyDecodedMessage +struct PyField { - explicit PyDecodedMessage(std::vector message_, const MetaDataStruct& meta_, PyMessageDatabase::ConstPtr parent_db_); + std::string name; + explicit PyField(std::vector message_, PyMessageDatabase::ConstPtr parent_db_, std::string name_); nb::dict& get_values() const; nb::dict& get_fields() const; nb::dict to_dict() const; @@ -32,14 +33,6 @@ struct PyDecodedMessage std::vector fields; - // MetaDataStruct with all fields that are no longer relevant after decoding dropped. - uint16_t message_id; - uint32_t message_crc; - std::string message_name; - PyGpsTime time; - MEASUREMENT_SOURCE measurement_source{MEASUREMENT_SOURCE::PRIMARY}; - CONSTELLATION constellation{CONSTELLATION::UNKNOWN}; - private: mutable nb::dict cached_values_; mutable nb::dict cached_fields_; @@ -47,4 +40,29 @@ struct PyDecodedMessage PyMessageDatabase::ConstPtr parent_db_; }; -} // namespace novatel::edie::oem \ No newline at end of file +struct PyMessage : public PyField +{ + public: + nb::object header; + + PyMessage(std::vector fields_, PyMessageDatabase::ConstPtr parent_db_, std::string name_, nb::object header_) + : PyField(std::move(fields_), std::move(parent_db_), std::move(name_)), header(std::move(header_)) {} +}; + +//struct PyMessage +//{ +// nb::object message_body; +// nb::object header; +// std::string name; +// +// PyMessage(nb::object message_body_, nb::object header_, std::string name_) : message_body(message_body_), header(header_), name(name_) {} +// +// std::string repr() const +// { +// std::stringstream repr; +// repr << ""; +// return repr.str(); +// } +//}; + +} // namespace novatel::edie::oem diff --git a/python/examples/converter_components.py b/python/examples/converter_components.py index f102068bb..604d8b132 100755 --- a/python/examples/converter_components.py +++ b/python/examples/converter_components.py @@ -34,7 +34,9 @@ from binascii import hexlify import novatel_edie as ne +from novatel_edie.messages import RANGE from novatel_edie import STATUS +from novatel_edie.enums import Datum def read_frames(input_file, framer): @@ -109,11 +111,19 @@ def main(): # Decode the log body. body = frame[meta.header_length:] - status, message = message_decoder.decode(body, meta) + status, message = message_decoder.decode(body, header, meta) status.raise_on_error("MessageDecoder.decode() failed") + # Get info from the log. + if isinstance(message, RANGE): + obs = message.obs + for ob in obs: + value = ob.psr + pass + + # Re-encode the log and write it to the output file. - status, encoded_message = encoder.encode(header, message, meta, encode_format) + status, encoded_message = encoder.encode(message, meta, encode_format) status.raise_on_error("Encoder.encode() failed") converted_logs_stream.write(encoded_message.message) @@ -121,6 +131,5 @@ def main(): except ne.DecoderException as e: logger.warn(str(e)) - if __name__ == "__main__": main() diff --git a/python/novatel_edie/__init__.py b/python/novatel_edie/__init__.py index 6f35a060d..687835eee 100644 --- a/python/novatel_edie/__init__.py +++ b/python/novatel_edie/__init__.py @@ -1,8 +1,23 @@ import importlib_resources -from .bindings import * - +from .bindings import ( + messages, enums, + HEADER_FORMAT, ENCODE_FORMAT, STATUS, FIELD_TYPE, DATA_TYPE, TIME_STATUS, MEASUREMENT_SOURCE, + MESSAGE_SIZE_MAX, MAX_ASCII_MESSAGE_LENGTH, MAX_SHORT_ASCII_MESSAGE_LENGTH, MAX_BINARY_MESSAGE_LENGTH, + NMEA_SYNC_LENGTH, NMEA_CRC_LENGTH, + OEM4_BINARY_HEADER_LENGTH, OEM4_SHORT_BINARY_HEADER_LENGTH, + OEM4_BINARY_CRC_LENGTH, OEM4_ASCII_CRC_LENGTH, + OEM4_BINARY_SYNC_LENGTH, OEM4_SHORT_ASCII_SYNC_LENGTH, OEM4_ASCII_SYNC_LENGTH, OEM4_SHORT_BINARY_SYNC_LENGTH, + OEM4_BINARY_SYNC1, OEM4_BINARY_SYNC2, OEM4_BINARY_SYNC3, + string_to_encode_format, pretty_version, + Header, Field, Message, + JsonDbReader, MessageDatabase, get_default_database, + Oem4BinaryHeader, Oem4BinaryShortHeader, MetaData, MessageData, MessageDefinition, BaseField, + Framer, Filter, HeaderDecoder, MessageDecoder, DecoderException, Encoder, Commander, Parser, FileParser, + RangeDecompressor, RxConfigHandler, + Logging, LogLevel +) def default_json_db_path(): """Returns a context manager that yields the path to the default JSON database.""" - return importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("messages_public.json")) + return importlib_resources.as_file(importlib_resources.files("novatel_edie").joinpath("database.json")) diff --git a/python/novatel_edie/enums.py b/python/novatel_edie/enums.py new file mode 100644 index 000000000..50ec26f0a --- /dev/null +++ b/python/novatel_edie/enums.py @@ -0,0 +1 @@ +from .bindings.enums import * diff --git a/python/novatel_edie/messages.py b/python/novatel_edie/messages.py new file mode 100644 index 000000000..cc657ab9c --- /dev/null +++ b/python/novatel_edie/messages.py @@ -0,0 +1 @@ +from .bindings.messages import * diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/__init__.py b/python/novatel_edie_customizer/novatel_edie_customizer/__init__.py new file mode 100644 index 000000000..a104f7c5a --- /dev/null +++ b/python/novatel_edie_customizer/novatel_edie_customizer/__init__.py @@ -0,0 +1,23 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py b/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py new file mode 100644 index 000000000..b2ccb37ed --- /dev/null +++ b/python/novatel_edie_customizer/novatel_edie_customizer/__main__.py @@ -0,0 +1,27 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +if __name__ == "__main__": + from .cli import app + app() diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/cli.py b/python/novatel_edie_customizer/novatel_edie_customizer/cli.py new file mode 100644 index 000000000..541bf9578 --- /dev/null +++ b/python/novatel_edie_customizer/novatel_edie_customizer/cli.py @@ -0,0 +1,39 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import typer + +from .stubgen import generate_stubs +from .installer import install_custom + +app = typer.Typer() + +# sub_app = typer.Typer() +# sub_app.command()(generate_stubs) + +app.command()(generate_stubs) +app.command()(install_custom) + +if __name__ == "__main__": + app() diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/installer.py b/python/novatel_edie_customizer/novatel_edie_customizer/installer.py new file mode 100644 index 000000000..3e8e3a937 --- /dev/null +++ b/python/novatel_edie_customizer/novatel_edie_customizer/installer.py @@ -0,0 +1,139 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +A module concerning the installation of a custom package. +""" + +import os +import shutil +import sys +from contextlib import contextmanager +import subprocess + +import typer +from typing_extensions import Annotated + +from novatel_edie_customizer.stubgen import StubGenerator + +@contextmanager +def open_library_clone(library: str): + """Creates a clone of a specified library within a temporary directory. + + Args: + library: The name of the library to clone. Must be within + the current Python environment. + """ + packages_path = os.path.join(sys.exec_prefix, 'Lib', 'site-packages') + library_path = os.path.join(packages_path, library) + if not os.path.exists(library_path): + raise FileNotFoundError(f"Source library {library} not found.") + library_dist_info = [ + d for d in os.listdir(packages_path) + if d.startswith(library) and d.endswith('.dist-info')] + if not library_dist_info: + raise FileNotFoundError( + f"No .dist-info directory found for library {library}.") + library_dist_info = library_dist_info[0] + library_dist_info_path = os.path.join(packages_path, library_dist_info) + + temp_dir = os.path.join(os.getcwd(), 'temp_dir') + destination_dir = os.path.join(temp_dir, 'wheel') + + cwd = os.getcwd() + + try: + if not os.path.exists(destination_dir): + os.makedirs(destination_dir) + shutil.copytree( + library_path, + os.path.join(destination_dir, library), + dirs_exist_ok=True) + shutil.copytree( + library_dist_info_path, + os.path.join(destination_dir, library_dist_info), + dirs_exist_ok=True) + + os.chdir(temp_dir) + yield + finally: + os.chdir(cwd) + shutil.rmtree(temp_dir) + +def copy_file(file_path: str, destination_path: str = None): + """Copies a file to the current working directory. + + Args: + file_path: The path of the file to copy. + destination_path: A relative path within current working directory + to copy the file to. + """ + if not os.path.exists(file_path): + raise FileNotFoundError(f"File {file_path} not found.") + + destination_path = (os.path.basename(file_path) + if destination_path is None + else destination_path) + + destination_path = os.path.join(os.getcwd(), destination_path) + shutil.copy2(file_path, destination_path) + +def install_package(): + """Installs a package in the current working directory.""" + try: + subprocess.check_call([ + sys.executable, '-m', 'wheel', 'pack', './wheel']) + pass + except subprocess.CalledProcessError as e: + print(f"Failed to pack package: {e}") + wheel_files = [f for f in os.listdir('.') if f.endswith('.whl')] + for wheel_file in wheel_files: + try: + subprocess.check_call([ + sys.executable, '-m', 'pip', + 'install', wheel_file, '--force-reinstall']) + except subprocess.CalledProcessError as e: + print(f"Failed to install {wheel_file}: {e}") + +def install_custom( + database: Annotated[ + str, + typer.Argument(help='A path to a database file.') + ]): + """Create a custom installation of novatel_edie based on the provided DB. + + The custom installation will have all messages and enums of the provided + database be directly importable from the 'enums' and 'messages' submodules. + + Args: + database: A path to a database file. + """ + database = os.path.abspath(database) + library_name = 'novatel_edie' + with open_library_clone(library_name): + copy_file( + database, os.path.join('wheel', library_name, 'database.json')) + + database = StubGenerator(database) + database.write_stub_files(os.path.join('wheel', library_name)) + + install_package() diff --git a/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py new file mode 100644 index 000000000..2e87a84ca --- /dev/null +++ b/python/novatel_edie_customizer/novatel_edie_customizer/stubgen.py @@ -0,0 +1,255 @@ +""" +MIT + +Copyright (c) 2023 NovAtel Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +A module concerning the generation of type hint stub files for the novatel_edie package. +""" +import os +import json +import re +import textwrap +from typing import Union + +import typer +from typing_extensions import Annotated + +class StubGenerator: + """Generator of type hint stub files for the novatel_edie package.""" + data_type_to_pytype = { + 'INT': 'int', + 'UNIT': 'int', + 'BOOL': 'bool', + 'CHAR': 'int', + 'UCHAR': 'int', + 'SHORT': 'int', + 'USHORT': 'int', + 'LONG': 'int', + 'ULONG': 'int', + 'LONGLONG': 'int', + 'ULONGLONG': 'int', + 'FLOAT': 'float', + 'DOUBLE': 'float', + 'HEXBYTE': 'int', + 'SATELLITEID': 'SatelliteId', + 'UNKNOWN': 'bytes' + } + def __init__(self, database: Union[str, dict]): + """Initializes a StubGenerator. + + Args: + database: A path to a message database json file or + a dictionary representation of a database. + """ + if isinstance(database, str): + with open(database, 'r') as f: + database = json.load(f) + if not isinstance(database, dict): + raise TypeError( + 'database must be a dict or a path to a JSON database file.') + self.database = database + self.database['enums_by_id'] = {enum['_id']: enum for enum in database['enums']} + + def write_stub_files(self, file_path: str): + """Writes package stub files to the specified directory. + + Args: + file_path: The directory to write the stub files to. + """ + file_specs = { + 'enums.pyi': self.get_enum_stubs(), + 'messages.pyi': self.get_message_stubs()} + + if not os.path.exists(file_path): + os.makedirs(file_path) + for file_name, file_contents in file_specs.items(): + with open(os.path.join(file_path, file_name), 'w') as f: + f.write(file_contents) + + def _convert_enum_def(self, enum_def) -> str: + """Create a type hint string for an enum definition. + + Args: + enum_def: A dictionary containing an enum definition. + Returns: + A string containing a type hint for the enum definition. + """ + type_hint = f'class {enum_def["name"]}(Enum):\n' + for enumerator in enum_def['enumerators']: + name = enumerator['name'] + name = re.sub(r'[)(\-+./\\]', '_', name) + if name[0].isdigit(): + name = f'_{name}' + type_hint += f' {name} = {enumerator["value"]}\n' + return type_hint + + def get_enum_stubs(self) -> str: + """Get a stub string for all enums in the database. + + Returns: + A string containing type hint stubs for all enums in the database. + """ + stub_str = ('from enum import Enum\n' + 'from typing import Any\n\n') + enums = self.database['enums'] + type_hints = [self._convert_enum_def(enum_def) for enum_def in enums] + type_hints_str = '\n'.join(type_hints) + stub_str += type_hints_str + return stub_str + + def _get_field_pytype(self, field: dict, parent: str) -> str: + """Get a type hint string for a field definition. + + Args: + field: A dictionary containing a field definition. + parent: The name of the parent class. May be used for naming + subfield classes. + Returns: + A string containing a type hint for the field definition. + """ + python_type = None + if field['type'] == 'SIMPLE': + python_type = self.data_type_to_pytype.get(field['dataType']['name']) + if field['type'] == 'ENUM': + enum_def = self.database['enums_by_id'].get(field['enumID']) + python_type = enum_def['name'] if enum_def else 'Any' + if field['type'] == 'STRING': + python_type = 'str' + if (field['type'] in { + 'FIXED_LENGTH_ARRAY', + 'VARIABLE_LENGTH_ARRAY'}): + if field['conversionString'] == r'%s': + python_type = 'str' + else: + python_type = f'list[{self.data_type_to_pytype.get(field["dataType"]["name"])}]' + if field['type'] == 'FIELD_ARRAY': + python_type = f'list[{parent}_{field["name"]}_Field]' + if not python_type: + python_type = 'Any' + return python_type + + def _convert_field_array_def(self, field_array_def: dict, parent: str) -> str: + """Convert a field array definition to a type hint string. + + Args: + field_array_def: A dictionary containing a field array definition. + parent: The name of the parent class. The name for the field array + class will be based on this. + Returns: + A string containing type hint stubs for the field array definition. + """ + subfield_hints = [] + + # Create MessageBodyField type hint + name = f'{parent}_{field_array_def["name"]}_Field' + type_hint = f'class {name}(Field):\n' + for field in field_array_def['fields']: + python_type = self._get_field_pytype(field, parent) + type_hint += (' @property\n' + f' def {field["name"]}(self) -> {python_type}: ...\n\n') + # Create hints for any subfields + if field['type'] == 'FIELD_ARRAY': + subfield_hints.append(self._convert_field_array_def(field, name)) + + # Combine all hints + hints = subfield_hints + [type_hint] + hint_str = '\n'.join(hints) + + return hint_str + + def _convert_message_def(self, message_def: dict) -> str: + """Create a type hint string for a message definition. + + Args: + message_def: A dictionary containing a message definition. + Returns: + A string containing type hint stubs for the message definition. + """ + subfield_hints = [] + + # Create the Message type hint + name = message_def['name'] + message_hint = textwrap.dedent(f"""\ + class {name}(Message): + @property + def header(self) -> Header: ... + """) + + # Create the MessageBody type hint + name = message_def["name"] + body_hint = f'class {name}(Message):\n' + fields = message_def['fields'][message_def['latestMsgDefCrc']] + if not fields: + body_hint += ' pass\n\n' + for field in fields: + python_type = self._get_field_pytype(field, name) + body_hint += ' @property\n' + body_hint += f' def {field["name"]}(self) -> {python_type}: ...\n\n' + + # Create hints for any subfields + if field['type'] == 'FIELD_ARRAY': + subfield_hints.append(self._convert_field_array_def(field, name)) + + # Combine all hints + hints = subfield_hints + [body_hint] + hint_str = '\n'.join(hints) + + return hint_str + + def get_message_stubs(self) -> str: + """Get a stub string for all messages in the database. + + Returns: + A string containing type hint stubs for all messages in the database. + """ + stub_str = 'from typing import Any\n\n' + stub_str += 'from novatel_edie import Header, Field, Message, SatelliteId\n' + stub_str += 'from novatel_edie.enums import *\n\n' + + messages = self.database['messages'] + type_hints = [self._convert_message_def(msg_def) + for msg_def in messages] + type_hints_str = '\n'.join(type_hints) + stub_str += type_hints_str + + return stub_str + + +def generate_stubs( + database: Annotated[ + str, + typer.Argument(help='A path to a database file.') + ], + output_dir: Annotated[ + str, + typer.Argument(help='The directory to write stub files to.') + ] = './stubs' + ): + """Generate type hint stub files for a provided database. + + Args: + database: A path to a database file. + output_dir: The directory to write stub files to. + """ + StubGenerator(database).write_stub_files(output_dir) + +if __name__ == '__main__': + typer.run(generate_stubs) diff --git a/python/novatel_edie_customizer/pyproject.toml b/python/novatel_edie_customizer/pyproject.toml new file mode 100644 index 000000000..d10ecd1b3 --- /dev/null +++ b/python/novatel_edie_customizer/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "novatel_edie_customizer" +description = "Tool for creating a custom installation of novatel_edie package" +version = "0.0.1" +license = {text = "MIT"} +requires-python = ">=3.8" +dependencies = [ + "typer>=0.15" +] + + +[project.scripts] +novatel_edie_customizer = "novatel_edie_customizer.cli:app" diff --git a/python/test/test_benchmark.py b/python/test/test_benchmark.py index 66909d037..b51070259 100644 --- a/python/test/test_benchmark.py +++ b/python/test/test_benchmark.py @@ -59,12 +59,12 @@ def run(self, log, encode_format): failed_once = True break body = log[meta_data.header_length:] - status, message = self.message_decoder.decode(body, meta_data) + status, message = self.message_decoder.decode(body, header, meta_data) if status != STATUS.SUCCESS: print("Failed to decode message: ", status) failed_once = True break - status, message_data = self.encoder.encode(header, message, meta_data, encode_format) + status, message_data = self.encoder.encode(message, meta_data, encode_format) if status != STATUS.SUCCESS: print("Failed to encode message: ", status) failed_once = True diff --git a/python/test/test_decode_encode.py b/python/test/test_decode_encode.py index 0b80292c6..b2e05c4ed 100644 --- a/python/test/test_decode_encode.py +++ b/python/test/test_decode_encode.py @@ -28,12 +28,14 @@ # Encoder and Filter. ################################################################################ import enum +from collections import namedtuple import novatel_edie as ne import pytest from novatel_edie import STATUS, ENCODE_FORMAT +from novatel_edie.messages import * from pytest import approx -from collections import namedtuple + # ------------------------------------------------------------------------------------------------------- # Decode/Encode Unit Tests @@ -66,11 +68,11 @@ def DecodeEncode(self, encode_format: ne.ENCODE_FORMAT, message_input: bytes, me return (Result.HEADER_DECODER_ERROR, None) if not return_message else (Result.HEADER_DECODER_ERROR, None, None) body = message_input[meta_data.header_length:] - status, message = self.message_decoder.decode(body, meta_data) + status, message = self.message_decoder.decode(body, header, meta_data) if status != STATUS.SUCCESS: return (Result.MESSAGE_DECODER_ERROR, None) if not return_message else (Result.MESSAGE_DECODER_ERROR, None, None) - status, message_data = self.encoder.encode(header, message, meta_data, encode_format) + status, message_data = self.encoder.encode(message, meta_data, encode_format) if status != STATUS.SUCCESS: return (Result.ENCODER_ERROR, None) if not return_message else (Result.ENCODER_ERROR, None, None) @@ -78,7 +80,7 @@ def DecodeEncode(self, encode_format: ne.ENCODE_FORMAT, message_input: bytes, me return Result.SUCCESS, message_data, message return Result.SUCCESS, message_data - + def TestDecodeEncode(self, format_, encoded_message): return self.DecodeEncode(format_, encoded_message, ne.MetaData())[0] @@ -107,29 +109,29 @@ def TestSameFormatCompare(self, format_: ne.ENCODE_FORMAT, expected_message_data return Result.MESSAGEDATA_COMPARISON_ERROR return Result.SUCCESS - + def TestConversion(self, format_: ENCODE_FORMAT, message_input: bytes, expected_message_data: ExpectedMessageData, expected_meta_data: ne.MetaData=None): test_meta_data = ne.MetaData() ret_code, test_message_data = self.DecodeEncode(format_, message_input, test_meta_data) if ret_code != Result.SUCCESS: return ret_code - + if expected_message_data.header != test_message_data.header: print(f"MessageData.header error (expected {len(expected_message_data.header)}, got {len(test_message_data.header)})") return Result.HEADER_LENGTH_ERROR - + if expected_message_data.message != test_message_data.message: print(f"MessageData.message error (expected {len(expected_message_data.message)}, got {len(test_message_data.message)})") return Result.LENGTH_ERROR - + if expected_meta_data is not None: if not compare_metadata(test_meta_data, expected_meta_data): return Result.METADATA_COMPARISON_ERROR - + if not compare_message_data(test_message_data, expected_message_data): return Result.MESSAGEDATA_COMPARISON_ERROR - + return Result.SUCCESS @@ -289,38 +291,39 @@ def test_ascii_log_roundtrip_gloalmanac(helper): def test_ascii_log_roundtrip_gloephem(helper): log = b"#GLOEPHEMERISA,COM1,11,67.0,SATTIME,2168,160218.000,02000820,8d29,32768;51,0,1,80,2168,161118000,10782,573,0,0,95,0,-2.3917966796875000e+07,4.8163881835937500e+06,7.4258510742187500e+06,-1.0062713623046875e+03,1.8321990966796875e+02,-3.3695755004882813e+03,1.86264514923095700e-06,-9.31322574615478510e-07,-0.00000000000000000,-6.69313594698905940e-05,5.587935448e-09,0.00000000000000000,84600,3,2,0,13*ad20fc5f\r\n" - ret_code, message_data, glo_ephemeris = helper.DecodeEncode(ENCODE_FORMAT.FLATTENED_BINARY, log, return_message=True) + ret_code, message_data, message = helper.DecodeEncode(ENCODE_FORMAT.FLATTENED_BINARY, log, return_message=True) assert ret_code == Result.SUCCESS - assert glo_ephemeris.sloto == 51 - assert glo_ephemeris.freqo == 0 - assert glo_ephemeris.sat_type == 1 - assert glo_ephemeris.false_iod == 80 - assert glo_ephemeris.ephem_week == 2168 - assert glo_ephemeris.ephem_time == 161118000 - assert glo_ephemeris.time_offset == 10782 - assert glo_ephemeris.nt == 573 - assert glo_ephemeris.GLOEPHEMERIS_reserved == 0 - assert glo_ephemeris.GLOEPHEMERIS_reserved_9 == 0 - assert glo_ephemeris.issue == 95 - assert glo_ephemeris.broadcast_health == 0 - assert glo_ephemeris.pos_x == -2.3917966796875000e+07 - assert glo_ephemeris.pos_y == 4.8163881835937500e+06 - assert glo_ephemeris.pos_z == 7.4258510742187500e+06 - assert glo_ephemeris.vel_x == -1.0062713623046875e+03 - assert glo_ephemeris.vel_y == 1.8321990966796875e+02 - assert glo_ephemeris.vel_z == -3.3695755004882813e+03 - assert glo_ephemeris.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) - assert glo_ephemeris.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) - assert glo_ephemeris.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) - assert glo_ephemeris.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) - assert glo_ephemeris.delta_tau == 5.587935448e-09 - assert glo_ephemeris.gamma == approx(0.00000000000000000, abs=0.0000000000000001) - assert glo_ephemeris.tk == 84600 - assert glo_ephemeris.p == 3 - assert glo_ephemeris.ft == 2 - assert glo_ephemeris.age == 0 - assert glo_ephemeris.flags == 13 + assert isinstance(message, GLOEPHEMERIS) + assert message.sloto == 51 + assert message.freqo == 0 + assert message.sat_type == 1 + assert message.false_iod == 80 + assert message.ephem_week == 2168 + assert message.ephem_time == 161118000 + assert message.time_offset == 10782 + assert message.nt == 573 + assert message.GLOEPHEMERIS_reserved == 0 + assert message.GLOEPHEMERIS_reserved_9 == 0 + assert message.issue == 95 + assert message.broadcast_health == 0 + assert message.pos_x == -2.3917966796875000e+07 + assert message.pos_y == 4.8163881835937500e+06 + assert message.pos_z == 7.4258510742187500e+06 + assert message.vel_x == -1.0062713623046875e+03 + assert message.vel_y == 1.8321990966796875e+02 + assert message.vel_z == -3.3695755004882813e+03 + assert message.ls_acc_x == approx(1.86264514923095700e-06, abs=0.0000000000000001e-06) + assert message.ls_acc_y == approx(-9.31322574615478510e-07, abs=0.0000000000000001e-07) + assert message.ls_acc_z == approx(-0.00000000000000000, abs=0.0000000000000001) + assert message.tau == approx(-6.69313594698905940e-05, abs=0.0000000000000001e-05) + assert message.delta_tau == 5.587935448e-09 + assert message.gamma == approx(0.00000000000000000, abs=0.0000000000000001) + assert message.tk == 84600 + assert message.p == 3 + assert message.ft == 2 + assert message.age == 0 + assert message.flags == 13 def test_ascii_log_roundtrip_loglist(helper): @@ -480,7 +483,7 @@ def test_short_binary_log_roundtrip_rawimu(helper): def test_flat_binary_log_decode_bestpos(helper): log = b" /dev/null + "$converter_components" "$script_dir/../database/database.json" "$script_dir/BESTUTMBIN.GPS" $FORMAT > /dev/null retval=$? if [ $retval -ne 0 ]; then echo "converter_components failed for $FORMAT failed with status $retval" diff --git a/src/decoders/common/test/main.cpp b/src/decoders/common/test/main.cpp index 5c437c9a5..c2590eac3 100644 --- a/src/decoders/common/test/main.cpp +++ b/src/decoders/common/test/main.cpp @@ -38,7 +38,7 @@ int main(int argc, char** argv) if (argc != 2) { throw std::invalid_argument("1 argument required.\nUsage: "); } - std::string strDatabaseVar = std::string(argv[1]) + "/database/messages_public.json"; + std::string strDatabaseVar = std::string(argv[1]) + "/database/database.json"; #ifdef _WIN32 if (_putenv_s("TEST_DATABASE_PATH", strDatabaseVar.c_str()) != 0) { throw std::runtime_error("Failed to set db path."); } diff --git a/src/decoders/oem/test/main.cpp b/src/decoders/oem/test/main.cpp index 517acbdca..d484b8661 100644 --- a/src/decoders/oem/test/main.cpp +++ b/src/decoders/oem/test/main.cpp @@ -38,7 +38,7 @@ int main(int argc, char** argv) if (argc != 2) { throw std::invalid_argument("1 argument required.\nUsage: "); } - std::string strDatabaseVar = std::string(argv[1]) + "/database/messages_public.json"; + std::string strDatabaseVar = std::string(argv[1]) + "/database/database.json"; std::string strResourceVar = std::string(argv[1]) + "/src/decoders/oem/test/resources/"; #ifdef _WIN32