Python Bindings for Publisher, Subscriber and Service Request feature…

…s. (#411) This PR creates the python bindings for the publisher, subscriber and request features from the repository. In order to avoid creating a new dependency to be able to create python bindings from protobuf messages, it was decided to use the methods that publishes, subscribes and make services request with serialized messages, i.e, use the methods PublishRaw, SubscribeRaw and RequestRaw. The bindings for these methods are wrapped by python methods in order to give the user an API similar to the one we have in C++. We would like to acknowledge and give credit to @srmainwaring for his code which served as a reference and inspiration for this implementation. The main difference with this implementation is that we are using the methods that uses serialized messages instead of creating bindings with the pybind11_protobuf library to the non-serialized methods. --------- Signed-off-by: Voldivh <[email protected]> Signed-off-by: Addisu Z. Taddese <[email protected]> Co-authored-by: Addisu Z. Taddese <[email protected]> Co-authored-by: Alejandro Hernández Cordero <[email protected]> Co-authored-by: Addisu Z. Taddese <[email protected]>
gazebosim · Jul 24, 2023 · da3c55d · da3c55d
1 parent 917bf04
commit da3c55d
Show file tree

Hide file tree

Showing 15 changed files with 1,289 additions and 0 deletions.
diff --git a/.github/ci/after_make.sh b/.github/ci/after_make.sh
@@ -0,0 +1,2 @@
+# It's necessary to install the python modules for the test.
+make install
diff --git a/.github/ci/packages.apt b/.github/ci/packages.apt
@@ -9,4 +9,10 @@ libsqlite3-dev
 libzmq3-dev
 pkg-config
 protobuf-compiler
+python3-dev
+python3-distutils
+python3-psutil
+python3-pybind11
+python3-pytest
+python3-gz-msgs10
 uuid-dev
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,6 @@ build_*
 # OS generated files
 .DS_Store
 *.swp
+
+# Python generated files
+*.pyc
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -15,6 +15,22 @@ set(GZ_CMAKE_VER ${gz-cmake3_VERSION_MAJOR})
 #============================================================================
 # Configure the project
 #============================================================================
+set(skip_pybind11_default_value OFF)
+option(SKIP_PYBIND11
+      "Skip generating Python bindings via pybind11"
+      ${skip_pybind11_default_value})
+
+# Python interfaces vars
+include(CMakeDependentOption)
+include(GzPython)
+cmake_dependent_option(USE_SYSTEM_PATHS_FOR_PYTHON_INSTALLATION
+      "Install python modules in standard system paths in the system"
+      OFF "NOT SKIP_PYBIND11" OFF)
+
+cmake_dependent_option(USE_DIST_PACKAGES_FOR_PYTHON
+      "Use dist-packages instead of site-package to install python modules"
+      OFF "NOT SKIP_PYBIND11" OFF)
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
@@ -72,6 +88,7 @@ gz_find_package(CPPZMQ REQUIRED PRIVATE
 #--------------------------------------
 # Find uuid
 if (MSVC)
+  set(skip_pybind11_default_value ON)
   message (STATUS "UUID: Using Windows RPC UuidCreate function\n")
 else()
   gz_find_package(UUID REQUIRED)
@@ -115,6 +132,31 @@ gz_find_package(SQLite3
 #============================================================================
 # Configure the build
 #============================================================================
+
+########################################
+# Python interfaces
+if (NOT PYTHON3_FOUND)
+  GZ_BUILD_WARNING("Python is missing: Python interfaces are disabled.")
+  message (STATUS "Searching for Python - not found.")
+else()
+  message (STATUS "Searching for Python - found version ${Python3_VERSION}.")
+
+  if (SKIP_PYBIND11)
+    message(STATUS "SKIP_PYBIND11 set - disabling python bindings")
+  else()
+    set(PYBIND11_PYTHON_VERSION 3)
+    find_package(pybind11 2.4 QUIET)
+
+    if (${pybind11_FOUND})
+      find_package(Python3 ${GZ_PYTHON_VERSION} REQUIRED COMPONENTS Development)
+      message (STATUS "Searching for pybind11 - found version ${pybind11_VERSION}.")
+    else()
+      GZ_BUILD_WARNING("pybind11 is missing: Python interfaces are disabled.")
+      message (STATUS "Searching for pybind11 - not found.")
+    endif()
+  endif()
+endif()
+
 gz_configure_build(QUIT_IF_BUILD_ERRORS
   COMPONENTS log parameters)
 
@@ -123,6 +165,13 @@ gz_configure_build(QUIT_IF_BUILD_ERRORS
 #============================================================================
 add_subdirectory(conf)
 
+#============================================================================
+# gz transport python bindings
+#============================================================================
+if (pybind11_FOUND AND NOT SKIP_PYBIND11)
+  add_subdirectory(python)
+endif()
+
 #============================================================================
 # Create package information
 #============================================================================

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
@@ -0,0 +1,96 @@
+if(WIN32 AND CMAKE_BUILD_TYPE STREQUAL "Debug")
+  # pybind11 logic for setting up a debug build when both a debug and release
+  # python interpreter are present in the system seems to be pretty much broken.
+  # This works around the issue.
+  set(PYTHON_LIBRARIES "${PYTHON_DEBUG_LIBRARIES}")
+endif()
+
+if(USE_SYSTEM_PATHS_FOR_PYTHON_INSTALLATION)
+  if(${CMAKE_VERSION} VERSION_LESS "3.12.0")
+    execute_process(
+      COMMAND "${PYTHON_EXECUTABLE}" -c "if True:
+  from distutils import sysconfig as sc
+  print(sc.get_python_lib(plat_specific=True))"
+      OUTPUT_VARIABLE Python3_SITEARCH
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+  else()
+    # Get install variable from Python3 module
+    # Python3_SITEARCH is available from 3.12 on, workaround if needed:
+    find_package(Python3 COMPONENTS Interpreter)
+  endif()
+
+  if(USE_DIST_PACKAGES_FOR_PYTHON)
+    string(REPLACE "site-packages" "dist-packages" GZ_PYTHON_INSTALL_PATH ${Python3_SITEARCH})
+  else()
+    # custom cmake command is returning dist-packages
+    string(REPLACE "dist-packages" "site-packages" GZ_PYTHON_INSTALL_PATH ${Python3_SITEARCH})
+  endif()
+else()
+  # If not a system installation, respect local paths
+  set(GZ_PYTHON_INSTALL_PATH ${GZ_LIB_INSTALL_DIR}/python/gz)
+endif()
+
+# Set the build location and install location for a CPython extension
+function(configure_build_install_location _library_name)
+  # Install library for actual use
+  install(TARGETS ${_library_name}
+    DESTINATION "${GZ_PYTHON_INSTALL_PATH}/transport${PROJECT_VERSION_MAJOR}"
+  )
+endfunction()
+
+message(STATUS "Building pybind11 interfaces")
+# We are creating the bindings name as the following because we 
+# created a python wrapper that does some modifications in order
+# to be able to provide an API similar to the one used in C++.
+set(BINDINGS_MODULE_NAME "_transport")
+# Split from main extension and converted to pybind11
+pybind11_add_module(${BINDINGS_MODULE_NAME} MODULE
+  src/transport/_gz_transport_pybind11.cc
+)
+
+target_link_libraries(${BINDINGS_MODULE_NAME} PRIVATE
+  ${PROJECT_LIBRARY_TARGET_NAME}
+)
+
+target_compile_definitions(${BINDINGS_MODULE_NAME} PRIVATE
+  BINDINGS_MODULE_NAME=${BINDINGS_MODULE_NAME})
+
+configure_build_install_location(${BINDINGS_MODULE_NAME})
+
+install(FILES
+  src/__init__.py  
+  DESTINATION "${GZ_PYTHON_INSTALL_PATH}/transport${PROJECT_VERSION_MAJOR}"
+)
+
+if (BUILD_TESTING AND NOT WIN32)
+  set(python_tests
+    pubSub_TEST
+    requester_TEST
+    options_TEST
+  )
+  execute_process(COMMAND "${Python3_EXECUTABLE}" -m pytest --version
+    OUTPUT_VARIABLE PYTEST_output
+    ERROR_VARIABLE  PYTEST_error
+    RESULT_VARIABLE PYTEST_result)
+  if(${PYTEST_result} EQUAL 0)
+    set(pytest_FOUND TRUE)
+  else()
+    message(WARNING "Pytest package not available: ${PYTEST_error}")
+  endif()
+
+  foreach (test ${python_tests})
+    if (pytest_FOUND)
+      add_test(NAME ${test}.py COMMAND
+        "${Python3_EXECUTABLE}" -m pytest "${CMAKE_SOURCE_DIR}/python/test/${test}.py" --junitxml "${CMAKE_BINARY_DIR}/test_results/${test}.xml")
+    else()
+      add_test(NAME ${test}.py COMMAND
+        "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/python/test/${test}.py")
+    endif()
+    set(_env_vars)
+    list(APPEND _env_vars "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}/bin")
+    list(APPEND _env_vars "PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/python/:${CMAKE_BINARY_DIR}/lib:$ENV{PYTHONPATH}")
+    list(APPEND _env_vars "LD_LIBRARY_PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}:$ENV{LD_LIBRARY_PATH}")
+    set_tests_properties(${test}.py PROPERTIES
+      ENVIRONMENT "${_env_vars}")
+  endforeach()
+endif()
diff --git a/python/examples/data_race_with_mutex.py b/python/examples/data_race_with_mutex.py
@@ -0,0 +1,74 @@
+# Copyright (C) 2023 Open Source Robotics Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from gz.msgs10.vector3d_pb2 import Vector3d
+from gz.transport13 import Node
+
+from threading import Lock
+import time
+
+x = 1
+y = 2
+z = 3
+
+mutex = Lock()
+
+def vector3_cb(msg: Vector3d):
+    global x, y, z, mutex
+    with mutex:
+        x = msg.x
+        y = msg.y
+        z = msg.z
+
+def main():
+    global x, y, z, mutex
+    # create a transport node
+    node = Node()
+    topic_vector3d = "/vector3d_topic"
+
+    pub_vector3d = node.advertise(topic_vector3d, Vector3d)
+
+    vector3d_msg = Vector3d()
+    vector3d_msg.x = 1
+    vector3d_msg.y = 2
+    vector3d_msg.z = 3
+
+    # subscribe to a topic by registering a callback
+    if node.subscribe(Vector3d, topic_vector3d, vector3_cb):
+        print("Subscribing to type {} on topic [{}]".format(
+            Vector3d, topic_vector3d))
+    else:
+        print("Error subscribing to topic [{}]".format(topic_vector3d))
+        return
+
+    # wait for shutdown
+    try:
+        count = 1
+        while True:
+            with mutex:
+                count += 1
+                vector3d_msg.x = vector3d_msg.x*count
+                vector3d_msg.y = vector3d_msg.y*count
+                vector3d_msg.z = vector3d_msg.z*count
+                pub_vector3d.publish(vector3d_msg)
+                if ((vector3d_msg.x - x) != 0) or ((vector3d_msg.y - y) != 0) or ((vector3d_msg.z - z) != 0):
+                    print("Race Condition happened")
+            time.sleep(0.01)
+    except KeyboardInterrupt:
+      pass
+    print("Done")
+
+if __name__ == "__main__":
+    main()
diff --git a/python/examples/data_race_without_mutex.py b/python/examples/data_race_without_mutex.py
@@ -0,0 +1,69 @@
+# Copyright (C) 2023 Open Source Robotics Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from gz.msgs10.vector3d_pb2 import Vector3d
+from gz.transport13 import Node
+
+import time
+
+x = 1
+y = 2
+z = 3
+
+def vector3_cb(msg: Vector3d):
+    global x, y, z
+    x = msg.x
+    y = msg.y
+    z = msg.z
+
+def main():
+    global x, y, z
+    # create a transport node
+    node = Node()
+    topic_vector3d = "/vector3d_topic"
+
+    pub_vector3d = node.advertise(topic_vector3d, Vector3d)
+
+    vector3d_msg = Vector3d()
+    vector3d_msg.x = 1
+    vector3d_msg.y = 2
+    vector3d_msg.z = 3
+
+    # subscribe to a topic by registering a callback
+    if node.subscribe(Vector3d, topic_vector3d, vector3_cb):
+        print("Subscribing to type {} on topic [{}]".format(
+            Vector3d, topic_vector3d))
+    else:
+        print("Error subscribing to topic [{}]".format(topic_vector3d))
+        return
+
+    # wait for shutdown
+    try:
+        count = 1
+        while True:
+            count += 1
+            vector3d_msg.x = vector3d_msg.x*count
+            vector3d_msg.y = vector3d_msg.y*count
+            vector3d_msg.z = vector3d_msg.z*count
+            pub_vector3d.publish(vector3d_msg)
+            if ((vector3d_msg.x - x) != 0) or ((vector3d_msg.y - y) != 0) or ((vector3d_msg.z - z) != 0):
+                print("Race Condition happened")
+            time.sleep(0.01)
+    except KeyboardInterrupt:
+      pass
+    print("Done")
+
+if __name__ == "__main__":
+    main()
diff --git a/python/examples/publisher.py b/python/examples/publisher.py
@@ -0,0 +1,54 @@
+# Copyright (C) 2023 Open Source Robotics Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from gz.msgs10.stringmsg_pb2 import StringMsg
+from gz.msgs10.vector3d_pb2 import Vector3d
+from gz.transport13 import AdvertiseMessageOptions
+from gz.transport13 import Node
+
+import time
+
+def main():
+    node = Node()
+    stringmsg_topic = "/example_stringmsg_topic"
+    vector3d_topic = "/example_vector3d_topic"
+    pub_stringmsg = node.advertise(stringmsg_topic, StringMsg)
+    pub_vector3d = node.advertise(vector3d_topic, Vector3d)
+
+    vector3d_msg = Vector3d()
+    vector3d_msg.x = 10
+    vector3d_msg.y = 15
+    vector3d_msg.z = 20
+
+    stringmsg_msg = StringMsg()
+    stringmsg_msg.data = "Hello" 
+    try:
+        count = 0
+        while True:
+          count += 1
+          vector3d_msg.x = count
+          if not (pub_stringmsg.publish(stringmsg_msg) or pub_vector3d.publish(vector3d_msg)):
+              break
+
+          print("Publishing 'Hello' on topic [{}]".format(stringmsg_topic))
+          print("Publishing a Vector3d on topic [{}]".format(vector3d_topic))
+          time.sleep(0.1)
+
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == "__main__":
+    main()