Add support for multiple service interfaces in a gRPC service (#75)

* Add support for multiple service interfaces in a gRPC service * Minor fixes * Rework findings * Update README * Test integration tests * Remove .velocitas.json * Fix protoIncludeDir * Add correct AppManifest * Fix download location * Change protoIncludeDir in tests * Fix protoIncludeDir in tests 2 * Add multiple proto files correctly * Revert common AppManifest * Fix import & remove test dependencies for python * Use correct names of service * Test * Test2 * Generate imports correctly * Give the right path to imports * Exclude sdv.databroker.v1 build because its in vehicla app SDK * Fix import in door proto * Chnage assert check to check for the right ones * Fix assert * Fix comments * Rework findings * initialize test repo as git * fix renaming * Add examples in README * Table in README & fixes for archive * Add DOWNLOAD_PATH & fix ARCHIVE_PATH * Delete ARCHIVE_PATH since structure of zip is the root cause * Rework findings * Fix if else * Update README * Use extend instead of append * Extend unit tests * Change import to right formatting in unit test * Use isabs in proto_include path * Fix proto include path * Rework findings 2 * Adopt changes on python and cpp side * Try to search and place in right directory * Fix gRPC code extractor * extract functions to lib & rework findings * Fix output cpp * First try extending integration tests * Adapt python tests to strucutur of nested zip * Fix directory names * Remove bcm door which is not generated * fix multiple services directories location * Fix Hornservice names * Make proto dir independent of service gen * Adapt Integration application * Minor fix * Minor fix * Adapt includes to camel case * Fix server Launcher * Use reference * Remove reference client code * Add missing colon * Use join instead of detach * Use own grpc context for every client * Change back to versioning and put proto files in right directory * update README * Move the service definition respective to the usage * Formatting * Remove commented lines --------- Co-authored-by: Markus Petke <[email protected]>
eclipse-velocitas · Aug 5, 2024 · 9913179 · 9913179
1 parent 0199f54
commit 9913179
Show file tree

Hide file tree

Showing 27 changed files with 1,333 additions and 144 deletions.
diff --git a/grpc-interface-support/README.md b/grpc-interface-support/README.md
@@ -15,41 +15,16 @@ velocitas exec grpc-interface-support generate-sdk
 
 Depending on the interfaces defined in the `AppManifest` file, either classes and SDKs for client, server or both are generated.
 
-For generating a client SDK:
-```json
-{
-    "manifestVersion": "v3",
-    "name": "App",
-    "interfaces": [
-        {
-            "type": "grpc-interface",
-            "config": {
-                "src": "<uri_to_proto_file>",
-                "required": {}
-            }
-        }
-    ]
-}
-```
 
-For generating a server SDK:
-```json
-{
-    "manifestVersion": "v3",
-    "name": "App",
-    "interfaces": [
-        {
-            "type": "grpc-interface",
-            "config": {
-                "src": "<uri_to_proto_file>",
-                "provided": {}
-            }
-        }
-    ]
-}
-```
+| parameters                          | meaning                                                          | client SDK                       | server SDK                       | local proto file (absolute path) | local proto file (relative path) | archive                          | downloadable file (raw not blob) |
+| -------------------------------- | ---------------------------------------------------------------- | -------------------------------- | -------------------------------- | -------------------------------- | -------------------------------- | -------------------------------- | -------------------------------- |
+| src                              | The source (absolute or relative) to the proto files (dir, zip, file, link) | /home/user/proto_file.proto       | /home/user/proto_file.proto       | /home/user/proto_file.proto       | proto_file.proto (located under workspace of template repository) | https://github.com/project/release.zip | https://github.com/project/raw/proto_file.proto |
+| required                         | The functions that the client uses                             | { client_function1(), client_function2(), ... }                 | not defined                      | see sdk examples                 | see sdk examples                 | see sdk examples                 | see sdk examples                 |
+| provided                         | Set if the server code shall be generated to {}                 | not defined                      | {}                               | see sdk examples                 | see sdk examples                 | see sdk examples                 | see sdk examples                 |
+| protoIncludeDir                  | The path to some imports in the protot files (default parent folder) | path_to_imports                  | path_to_imports                  | path_to_imports                  | path_to_imports                  | path_to_imports                  | path_to_imports                  |
+| pathInZip                        | If you have multiple folders in a zip and just want one to be generated | undefined                        | undefined                        | undefined                        | undefined                        | rel_path_archive                | undefined                        |
 
-For generating both:
+Example json:
 ```json
 {
     "manifestVersion": "v3",
@@ -58,9 +33,10 @@ For generating both:
         {
             "type": "grpc-interface",
             "config": {
-                "src": "<uri_to_proto_file>",
+                "src": "home/user/proto_file.proto",
+                "protoIncludeDir": "<path_to_imports>",
                 "required": {},
-                "provided": {}
+                "provided": {},
             }
         }
     ]

diff --git a/grpc-interface-support/data/templates/cpp/CMakeLists.txt b/grpc-interface-support/data/templates/cpp/CMakeLists.txt
@@ -15,16 +15,23 @@ target_include_directories(${PROJECT_NAME}
 include_directories(
     ${CMAKE_INCLUDE_PATH}
     include/services/${{ service_name_lower }}
+    include/${{ proto_location }}
 )
 
 set(HEADERS
     ${{ cmake_headers }}
 )
 
+set(PROTO_HEADERS
+    ${{ proto_headers }}
+)
+
 set_target_properties(${PROJECT_NAME} PROPERTIES
     PUBLIC_HEADER "${HEADERS}"
 )
 
 install(TARGETS ${PROJECT_NAME}
-    PUBLIC_HEADER DESTINATION include/services/${{ service_name_lower }}
+    PUBLIC_HEADER DESTINATION include/services/${{ service_name_lower }}/${{ proto_location }}
 )
+
+install(FILES ${PROTO_HEADERS} DESTINATION include/${{ proto_location }})
diff --git a/grpc-interface-support/data/templates/cpp/ServiceImpl.cpp b/grpc-interface-support/data/templates/cpp/ServiceImpl.cpp
@@ -14,7 +14,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-#include "${{ service_name }}ServiceImpl.h"
+#include "${{ service_name_camel_case }}ServiceImpl.h"
 
 namespace velocitas {
 

diff --git a/grpc-interface-support/requirements.txt b/grpc-interface-support/requirements.txt
@@ -1 +1,2 @@
-velocitas-lib==0.0.10
+velocitas-lib==0.0.11
+proto-schema-parser==1.3.4
diff --git a/grpc-interface-support/src/cpp.py b/grpc-interface-support/src/cpp.py
@@ -32,13 +32,12 @@
     export_conan_project,
     get_required_sdk_version,
 )
+from velocitas_lib.templates import CopySpec, copy_templates
 from velocitas_lib.text_utils import (
     capture_area_in_file,
     to_camel_case,
 )
 
-from velocitas_lib.templates import CopySpec, copy_templates
-
 CONAN_PROFILE_NAME = "host"
 
 
@@ -104,24 +103,34 @@ def __init__(
         self.__proto_file_handle = proto_file_handle
         self.__verbose = verbose
         self.__proto_include_path = proto_include_path
+        self.__proto_include_rel_path = os.path.relpath(
+            str(Path(self.__proto_file_handle.file_path).parent),
+            self.__proto_include_path,
+        )
+        self.__service_name = self.__proto_file_handle.get_service_name()
+        self.__service_name_lower = self.__service_name.lower()
+        self.__output_path = os.path.join(
+            self.__package_directory_path,
+            os.path.relpath(
+                str(Path(self.__proto_file_handle.file_path).parent),
+                self.__proto_include_path,
+            ),
+        )
 
     def __get_binary_path(self, binary_name: str) -> str:
         path = shutil.which(binary_name)
         if path is None:
             raise KeyError(f"{binary_name!r} missing!")
         return path
 
-    def __invoke_code_generator(self, include_path: str, output_path: str) -> None:
+    def __invoke_code_generator(self) -> None:
         print("Invoking gRPC code generator")
-        self.__output_path = os.path.join(
-            output_path, os.path.relpath(include_path, self.__proto_include_path)
-        )
         args = [
             self.__get_binary_path("protoc"),
             f"--plugin=protoc-gen-grpc={self.__get_binary_path('grpc_cpp_plugin')}",
             f"-I{self.__proto_include_path}",
-            f"--cpp_out={output_path}",
-            f"--grpc_out={output_path}",
+            f"--cpp_out={self.__package_directory_path}",
+            f"--grpc_out={self.__package_directory_path}",
             self.__proto_file_handle.file_path,
         ]
         subprocess.check_call(
@@ -131,21 +140,33 @@ def __invoke_code_generator(self, include_path: str, output_path: str) -> None:
             stdout=subprocess.DEVNULL if not self.__verbose else None,
         )
 
-    def generate_package(self, client_required: bool, server_required: bool) -> None:
-        self.__invoke_code_generator(
-            str(Path(self.__proto_file_handle.file_path).parent),
-            self.__package_directory_path,
-        )
+        imports = self.__proto_file_handle.get_imports()
+
+        for element in imports:
+            path = os.path.join(self.__proto_include_path, element)
+            args = [
+                self.__get_binary_path("protoc"),
+                f"-I{self.__proto_include_path}",
+                f"--cpp_out={self.__package_directory_path}",
+                path,
+            ]
+            subprocess.check_call(
+                args,
+                cwd=self.__proto_include_path,
+                env=os.environ,
+                stdout=subprocess.DEVNULL if not self.__verbose else None,
+            )
 
-        service_name = self.__proto_file_handle.get_service_name()
+    def generate_package(self, client_required: bool, server_required: bool) -> None:
+        self.__invoke_code_generator()
 
         files_to_copy: List[CopySpec] = []
 
         if client_required:
-            files_to_copy.extend(self.__get_service_client_files(service_name))
+            files_to_copy.extend(self.__get_service_client_files(self.__service_name))
 
         if server_required:
-            files_to_copy.extend(self.__get_service_server_files(service_name))
+            files_to_copy.extend(self.__get_service_server_files(self.__service_name))
 
         copy_templates(
             get_template_dir(),
@@ -155,24 +176,22 @@ def generate_package(self, client_required: bool, server_required: bool) -> None
         )
 
     def __get_template_variables(self) -> Dict[str, str]:
-        service_name = self.__proto_file_handle.get_service_name()
         return {
-            "service_name": service_name,
-            "service_name_lower": service_name.lower(),
-            "service_name_camel_case": to_camel_case(service_name),
+            "service_name": self.__service_name,
+            "service_name_lower": self.__service_name_lower,
+            "service_name_camel_case": to_camel_case(self.__service_name),
             "package_id": self.__proto_file_handle.get_package().replace(".", "::"),
             "core_sdk_version": str(get_required_sdk_version()),
             "service_include_dir": self.__get_relative_file_dir(),
+            "proto_location": self.__proto_include_rel_path,
             "grpc_service_header_path": os.path.join(
                 self.__get_relative_file_dir(),
                 f"{Path(self.__proto_file_handle.file_path).stem}.grpc.pb.h",
             ),
         }
 
     def __get_relative_file_dir(self) -> str:
-        rel_path = os.path.relpath(
-            Path(self.__proto_file_handle.file_path).parent, self.__proto_include_path
-        )
+        rel_path = self.__proto_include_rel_path
         if rel_path == ".":
             return f"services/{self.__proto_file_handle.get_service_name().lower()}"
 
@@ -250,7 +269,7 @@ def __move_generated_sources(
         return headers_relative, sources_relative
 
     def install_package(self) -> None:
-        self.__move_generated_sources(
+        proto_headers, proto_sources = self.__move_generated_sources(
             self.__package_directory_path,
             self.__get_include_dir(),
             self.__get_source_dir(),
@@ -279,6 +298,8 @@ def install_package(self) -> None:
 
         variables["cmake_headers"] = "\n\t".join(cmake_headers)
         variables["cmake_sources"] = "\n\t".join(cmake_sources)
+        variables["proto_headers"] = "\n\t".join(proto_headers)
+        variables["proto_sources"] = "\n\t".join(proto_sources)
 
         copy_templates(
             get_template_dir(),
@@ -290,8 +311,7 @@ def install_package(self) -> None:
         export_conan_project(self.__package_directory_path)
 
     def __transform_header_stub_code(self, lines: List[str]) -> List[str]:
-        service_name = self.__proto_file_handle.get_service_name()
-        service_class_name = to_camel_case(service_name) + "Service"
+        service_class_name = to_camel_case(self.__service_name) + "Service"
 
         result: List[str] = []
 
@@ -303,13 +323,14 @@ def __transform_header_stub_code(self, lines: List[str]) -> List[str]:
         return result
 
     def __transform_source_stub_code(self, lines: List[str]) -> List[str]:
-        service_name = self.__proto_file_handle.get_service_name()
-        service_class_name = to_camel_case(service_name) + "Service"
+        service_class_name = to_camel_case(self.__service_name) + "Service"
 
         result: List[str] = []
 
         for line in lines:
-            result.append(line.replace(f"{service_name}::Service", service_class_name))
+            result.append(
+                line.replace(f"{self.__service_name}::Service", service_class_name)
+            )
 
         return result
 
@@ -321,9 +342,7 @@ def __create_or_update_service_header(self) -> None:
         header_stub_code = self.__transform_header_stub_code(header_stub_code)
 
         app_source_dir = os.path.join(get_workspace_dir(), "app", "src")
-        service_header_file_name = (
-            f"{self.__proto_file_handle.get_service_name()}ServiceImpl.h"
-        )
+        service_header_file_name = f"{to_camel_case(self.__service_name)}ServiceImpl.h"
         service_header_file_path = os.path.join(
             app_source_dir, service_header_file_name
         )
@@ -353,7 +372,7 @@ def __create_or_update_service_header(self) -> None:
     def __create_service_source(self) -> None:
         app_source_dir = os.path.join(get_workspace_dir(), "app", "src")
         service_source_file_name = (
-            f"{self.__proto_file_handle.get_service_name()}ServiceImpl.cpp"
+            f"{to_camel_case(self.__service_name)}ServiceImpl.cpp"
         )
         service_source_file_path = os.path.join(
             app_source_dir, service_source_file_name
@@ -382,7 +401,7 @@ def update_package_references(self) -> None:
         """Update all references to the generated package."""
 
         add_dependency_to_conanfile(
-            f"{self.__proto_file_handle.get_service_name().lower()}-service-sdk",
+            f"{self.__service_name_lower}-service-sdk",
             "generated",
         )
 

diff --git a/grpc-interface-support/src/generator.py b/grpc-interface-support/src/generator.py
@@ -67,6 +67,7 @@ def create_service_generator(
             output_path (str): Path where the SDK shall be generated at.
             proto_file_handle (proto.ProtoFileHandle): The proto file which serves
                 as the input for the generator.
+            proto_include_path (str): The path which is used to look for imports.
 
         Returns:
             GrpcServiceSdkGenerator: A new GrpcServiceSdkGenerator which can