#685: Update bazel (#439)

related to exasol/script-languages-release#685

.github/workflows/check_bazel_tests.yml

@@ -27,12 +27,12 @@ jobs:
         sudo apt-get install -y openjdk-11-jdk libzmq3-dev
     - name: Java Tests
       run: |
-        export USE_BAZEL_VERSION=6.4.0
-        bazel test //javacontainer/test/...
-      working-directory: ./exaudfclient/base
+        export USE_BAZEL_VERSION=7.2.1
+        bazel test //base/javacontainer/test/...
+      working-directory: ./exaudfclient/
     - name: ExaudfLib Tests
       run: |
-        export USE_BAZEL_VERSION=6.4.0
-        bazel test //exaudflib/test/...
-      working-directory: ./exaudfclient/base
+        export USE_BAZEL_VERSION=7.2.1
+        bazel test //base/exaudflib/test/...
+      working-directory: ./exaudfclient/
 

exaudfclient/base/.bazelrc → exaudfclient/.bazelrc

@@ -1,9 +1,9 @@
-build --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX 
+build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX
 # TODO add environment variables for R libraries
 build:benchmark --define benchmark=true
 build:java  --define java=true --action_env=JAVA_PREFIX
 build:python --define python=true  --action_env=PYTHON2_SYSPATH --action_env=PYTHON2_PREFIX --action_env=PYTHON2_VERSION --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION
-build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient_py3
+build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient_py3
 build:slow-wrapper-py3 --define binary_type=slow_wrapper //:exaudfclient_py3
 build:static-binary-py3 //:exaudfclient_py3_static
 build:test-binaries-py3 --config=static-binary-py3 --config=slow-wrapper-py3

exaudfclient/base/.gitignore → exaudfclient/.gitignore

@@ -3,5 +3,7 @@ bazel-genfiles
 bazel-out
 bazel-src
 bazel-testlogs
+bazel-exaudfclient
 graph.in
-graph.png
+graph.png
+MODULE.bazel.lock

exaudfclient/BUILD

@@ -0,0 +1,156 @@
+
+load("//:variables.bzl", "VM_ENABLED_DEFINES")
+
+config_setting(
+    name = "benchmark",
+    define_values = {"benchmark": "true"},
+)
+
+config_setting(
+    name = "python",
+    define_values = {"python": "true"},
+)
+
+config_setting(
+    name = "java",
+    define_values = {"java": "true"},
+)
+
+config_setting(
+    name = "bash",
+    define_values = {"bash": "true"},
+)
+
+config_setting(
+    name = "fast_binary",
+    define_values = {
+        "binary_type": "fast_binary",
+        },
+)
+
+config_setting(
+    name = "slow_wrapper",
+    define_values = {
+        "binary_type": "slow_wrapper",
+        },
+)
+
+config_setting(
+    name = "valgrind_wrapper",
+    define_values = {
+        "wrapper_type": "valgrind_wrapper",
+        },
+)
+
+config_setting(
+    name = "valgrind_massif_wrapper",
+    define_values = {
+        "wrapper_type": "valgrind_massif_wrapper",
+        },
+)
+
+config_setting(
+    name = "stdout_to_bucketfs",
+    define_values = {
+        "wrapper_type": "stdout_to_bucketfs",
+        },
+)
+
+
+VM_ENABLED_DEPS=select({
+        "//:benchmark": ["//base/benchmark_container:benchmark_container"],
+        "//conditions:default": []
+    }) + select({
+        "//:java": ["//base/javacontainer:javacontainer"],
+        "//conditions:default": []
+    }) + select({
+        "//:bash": ["//base/streaming_container:streamingcontainer"],
+        "//conditions:default": []
+    })
+
+VM_PYTHON3_DEPS=select({
+        "//:python": ["//base/python/python3:pythoncontainer"],
+        "//conditions:default": []
+    })
+
+cc_binary(
+    name = "exaudfclient_py3_bin",
+    srcs = ["exaudfclient.cc", "//base:load_dynamic"],
+    linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
+    deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+
+           ["//base/exaudflib:exaudflib-deps"],
+    defines = VM_ENABLED_DEFINES,
+    data = ["//base:libexaudflib_complete.so"]
+)
+
+## The purpose of the static binaries is to verify if the linker namespace test work correctly.
+## It is crucial that all linker symbols of exaudflib and all it's dependency are loaded into a new linker namespace,
+## so that the user can load it's own versions of those dependencies without any conflict.
+## With this binary we simulate an error in our build system, that is a direct depedency to protobuf/zmq,
+## which then must be detected with the linker namespace tests:
+## test/linker_namespace_sanity/linker_namespace_sanity.py checks the wrong configuration
+## Besides this the test under test/python3/all/linker_namespace.py checks the normal build, which expects
+## not to find any occurence of the dependencies (protobuf/zmq) in the primary linker namespace.
+##
+## We need to explicitly declare the dependency of protobuf/zmq here, as the exaudflib is a static lib (//base/exaudflib:exaudflib)
+## and hence does not contain dependency information. We cannot declare the shared lib (:exaudflib_complete.so)
+## as dependency as it is a binary for bazel.
+
+cc_binary(
+    name = "exaudfclient_py3_static_bin",
+    srcs = ["exaudfclient.cc", "//base:load_dynamic"],
+    linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
+    deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+
+            ["//base/exaudflib:exaudflib-deps"] + [ "@zmq//:zmq", "@protobuf//:protobuf"],
+    defines = VM_ENABLED_DEFINES,
+    data = ["//base:libexaudflib_complete.so"],
+)
+
+# Workarround for the hardcoded paths in exaudfclient for libexaudflib_complete.so and python_ext_dataframe.cc
+# - libexaudflib_complete.so and python_ext_dataframe.cc get dynamically loaded, therefore the exaudfclient needs to know their paths
+# - Most flexible way to provides these paths would environment variables
+# - The exasol database can't provide these paths, because they depend on the container
+# - A workarround to provide these paths would be wrapper bash script which set these environment variables
+# - For performance reason, we can not wrap the binary into a shell script, as such this is only for local debugging and testing
+#   and we hardcode the paths for the production container
+
+sh_library(
+    name = "wrapper_generator_bin",
+    srcs=select({
+        "//:valgrind_wrapper": ["//base:create_binary_wrapper_valgrind.sh"],
+        "//:valgrind_massif_wrapper": ["//base:create_binary_wrapper_valgrind_massif.sh"],
+        "//:stdout_to_bucketfs": ["//base:create_binary_wrapper_stdout_to_bucketfs.sh"],
+        "//conditions:default": ["//base:create_binary_wrapper.sh"]
+    })
+)
+
+SLOW_WRAPPER_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" "$(location //base:exaudfclient.template.sh)" """
+FAST_BINARY_PY3="""cp "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" """
+CREATE_BINARY_PY3_SCRIPT=select({
+        "//:fast_binary": FAST_BINARY_PY3,
+        "//:slow_wrapper": SLOW_WRAPPER_BINARY_PY3,
+        "//conditions:default": FAST_BINARY_PY3
+    })
+genrule(
+    name = "exaudfclient_py3",
+    cmd = CREATE_BINARY_PY3_SCRIPT,
+    outs = ["exaudfclient_py3"],
+    srcs = [":exaudfclient_py3_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
+    output_to_bindir = True
+)
+
+SLOW_WRAPPER_STATIC_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" "$(location //base:exaudfclient.template.sh)" """
+FAST_BINARY_STATIC_PY3="""cp "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" """
+CREATE_STATIC_BINARY_PY3_SCRIPT=select({
+        "//:fast_binary": FAST_BINARY_STATIC_PY3,
+        "//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY_PY3,
+        "//conditions:default": FAST_BINARY_STATIC_PY3
+    })
+
+genrule(
+    name = "exaudfclient_py3_static",
+    cmd = CREATE_STATIC_BINARY_PY3_SCRIPT,
+    outs = ["exaudfclient_py3_static"],
+    srcs = [":exaudfclient_py3_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
+    output_to_bindir = True
+)

exaudfclient/MODULE.bazel

@@ -0,0 +1,3 @@
+module(name="exaudfclient", version = "1.0")
+
+include("//base:base.MODULE.bazel")

exaudfclient/base/README.md → exaudfclient/README.md

@@ -70,7 +70,7 @@ The usage of multiple linker namespace requires some precautions in the build pr
 
 ## Precautions in the build process
 
-In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have to main output targets //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language container live in the same namespace as //:exaudfclient. This namespace must not know anyhing about protobuf and zeromq, because it is possible that a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //exaudflib:exaudflib in //:exaudfclient or the langauge container, because this would leak zeromq and protobuf. If you need to depend on the other dependency of //exaudflib:exaudflib which not depend on protobuf or zeromq them self, such as //exaudflib:script_data_transfer_objects, //exaudflib:script_data_transfer_objects_wrapper, //exaudflib:scriptoptionlines, use either their target as self, the collection of libraries //exaudflib:exaudflib-deps or the collection of headers //exaudflib:header.
+In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have to main output targets //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language container live in the same namespace as //:exaudfclient. This namespace must not know anyhing about protobuf and zeromq, because it is possible that a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //base/exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //base/exaudflib:exaudflib in //:exaudfclient or the langauge container, because this would leak zeromq and protobuf. If you need to depend on the other dependency of //base/exaudflib:exaudflib which not depend on protobuf or zeromq them self, such as //base/exaudflib:script_data_transfer_objects, //base/exaudflib:script_data_transfer_objects_wrapper, //base/exaudflib:scriptoptionlines, use either their target as self, the collection of libraries //base/exaudflib:exaudflib-deps or the collection of headers //base/exaudflib:header.
 
 ## Precautions in the implementations