diff --git a/exaudfclient/.bazelrc b/exaudfclient/.bazelrc index 032c4ce2..cdcb2d96 100644 --- a/exaudfclient/.bazelrc +++ b/exaudfclient/.bazelrc @@ -3,10 +3,10 @@ build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_ build:benchmark --define benchmark=true build:java --define java=true --action_env=JAVA_PREFIX build:python --define python=true --action_env=PYTHON2_SYSPATH --action_env=PYTHON2_PREFIX --action_env=PYTHON2_VERSION --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION -build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient_py3 -build:slow-wrapper-py3 --define binary_type=slow_wrapper //:exaudfclient_py3 -build:static-binary-py3 //:exaudfclient_py3_static -build:test-binaries-py3 --config=static-binary-py3 --config=slow-wrapper-py3 +build:fast-binary --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient +build:slow-wrapper --define binary_type=slow_wrapper //:exaudfclient +build:static-binary //:exaudfclient_static +build:test-binaries --config=static-binary --config=slow-wrapper build:verbose --copt='-v' --subcommands --verbose_failures --announce_rc #TODO test linkopts="-flto" build:optimize --copt="-g0" --copt="-DNDEBUG" --copt=-fstack-protector-strong --copt=-fomit-frame-pointer --copt=-ffunction-sections --copt=-fdata-sections --copt="-O3" --copt="-U_FORTIFY_SOURCE" --copt="-flto" --copt="-fuse-linker-plugin" diff --git a/exaudfclient/BUILD b/exaudfclient/BUILD index f72c416f..a0c3665c 100644 --- a/exaudfclient/BUILD +++ b/exaudfclient/BUILD @@ -74,7 +74,7 @@ VM_PYTHON3_DEPS=select({ }) cc_binary( - name = "exaudfclient_py3_bin", + name = "exaudfclient_bin", srcs = ["exaudfclient.cc", "//base:load_dynamic"], linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ @@ -97,7 +97,7 @@ cc_binary( ## as dependency as it is a binary for bazel. cc_binary( - name = "exaudfclient_py3_static_bin", + name = "exaudfclient_static_bin", srcs = ["exaudfclient.cc", "//base:load_dynamic"], linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ @@ -125,33 +125,33 @@ sh_library( }) ) -SLOW_WRAPPER_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" "$(location //base:exaudfclient.template.sh)" """ -FAST_BINARY_PY3="""cp "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" """ -CREATE_BINARY_PY3_SCRIPT=select({ - "//:fast_binary": FAST_BINARY_PY3, - "//:slow_wrapper": SLOW_WRAPPER_BINARY_PY3, - "//conditions:default": FAST_BINARY_PY3 +SLOW_WRAPPER_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_bin)" "$(location exaudfclient)" "$(location //base:exaudfclient.template.sh)" """ +FAST_BINARY="""cp "$(location exaudfclient_bin)" "$(location exaudfclient)" """ +CREATE_BINARY_SCRIPT=select({ + "//:fast_binary": FAST_BINARY, + "//:slow_wrapper": SLOW_WRAPPER_BINARY, + "//conditions:default": FAST_BINARY }) genrule( - name = "exaudfclient_py3", - cmd = CREATE_BINARY_PY3_SCRIPT, - outs = ["exaudfclient_py3"], - srcs = [":exaudfclient_py3_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"], + name = "exaudfclient", + cmd = CREATE_BINARY_SCRIPT, + outs = ["exaudfclient"], + srcs = [":exaudfclient_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"], output_to_bindir = True ) -SLOW_WRAPPER_STATIC_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" "$(location //base:exaudfclient.template.sh)" """ -FAST_BINARY_STATIC_PY3="""cp "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" """ -CREATE_STATIC_BINARY_PY3_SCRIPT=select({ - "//:fast_binary": FAST_BINARY_STATIC_PY3, - "//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY_PY3, - "//conditions:default": FAST_BINARY_STATIC_PY3 +SLOW_WRAPPER_STATIC_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" "$(location //base:exaudfclient.template.sh)" """ +FAST_BINARY_STATIC="""cp "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" """ +CREATE_STATIC_BINARY_SCRIPT=select({ + "//:fast_binary": FAST_BINARY_STATIC, + "//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY, + "//conditions:default": FAST_BINARY_STATIC }) genrule( - name = "exaudfclient_py3_static", - cmd = CREATE_STATIC_BINARY_PY3_SCRIPT, - outs = ["exaudfclient_py3_static"], - srcs = [":exaudfclient_py3_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"], + name = "exaudfclient_static", + cmd = CREATE_STATIC_BINARY_SCRIPT, + outs = ["exaudfclient_static"], + srcs = [":exaudfclient_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"], output_to_bindir = True ) diff --git a/exaudfclient/README.md b/exaudfclient/README.md index c15b8691..b33e1e6a 100644 --- a/exaudfclient/README.md +++ b/exaudfclient/README.md @@ -1,6 +1,6 @@ # What is the exaudfclient? -The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in the language Python, Java and R. Further languages can be integrated via language binding between C/C++ and the desired langauge. Python 2/3, Java and R use [SWIG](http://www.swig.org/) for the language binding. +The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in the language Python, Java and R. Further languages can be integrated via language binding between C/C++ and the desired langauge. Python 3, Java and R use [SWIG](http://www.swig.org/) for the language binding. # How to build the exaudfclient? @@ -8,23 +8,23 @@ The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and f For the build system: -- Open JDK 8 -- bazel-0.22.0 for more details see [Bazel documentation](https://docs.bazel.build/versions/master/install.html) +- Open JDK 11 +- bazel-7.2.1 for more details see [Bazel documentation](https://docs.bazel.build/versions/master/install.html) The exaudfclient was tested with the following versions of its dependencies: - swig-2.0.4 or swig-3.0.12 -- protobuf 3.0.0 and 3.5.1 +- protobuf 3.12.4 - we need both compiler and library, they must be in the same version - don't install two version, because then its possible that you compile against the wrong header or link against wrong library -- zmq 4.2.5 +- zmq 4.3.4 For the language support: -- Python 2.7 or Python 3.6 for pythoncontainer +- Python 3.10 for pythoncontainer - for Python 3 the build requires [Numpy](http://www.numpy.org/) and [Pandas](https://pandas.pydata.org/) in addition for the Pandas Dataframe Support -- OpenJDK 9 or 11 for javacontainer -- R 3.4 or 3.5 for the rcontainer +- OpenJDK 11 for javacontainer +- R 4.4 for the rcontainer ## Start a build @@ -52,11 +52,11 @@ With Bazel defines you can specify which language support is actually compiled i --define benchmark=true # This language is only for test and development purpose and benchmarks the performance of the C++ Implementation -The main targets are //:exaudfclient and //:exaudfclient_py3. The former one compiles with Python 2 support if Python is via defines enable. The later one instead compiles with Python 3 support. +The main targets is //:exaudfclient. ## Visualizing the build dependencies -Bazel allows to query the dependencies of a target. Furthermore, it can export the dependencies as .dot file. With Graphviz you can generate figures from the .dot file. The script visualize_deps.sh and visualize_all.sh wrap this process. The script visualize_all.sh visualizes the dependencies of the main targets //:exaudfclient and //:exaudfclient_py3. The script visualize_deps.sh visualizes the dependencies of given targets. +Bazel allows to query the dependencies of a target. Furthermore, it can export the dependencies as .dot file. With Graphviz you can generate figures from the .dot file. The script visualize_deps.sh and visualize_all.sh wrap this process. The script visualize_all.sh visualizes the dependencies of the main targets //:exaudfclient. The script visualize_deps.sh visualizes the dependencies of given targets. visualize_deps.sh diff --git a/exaudfclient/build_local_all.sh b/exaudfclient/build_local_all.sh index bc8033b6..c91abe5c 100755 --- a/exaudfclient/build_local_all.sh +++ b/exaudfclient/build_local_all.sh @@ -1,3 +1,3 @@ #!/bin/bash -bash build_local.sh "$@" --config no-tty --config python --config java --config slow-wrapper-py3 +bash build_local.sh "$@" --config no-tty --config python --config java --config slow-wrapper diff --git a/exaudfclient/visualize_all.sh b/exaudfclient/visualize_all.sh index f72691bf..0b70bf67 100644 --- a/exaudfclient/visualize_all.sh +++ b/exaudfclient/visualize_all.sh @@ -1,3 +1,3 @@ #!/bin/bash -bash visualize_deps.sh "//:exaudfclient //:exaudfclient_py3" "$@" \ No newline at end of file +bash visualize_deps.sh "//:exaudfclient" "$@" \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile index 09812f99..e4dcea21 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile @@ -12,12 +12,12 @@ WORKDIR /exaudfclient/ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ -RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3 +RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static +RUN ./test_udfclient.sh /exaudf/exaudfclient +RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile index 434fced7..1d68e1e3 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile @@ -7,11 +7,11 @@ WORKDIR /exaudfclient/ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ -RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3 +RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 +RUN ./test_udfclient.sh /exaudf/exaudfclient WORKDIR /exaudf diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_definition b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_definition index 82e76acf..0576fe70 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_definition +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_definition @@ -1 +1 @@ -PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3 +PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile index fddfb7be..7cbacd82 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile @@ -12,12 +12,12 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ # We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf -RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3 +RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static +RUN ./test_udfclient.sh /exaudf/exaudfclient +RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_run/Dockerfile index 434fced7..1d68e1e3 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_run/Dockerfile @@ -7,11 +7,11 @@ WORKDIR /exaudfclient/ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ -RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3 +RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 +RUN ./test_udfclient.sh /exaudf/exaudfclient WORKDIR /exaudf diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/language_definition b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/language_definition index 82e76acf..0576fe70 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/language_definition +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/language_definition @@ -1 +1 @@ -PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3 +PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile index 3038297b..b87c2bfc 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile @@ -27,12 +27,12 @@ RUN mkdir /exaudfclient /exaudf COPY /exaudfclient/ /exaudfclient/ WORKDIR /exaudfclient/ -RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries-py3"] +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries"] RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static +RUN ./test_udfclient.sh /exaudf/exaudfclient +RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/build_run/Dockerfile index 1ac4548f..4863abe9 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/build_run/Dockerfile @@ -25,11 +25,11 @@ RUN mkdir /exaudfclient /exaudf COPY exaudfclient/ /exaudfclient/ WORKDIR /exaudfclient/ -RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary-py3"] +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary"] RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base -RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 +RUN ./test_udfclient.sh /exaudf/exaudfclient WORKDIR /exaudf diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/language_definition b/flavors/template-Exasol-all-python-3.10/flavor_base/language_definition index 82e76acf..0576fe70 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/language_definition +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/language_definition @@ -1 +1 @@ -PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3 +PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient