Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#944: Renamed exaudfclient_py3_bin -> exaudfclient_bin #452

Merged
merged 1 commit into from
Oct 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions exaudfclient/.bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_
build:benchmark --define benchmark=true
build:java --define java=true --action_env=JAVA_PREFIX
build:python --define python=true --action_env=PYTHON2_SYSPATH --action_env=PYTHON2_PREFIX --action_env=PYTHON2_VERSION --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION
build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient_py3
build:slow-wrapper-py3 --define binary_type=slow_wrapper //:exaudfclient_py3
build:static-binary-py3 //:exaudfclient_py3_static
build:test-binaries-py3 --config=static-binary-py3 --config=slow-wrapper-py3
build:fast-binary --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient
build:slow-wrapper --define binary_type=slow_wrapper //:exaudfclient
build:static-binary //:exaudfclient_static
build:test-binaries --config=static-binary --config=slow-wrapper
build:verbose --copt='-v' --subcommands --verbose_failures --announce_rc
#TODO test linkopts="-flto"
build:optimize --copt="-g0" --copt="-DNDEBUG" --copt=-fstack-protector-strong --copt=-fomit-frame-pointer --copt=-ffunction-sections --copt=-fdata-sections --copt="-O3" --copt="-U_FORTIFY_SOURCE" --copt="-flto" --copt="-fuse-linker-plugin"
Expand Down
44 changes: 22 additions & 22 deletions exaudfclient/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ VM_PYTHON3_DEPS=select({
})

cc_binary(
name = "exaudfclient_py3_bin",
name = "exaudfclient_bin",
srcs = ["exaudfclient.cc", "//base:load_dynamic"],
linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+
Expand All @@ -97,7 +97,7 @@ cc_binary(
## as dependency as it is a binary for bazel.

cc_binary(
name = "exaudfclient_py3_static_bin",
name = "exaudfclient_static_bin",
srcs = ["exaudfclient.cc", "//base:load_dynamic"],
linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+
Expand Down Expand Up @@ -125,33 +125,33 @@ sh_library(
})
)

SLOW_WRAPPER_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" "$(location //base:exaudfclient.template.sh)" """
FAST_BINARY_PY3="""cp "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" """
CREATE_BINARY_PY3_SCRIPT=select({
"//:fast_binary": FAST_BINARY_PY3,
"//:slow_wrapper": SLOW_WRAPPER_BINARY_PY3,
"//conditions:default": FAST_BINARY_PY3
SLOW_WRAPPER_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_bin)" "$(location exaudfclient)" "$(location //base:exaudfclient.template.sh)" """
FAST_BINARY="""cp "$(location exaudfclient_bin)" "$(location exaudfclient)" """
CREATE_BINARY_SCRIPT=select({
"//:fast_binary": FAST_BINARY,
"//:slow_wrapper": SLOW_WRAPPER_BINARY,
"//conditions:default": FAST_BINARY
})
genrule(
name = "exaudfclient_py3",
cmd = CREATE_BINARY_PY3_SCRIPT,
outs = ["exaudfclient_py3"],
srcs = [":exaudfclient_py3_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
name = "exaudfclient",
cmd = CREATE_BINARY_SCRIPT,
outs = ["exaudfclient"],
srcs = [":exaudfclient_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
output_to_bindir = True
)

SLOW_WRAPPER_STATIC_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" "$(location //base:exaudfclient.template.sh)" """
FAST_BINARY_STATIC_PY3="""cp "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" """
CREATE_STATIC_BINARY_PY3_SCRIPT=select({
"//:fast_binary": FAST_BINARY_STATIC_PY3,
"//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY_PY3,
"//conditions:default": FAST_BINARY_STATIC_PY3
SLOW_WRAPPER_STATIC_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" "$(location //base:exaudfclient.template.sh)" """
FAST_BINARY_STATIC="""cp "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" """
CREATE_STATIC_BINARY_SCRIPT=select({
"//:fast_binary": FAST_BINARY_STATIC,
"//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY,
"//conditions:default": FAST_BINARY_STATIC
})

genrule(
name = "exaudfclient_py3_static",
cmd = CREATE_STATIC_BINARY_PY3_SCRIPT,
outs = ["exaudfclient_py3_static"],
srcs = [":exaudfclient_py3_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
name = "exaudfclient_static",
cmd = CREATE_STATIC_BINARY_SCRIPT,
outs = ["exaudfclient_static"],
srcs = [":exaudfclient_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
output_to_bindir = True
)
20 changes: 10 additions & 10 deletions exaudfclient/README.md
Original file line number Diff line number Diff line change
@@ -1,30 +1,30 @@
# What is the exaudfclient?

The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in the language Python, Java and R. Further languages can be integrated via language binding between C/C++ and the desired langauge. Python 2/3, Java and R use [SWIG](http://www.swig.org/) for the language binding.
The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in the language Python, Java and R. Further languages can be integrated via language binding between C/C++ and the desired langauge. Python 3, Java and R use [SWIG](http://www.swig.org/) for the language binding.

# How to build the exaudfclient?

## Prerequisites

For the build system:

- Open JDK 8
- bazel-0.22.0 for more details see [Bazel documentation](https://docs.bazel.build/versions/master/install.html)
- Open JDK 11
- bazel-7.2.1 for more details see [Bazel documentation](https://docs.bazel.build/versions/master/install.html)

The exaudfclient was tested with the following versions of its dependencies:

- swig-2.0.4 or swig-3.0.12
- protobuf 3.0.0 and 3.5.1
- protobuf 3.12.4
- we need both compiler and library, they must be in the same version
- don't install two version, because then its possible that you compile against the wrong header or link against wrong library
- zmq 4.2.5
- zmq 4.3.4

For the language support:

- Python 2.7 or Python 3.6 for pythoncontainer
- Python 3.10 for pythoncontainer
- for Python 3 the build requires [Numpy](http://www.numpy.org/) and [Pandas](https://pandas.pydata.org/) in addition for the Pandas Dataframe Support
- OpenJDK 9 or 11 for javacontainer
- R 3.4 or 3.5 for the rcontainer
- OpenJDK 11 for javacontainer
- R 4.4 for the rcontainer

## Start a build

Expand Down Expand Up @@ -52,11 +52,11 @@ With Bazel defines you can specify which language support is actually compiled i
--define benchmark=true # This language is only for test and development purpose and benchmarks the performance of the C++ Implementation


The main targets are //:exaudfclient and //:exaudfclient_py3. The former one compiles with Python 2 support if Python is via defines enable. The later one instead compiles with Python 3 support.
The main targets is //:exaudfclient.

## Visualizing the build dependencies

Bazel allows to query the dependencies of a target. Furthermore, it can export the dependencies as .dot file. With Graphviz you can generate figures from the .dot file. The script visualize_deps.sh and visualize_all.sh wrap this process. The script visualize_all.sh visualizes the dependencies of the main targets //:exaudfclient and //:exaudfclient_py3. The script visualize_deps.sh visualizes the dependencies of given targets.
Bazel allows to query the dependencies of a target. Furthermore, it can export the dependencies as .dot file. With Graphviz you can generate figures from the .dot file. The script visualize_deps.sh and visualize_all.sh wrap this process. The script visualize_all.sh visualizes the dependencies of the main targets //:exaudfclient. The script visualize_deps.sh visualizes the dependencies of given targets.

visualize_deps.sh <targets>

Expand Down
2 changes: 1 addition & 1 deletion exaudfclient/build_local_all.sh
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#!/bin/bash

bash build_local.sh "$@" --config no-tty --config python --config java --config slow-wrapper-py3
bash build_local.sh "$@" --config no-tty --config python --config java --config slow-wrapper
2 changes: 1 addition & 1 deletion exaudfclient/visualize_all.sh
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#!/bin/bash

bash visualize_deps.sh "//:exaudfclient //:exaudfclient_py3" "$@"
bash visualize_deps.sh "//:exaudfclient" "$@"
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ WORKDIR /exaudfclient/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static
RUN ./test_udfclient.sh /exaudf/exaudfclient
RUN ./test_udfclient.sh /exaudf/exaudfclient_static

WORKDIR /
RUN mkdir /exasol_emulator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ WORKDIR /exaudfclient/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient

WORKDIR /exaudf

Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
# We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static
RUN ./test_udfclient.sh /exaudf/exaudfclient
RUN ./test_udfclient.sh /exaudf/exaudfclient_static

WORKDIR /
RUN mkdir /exasol_emulator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ WORKDIR /exaudfclient/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient

WORKDIR /exaudf

Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ RUN mkdir /exaudfclient /exaudf
COPY /exaudfclient/ /exaudfclient/

WORKDIR /exaudfclient/
RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries-py3"]
RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries"]
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static
RUN ./test_udfclient.sh /exaudf/exaudfclient
RUN ./test_udfclient.sh /exaudf/exaudfclient_static

WORKDIR /
RUN mkdir /exasol_emulator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ RUN mkdir /exaudfclient /exaudf
COPY exaudfclient/ /exaudfclient/

WORKDIR /exaudfclient/
RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary-py3"]
RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary"]
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient

WORKDIR /exaudf

Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient