Skip to content

Commit

Permalink
Renamed exaudfclient_py3_bin -> exaudfclient_bin
Browse files Browse the repository at this point in the history
  • Loading branch information
tomuben committed Oct 9, 2024
1 parent ca29b6f commit 686cd39
Show file tree
Hide file tree
Showing 14 changed files with 56 additions and 56 deletions.
8 changes: 4 additions & 4 deletions exaudfclient/.bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_
build:benchmark --define benchmark=true
build:java --define java=true --action_env=JAVA_PREFIX
build:python --define python=true --action_env=PYTHON2_SYSPATH --action_env=PYTHON2_PREFIX --action_env=PYTHON2_VERSION --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION
build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient_py3
build:slow-wrapper-py3 --define binary_type=slow_wrapper //:exaudfclient_py3
build:static-binary-py3 //:exaudfclient_py3_static
build:test-binaries-py3 --config=static-binary-py3 --config=slow-wrapper-py3
build:fast-binary --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient
build:slow-wrapper --define binary_type=slow_wrapper //:exaudfclient
build:static-binary //:exaudfclient_static
build:test-binaries --config=static-binary --config=slow-wrapper
build:verbose --copt='-v' --subcommands --verbose_failures --announce_rc
#TODO test linkopts="-flto"
build:optimize --copt="-g0" --copt="-DNDEBUG" --copt=-fstack-protector-strong --copt=-fomit-frame-pointer --copt=-ffunction-sections --copt=-fdata-sections --copt="-O3" --copt="-U_FORTIFY_SOURCE" --copt="-flto" --copt="-fuse-linker-plugin"
Expand Down
44 changes: 22 additions & 22 deletions exaudfclient/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ VM_PYTHON3_DEPS=select({
})

cc_binary(
name = "exaudfclient_py3_bin",
name = "exaudfclient_bin",
srcs = ["exaudfclient.cc", "//base:load_dynamic"],
linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+
Expand All @@ -97,7 +97,7 @@ cc_binary(
## as dependency as it is a binary for bazel.

cc_binary(
name = "exaudfclient_py3_static_bin",
name = "exaudfclient_static_bin",
srcs = ["exaudfclient.cc", "//base:load_dynamic"],
linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+
Expand Down Expand Up @@ -125,33 +125,33 @@ sh_library(
})
)

SLOW_WRAPPER_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" "$(location //base:exaudfclient.template.sh)" """
FAST_BINARY_PY3="""cp "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" """
CREATE_BINARY_PY3_SCRIPT=select({
"//:fast_binary": FAST_BINARY_PY3,
"//:slow_wrapper": SLOW_WRAPPER_BINARY_PY3,
"//conditions:default": FAST_BINARY_PY3
SLOW_WRAPPER_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_bin)" "$(location exaudfclient)" "$(location //base:exaudfclient.template.sh)" """
FAST_BINARY="""cp "$(location exaudfclient_bin)" "$(location exaudfclient)" """
CREATE_BINARY_SCRIPT=select({
"//:fast_binary": FAST_BINARY,
"//:slow_wrapper": SLOW_WRAPPER_BINARY,
"//conditions:default": FAST_BINARY
})
genrule(
name = "exaudfclient_py3",
cmd = CREATE_BINARY_PY3_SCRIPT,
outs = ["exaudfclient_py3"],
srcs = [":exaudfclient_py3_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
name = "exaudfclient",
cmd = CREATE_BINARY_SCRIPT,
outs = ["exaudfclient"],
srcs = [":exaudfclient_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
output_to_bindir = True
)

SLOW_WRAPPER_STATIC_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" "$(location //base:exaudfclient.template.sh)" """
FAST_BINARY_STATIC_PY3="""cp "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" """
CREATE_STATIC_BINARY_PY3_SCRIPT=select({
"//:fast_binary": FAST_BINARY_STATIC_PY3,
"//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY_PY3,
"//conditions:default": FAST_BINARY_STATIC_PY3
SLOW_WRAPPER_STATIC_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" "$(location //base:exaudfclient.template.sh)" """
FAST_BINARY_STATIC="""cp "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" """
CREATE_STATIC_BINARY_SCRIPT=select({
"//:fast_binary": FAST_BINARY_STATIC,
"//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY,
"//conditions:default": FAST_BINARY_STATIC
})

genrule(
name = "exaudfclient_py3_static",
cmd = CREATE_STATIC_BINARY_PY3_SCRIPT,
outs = ["exaudfclient_py3_static"],
srcs = [":exaudfclient_py3_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
name = "exaudfclient_static",
cmd = CREATE_STATIC_BINARY_SCRIPT,
outs = ["exaudfclient_static"],
srcs = [":exaudfclient_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
output_to_bindir = True
)
20 changes: 10 additions & 10 deletions exaudfclient/README.md
Original file line number Diff line number Diff line change
@@ -1,30 +1,30 @@
# What is the exaudfclient?

The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in the language Python, Java and R. Further languages can be integrated via language binding between C/C++ and the desired langauge. Python 2/3, Java and R use [SWIG](http://www.swig.org/) for the language binding.
The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in the language Python, Java and R. Further languages can be integrated via language binding between C/C++ and the desired langauge. Python 3, Java and R use [SWIG](http://www.swig.org/) for the language binding.

# How to build the exaudfclient?

## Prerequisites

For the build system:

- Open JDK 8
- bazel-0.22.0 for more details see [Bazel documentation](https://docs.bazel.build/versions/master/install.html)
- Open JDK 11
- bazel-7.2.1 for more details see [Bazel documentation](https://docs.bazel.build/versions/master/install.html)

The exaudfclient was tested with the following versions of its dependencies:

- swig-2.0.4 or swig-3.0.12
- protobuf 3.0.0 and 3.5.1
- protobuf 3.12.4
- we need both compiler and library, they must be in the same version
- don't install two version, because then its possible that you compile against the wrong header or link against wrong library
- zmq 4.2.5
- zmq 4.3.4

For the language support:

- Python 2.7 or Python 3.6 for pythoncontainer
- Python 3.10 for pythoncontainer
- for Python 3 the build requires [Numpy](http://www.numpy.org/) and [Pandas](https://pandas.pydata.org/) in addition for the Pandas Dataframe Support
- OpenJDK 9 or 11 for javacontainer
- R 3.4 or 3.5 for the rcontainer
- OpenJDK 11 for javacontainer
- R 4.4 for the rcontainer

## Start a build

Expand Down Expand Up @@ -52,11 +52,11 @@ With Bazel defines you can specify which language support is actually compiled i
--define benchmark=true # This language is only for test and development purpose and benchmarks the performance of the C++ Implementation


The main targets are //:exaudfclient and //:exaudfclient_py3. The former one compiles with Python 2 support if Python is via defines enable. The later one instead compiles with Python 3 support.
The main targets is //:exaudfclient.

## Visualizing the build dependencies

Bazel allows to query the dependencies of a target. Furthermore, it can export the dependencies as .dot file. With Graphviz you can generate figures from the .dot file. The script visualize_deps.sh and visualize_all.sh wrap this process. The script visualize_all.sh visualizes the dependencies of the main targets //:exaudfclient and //:exaudfclient_py3. The script visualize_deps.sh visualizes the dependencies of given targets.
Bazel allows to query the dependencies of a target. Furthermore, it can export the dependencies as .dot file. With Graphviz you can generate figures from the .dot file. The script visualize_deps.sh and visualize_all.sh wrap this process. The script visualize_all.sh visualizes the dependencies of the main targets //:exaudfclient. The script visualize_deps.sh visualizes the dependencies of given targets.

visualize_deps.sh <targets>

Expand Down
2 changes: 1 addition & 1 deletion exaudfclient/build_local_all.sh
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#!/bin/bash

bash build_local.sh "$@" --config no-tty --config python --config java --config slow-wrapper-py3
bash build_local.sh "$@" --config no-tty --config python --config java --config slow-wrapper
2 changes: 1 addition & 1 deletion exaudfclient/visualize_all.sh
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#!/bin/bash

bash visualize_deps.sh "//:exaudfclient //:exaudfclient_py3" "$@"
bash visualize_deps.sh "//:exaudfclient" "$@"
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ WORKDIR /exaudfclient/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static
RUN ./test_udfclient.sh /exaudf/exaudfclient
RUN ./test_udfclient.sh /exaudf/exaudfclient_static

WORKDIR /
RUN mkdir /exasol_emulator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ WORKDIR /exaudfclient/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient

WORKDIR /exaudf

Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
# We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static
RUN ./test_udfclient.sh /exaudf/exaudfclient
RUN ./test_udfclient.sh /exaudf/exaudfclient_static

WORKDIR /
RUN mkdir /exasol_emulator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ WORKDIR /exaudfclient/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient

WORKDIR /exaudf

Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ RUN mkdir /exaudfclient /exaudf
COPY /exaudfclient/ /exaudfclient/

WORKDIR /exaudfclient/
RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries-py3"]
RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries"]
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static
RUN ./test_udfclient.sh /exaudf/exaudfclient
RUN ./test_udfclient.sh /exaudf/exaudfclient_static

WORKDIR /
RUN mkdir /exasol_emulator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ RUN mkdir /exaudfclient /exaudf
COPY exaudfclient/ /exaudfclient/

WORKDIR /exaudfclient/
RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary-py3"]
RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary"]
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
RUN ./test_udfclient.sh /exaudf/exaudfclient

WORKDIR /exaudf

Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient

0 comments on commit 686cd39

Please sign in to comment.