From 733a3c6e82d900522fe64217ae234ea4c4afbe7a Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Tue, 17 Nov 2020 12:24:57 +0100 Subject: [PATCH 01/11] Allow linkage to system protobuf --- tensorflow_cc/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tensorflow_cc/CMakeLists.txt b/tensorflow_cc/CMakeLists.txt index 0739ad4..f7142c3 100644 --- a/tensorflow_cc/CMakeLists.txt +++ b/tensorflow_cc/CMakeLists.txt @@ -9,6 +9,7 @@ project( option(ALLOW_CUDA "Try to find and use CUDA." ON) option(REQUIRE_CUDA "Make sure to find and use CUDA (implies ALLOW_CUDA)." OFF) set(TENSORFLOW_TAG "v${version}" CACHE STRING "The tensorflow release tag to be checked out (default v${version}).") +option(SYSTEM_PROTOBUF "Link to system protobuf." OFF) set(TARGET_CXX_STANDARD "cxx_std_11" CACHE STRING "C++ standard to be enforced when linking to TensorflowCC targets (e.g., cxx_std_11).") # ------------- @@ -53,6 +54,17 @@ target_link_libraries( "${CMAKE_INSTALL_PREFIX}/lib/libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}" dl pthread ) +if(SYSTEM_PROTOBUF) + find_package(Protobuf REQUIRED) + target_include_directories( + tensorflow_cc INTERFACE + "${Protobuf_INCLUDE_DIRS}" + ) + target_link_libraries( + tensorflow_cc INTERFACE + "${Protobuf_LIBRARIES}" + ) +endif() # ---------------------------------------- # Configure CMake Config and Version Files From db7768bb5a32adb34253845ef04deb2ead8a3978 Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Sat, 21 May 2022 20:51:49 +0200 Subject: [PATCH 02/11] Install protobuf version compatible with the current TensorFlow version --- tensorflow_cc/CMakeLists.txt | 20 ++++++++------------ tensorflow_cc/cmake/ProtobufExternal.cmake | 20 ++++++++++++++++++++ tensorflow_cc/cmake/TensorflowBase.cmake | 3 ++- 3 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 tensorflow_cc/cmake/ProtobufExternal.cmake diff --git a/tensorflow_cc/CMakeLists.txt b/tensorflow_cc/CMakeLists.txt index 9f6b5cf..1dc1c67 100644 --- a/tensorflow_cc/CMakeLists.txt +++ b/tensorflow_cc/CMakeLists.txt @@ -11,7 +11,7 @@ option(REQUIRE_CUDA "Make sure to find and use CUDA (implies ALLOW_CUDA)." OFF) set(LOCAL_RAM_RESOURCES 4096 CACHE STRING "The amount of local RAM resources passed to bazel scheduler (e.g., 4096).") set(LOCAL_CPU_RESOURCES HOST_CPUS CACHE STRING "The amount of local CPU cores passed to bazel scheduler (e.g., 2).") set(TENSORFLOW_TAG "v${version}" CACHE STRING "The tensorflow release tag to be checked out (default v${version}).") -option(SYSTEM_PROTOBUF "Link to system protobuf." OFF) +option(INSTALL_PROTOBUF "Install protobuf compatible with tensorflow version." OFF) set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ standard for building and linking the library (e.g., 14).") # ------------- @@ -32,6 +32,9 @@ configure_file("cmake/build_tensorflow.sh.in" "build_tensorflow.sh" @ONLY) # ---------------------------------------------- include(TensorflowBase) +if(INSTALL_PROTOBUF) + include(ProtobufExternal) +endif() # ------------------------------ # Define Tensorflow_CC Interface @@ -40,6 +43,10 @@ include(TensorflowBase) add_library(tensorflow_cc INTERFACE) target_compile_features(tensorflow_cc INTERFACE "cxx_std_${CMAKE_CXX_STANDARD}") +if(INSTALL_PROTOBUF) + add_dependencies(tensorflow_cc protobuf-external) +endif() + # The include folders are sometimes contained under bazel-bin/bin/ and sometimes just bazel-bin. target_include_directories( tensorflow_cc INTERFACE @@ -53,17 +60,6 @@ target_link_libraries( "${CMAKE_INSTALL_PREFIX}/lib/libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}" dl pthread ) -if(SYSTEM_PROTOBUF) - find_package(Protobuf REQUIRED) - target_include_directories( - tensorflow_cc INTERFACE - "${Protobuf_INCLUDE_DIRS}" - ) - target_link_libraries( - tensorflow_cc INTERFACE - "${Protobuf_LIBRARIES}" - ) -endif() # ---------------------------------------- # Configure CMake Config and Version Files diff --git a/tensorflow_cc/cmake/ProtobufExternal.cmake b/tensorflow_cc/cmake/ProtobufExternal.cmake new file mode 100644 index 0000000..ae9909e --- /dev/null +++ b/tensorflow_cc/cmake/ProtobufExternal.cmake @@ -0,0 +1,20 @@ +set(PROTOBUF_ARCHIVE https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip) + +ExternalProject_Add( + protobuf-external + PREFIX protobuf + URL "${PROTOBUF_ARCHIVE}" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/protobuf" + CMAKE_CACHE_ARGS + "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" + "-Dprotobuf_BUILD_TESTS:BOOL=OFF" + "-Dprotobuf_BUILD_EXAMPLES:BOOL=OFF" + "-DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}" + SOURCE_SUBDIR cmake + BUILD_ALWAYS 1 + STEP_TARGETS build + INSTALL_COMMAND "" +) + +install(SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/protobuf/cmake_install.cmake") + diff --git a/tensorflow_cc/cmake/TensorflowBase.cmake b/tensorflow_cc/cmake/TensorflowBase.cmake index 0e29601..74d2478 100644 --- a/tensorflow_cc/cmake/TensorflowBase.cmake +++ b/tensorflow_cc/cmake/TensorflowBase.cmake @@ -11,7 +11,8 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 UPDATE_COMMAND "" CONFIGURE_COMMAND "" - BUILD_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/build_tensorflow.sh" + BUILD_COMMAND echo "*" > "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/.bazelversion" + COMMAND "${CMAKE_CURRENT_BINARY_DIR}/build_tensorflow.sh" INSTALL_COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/cmake/copy_links.sh" bazel-bin # The include folders are sometimes contained under bazel-bin/bin/ and sometimes just bazel-bin. # Later, we include both so let's make sure they both exist so that CMake does not complain. From 3a1aaf97105561b891800e4e9cedc9f24bc907a1 Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Sat, 21 May 2022 21:13:04 +0200 Subject: [PATCH 03/11] Find the protobuf version automatically --- tensorflow_cc/cmake/ProtobufExternal.cmake | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tensorflow_cc/cmake/ProtobufExternal.cmake b/tensorflow_cc/cmake/ProtobufExternal.cmake index ae9909e..61f3b40 100644 --- a/tensorflow_cc/cmake/ProtobufExternal.cmake +++ b/tensorflow_cc/cmake/ProtobufExternal.cmake @@ -1,4 +1,18 @@ -set(PROTOBUF_ARCHIVE https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip) +# Find the proper protobuf archive url. +file(DOWNLOAD + "https://raw.githubusercontent.com/tensorflow/tensorflow/${TENSORFLOW_TAG}/tensorflow/workspace2.bzl" + "${CMAKE_CURRENT_BINARY_DIR}/tmp/workspace2.bzl" +) +file(READ + "${CMAKE_CURRENT_BINARY_DIR}/tmp/workspace2.bzl" + workspace2_str +) +string(REGEX MATCH + "https://github.com/protocolbuffers/protobuf/archive/v[.0-9]+.zip" + PROTOBUF_ARCHIVE + "${workspace2_str}" +) +message("Will build Protobuf from '${PROTOBUF_ARCHIVE}'.") ExternalProject_Add( protobuf-external From e8a2a2c5278198f4ce7ef26b54afab935c4d2be3 Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Sat, 21 May 2022 22:12:22 +0200 Subject: [PATCH 04/11] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e033de5..8fbce1e 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,10 @@ sudo pacman -S cuda cudnn nvidia ``` **Warning:** Newer versions of TensorFlow sometimes fail to build with the latest version of Bazel. You may wish -to install an older version of Bazel (e.g., 3.7.2). +to install an older version of Bazel (e.g., 5.1.1). + +**Warning:** If your program uses Protobuf and you encounter linkage problems, you can try `-DINSTALL_PROTOBUF=ON` option +to install a Protobuf version matching the version bundled with TensorFlow. #### 2) Clone this repository ``` From a868dabe2749200e0d384e3d0b368ab33f13b3ea Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Sun, 22 May 2022 11:14:19 +0200 Subject: [PATCH 05/11] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8fbce1e..d5a19f0 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,10 @@ sudo pacman -S cuda cudnn nvidia **Warning:** Newer versions of TensorFlow sometimes fail to build with the latest version of Bazel. You may wish to install an older version of Bazel (e.g., 5.1.1). -**Warning:** If your program uses Protobuf and you encounter linkage problems, you can try `-DINSTALL_PROTOBUF=ON` option -to install a Protobuf version matching the version bundled with TensorFlow. +**Warning:** If your program uses Protobuf and you encounter linkage or other problems, you can +try `-DINSTALL_PROTOBUF=ON` option to install a Protobuf version matching the version bundled with TensorFlow. +Our Dockerfiles are already built with the right version of Protobuf, so you may want to try +your program in the Dockerfile first. #### 2) Clone this repository ``` From e8e10efc3cb6d6b8a83bc9dfe2970844c517c2eb Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Sun, 22 May 2022 11:47:16 +0200 Subject: [PATCH 06/11] Simplify external protobuf script --- tensorflow_cc/cmake/ProtobufExternal.cmake | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorflow_cc/cmake/ProtobufExternal.cmake b/tensorflow_cc/cmake/ProtobufExternal.cmake index 61f3b40..3957181 100644 --- a/tensorflow_cc/cmake/ProtobufExternal.cmake +++ b/tensorflow_cc/cmake/ProtobufExternal.cmake @@ -18,17 +18,14 @@ ExternalProject_Add( protobuf-external PREFIX protobuf URL "${PROTOBUF_ARCHIVE}" - BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/protobuf" CMAKE_CACHE_ARGS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" "-Dprotobuf_BUILD_TESTS:BOOL=OFF" "-Dprotobuf_BUILD_EXAMPLES:BOOL=OFF" "-DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}" SOURCE_SUBDIR cmake - BUILD_ALWAYS 1 - STEP_TARGETS build INSTALL_COMMAND "" ) -install(SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/protobuf/cmake_install.cmake") +install(SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf-external-build/cmake_install.cmake") From bf4ce3dcb2d5d9504080708788a14f5ea717da84 Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Sun, 22 May 2022 11:47:41 +0200 Subject: [PATCH 07/11] Tick CMake minimum version --- tensorflow_cc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_cc/CMakeLists.txt b/tensorflow_cc/CMakeLists.txt index 54f9522..9e23a37 100644 --- a/tensorflow_cc/CMakeLists.txt +++ b/tensorflow_cc/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.3 FATAL_ERROR) +cmake_minimum_required(VERSION 3.7 FATAL_ERROR) cmake_policy(SET CMP0048 NEW) # Enable version parameter in project(). file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/PROJECT_VERSION" version) project( From 38cf926673601e0ba57a47f46949857b7f96ab6f Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Sun, 22 May 2022 11:48:21 +0200 Subject: [PATCH 08/11] Install Protobuf in Dockerfiles by default --- Dockerfiles/install-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfiles/install-common.sh b/Dockerfiles/install-common.sh index 1cbf91a..56d7950 100755 --- a/Dockerfiles/install-common.sh +++ b/Dockerfiles/install-common.sh @@ -19,7 +19,7 @@ chmod go+rX tensorflow_cc/tensorflow_cc/build cd tensorflow_cc/tensorflow_cc/build # build and install -cmake -DLOCAL_RAM_RESOURCES=2048 -DLOCAL_CPU_RESOURCES=2 .. +cmake -DLOCAL_RAM_RESOURCES=2048 -DLOCAL_CPU_RESOURCES=2 -DINSTALL_PROTOBUF=ON .. make rm -rf /home/tensorflow_cc/.cache rm -rf /root/.cache From 81dba179df0ee0f42c51a5abcecb705383b02878 Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Sun, 22 May 2022 11:53:53 +0200 Subject: [PATCH 09/11] Add .dockerignore --- .dockerignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cae295a --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +tensorflow_cc/build/ From 54b58a3cd3070e7d12b126bff2be78b843740b0a Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Mon, 23 May 2022 14:21:59 +0200 Subject: [PATCH 10/11] Assign all neccessary CC variables --- tensorflow_cc/cmake/build_tensorflow.sh.in | 31 +++++++++++----------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tensorflow_cc/cmake/build_tensorflow.sh.in b/tensorflow_cc/cmake/build_tensorflow.sh.in index a425f9c..a9e2689 100755 --- a/tensorflow_cc/cmake/build_tensorflow.sh.in +++ b/tensorflow_cc/cmake/build_tensorflow.sh.in @@ -64,28 +64,27 @@ if [ "$cuda_allowed" == true ] && [ "$cuda_available" == true ]; then export TF_CUDNN_VERSION="$(find /opt /usr -name 'libcudnn.so.*' -path '*/cuda*' | tail -n1 | sed -r 's/^.*\.so\.//')" # choose the right version of CUDA compiler - if [ -z "$GCC_HOST_COMPILER_PATH" ]; then - if hash gcc-11 2>/dev/null && version_gt 11.1 `gcc-11 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-11"} + if [ -z "$CC" ] || [ -z $GCC ]; then + if hash gcc-11 2>/dev/null && version_gt 11.4 `gcc-11 -dumpversion`; then + export CC=${CC:-"gcc-11"} + export CXX=${CXX:-"g++-11"} elif hash gcc-10 2>/dev/null && version_gt 10.3 `gcc-10 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-10"} + export CC=${CC:-"gcc-10"} + export CXX=${CXX:-"g++-10"} elif hash gcc-9 2>/dev/null && version_gt 9.4 `gcc-9 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-9"} + export CC=${CC:-"gcc-9"} + export CXX=${CXX:-"g++-9"} elif hash gcc-8 2>/dev/null && version_gt 8.5 `gcc-8 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-8"} - elif hash gcc-7 2>/dev/null && version_gt 7.5 `gcc-7 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-7"} - elif hash gcc-6 2>/dev/null && version_gt 6.4 `gcc-6 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-6"} - elif hash gcc-5 2>/dev/null && version_gt 5.5 `gcc-5 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-5"} - elif hash gcc-4 2>/dev/null && version_gt 4.9 `gcc-4 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-4"} + export CC=${CC:-"gcc-8"} + export CXX=${CXX:-"g++-8"} else - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc"} + export CC=${CC:-"gcc"} + export CXX=${CXX:-"g++"} fi fi - + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which $CC`} + export HOST_C_COMPILER=${HOST_C_COMPILER:-`which $CC`} + export HOST_CXX_COMPILER=${HOST_CXX_COMPILER:-`which $CXX`} export CLANG_CUDA_COMPILER_PATH=${CLANG_CUDA_COMPILER_PATH:-"/usr/bin/clang"} export TF_CUDA_CLANG=${TF_CUDA_CLANG:-0} else From fb33ad4330dd284b0ab6754d85c2ebc096df2949 Mon Sep 17 00:00:00 2001 From: Filip Matzner Date: Mon, 23 May 2022 18:01:15 +0200 Subject: [PATCH 11/11] Simplify build_tensorflow.sh.in script --- Dockerfiles/install-archlinux.sh | 1 + tensorflow_cc/cmake/build_tensorflow.sh.in | 21 +++++++-------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Dockerfiles/install-archlinux.sh b/Dockerfiles/install-archlinux.sh index 0c35895..9384114 100755 --- a/Dockerfiles/install-archlinux.sh +++ b/Dockerfiles/install-archlinux.sh @@ -15,6 +15,7 @@ done # install requirements pacman -Syu --noconfirm --needed \ base-devel \ + gcc11 \ cmake \ git \ python \ diff --git a/tensorflow_cc/cmake/build_tensorflow.sh.in b/tensorflow_cc/cmake/build_tensorflow.sh.in index a9e2689..3234a50 100755 --- a/tensorflow_cc/cmake/build_tensorflow.sh.in +++ b/tensorflow_cc/cmake/build_tensorflow.sh.in @@ -64,27 +64,20 @@ if [ "$cuda_allowed" == true ] && [ "$cuda_available" == true ]; then export TF_CUDNN_VERSION="$(find /opt /usr -name 'libcudnn.so.*' -path '*/cuda*' | tail -n1 | sed -r 's/^.*\.so\.//')" # choose the right version of CUDA compiler - if [ -z "$CC" ] || [ -z $GCC ]; then + if [ -z "${GCC_HOST_COMPILER_PATH}" ]; then if hash gcc-11 2>/dev/null && version_gt 11.4 `gcc-11 -dumpversion`; then - export CC=${CC:-"gcc-11"} - export CXX=${CXX:-"g++-11"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-11`} elif hash gcc-10 2>/dev/null && version_gt 10.3 `gcc-10 -dumpversion`; then - export CC=${CC:-"gcc-10"} - export CXX=${CXX:-"g++-10"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-10`} elif hash gcc-9 2>/dev/null && version_gt 9.4 `gcc-9 -dumpversion`; then - export CC=${CC:-"gcc-9"} - export CXX=${CXX:-"g++-9"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-9`} elif hash gcc-8 2>/dev/null && version_gt 8.5 `gcc-8 -dumpversion`; then - export CC=${CC:-"gcc-8"} - export CXX=${CXX:-"g++-8"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-8`} else - export CC=${CC:-"gcc"} - export CXX=${CXX:-"g++"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc`} fi fi - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which $CC`} - export HOST_C_COMPILER=${HOST_C_COMPILER:-`which $CC`} - export HOST_CXX_COMPILER=${HOST_CXX_COMPILER:-`which $CXX`} + export CLANG_CUDA_COMPILER_PATH=${CLANG_CUDA_COMPILER_PATH:-"/usr/bin/clang"} export TF_CUDA_CLANG=${TF_CUDA_CLANG:-0} else