Internal Change

PiperOrigin-RevId: 566561418

WORKSPACE

@@ -11,12 +11,17 @@ workspace(name = "yggdrasil_decision_forests")
 
 # Load the dependencies of YDF.
 load("//yggdrasil_decision_forests:library.bzl", ydf_load_deps = "load_dependencies")
-ydf_load_deps(exclude_repo="tensorflow")
+ydf_load_deps(exclude_repo=["tensorflow", "protobuf"])
 
 # Load the dependencies of YDF that cannot be loaded above.
-load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
-protobuf_deps()
+# Protobuf is loaded from pybind11_protobuf or Tensorflow. 
+# TODO: Unify these dependencies.
+# load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+# protobuf_deps()
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
 grpc_deps()
 load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
 grpc_extra_deps()
+
+load("@pybind11_bazel//:python_configure.bzl", "python_configure")
+python_configure(name = "local_config_python")

WORKSPACE_NO_TF

@@ -7,20 +7,23 @@
 
 workspace(name = "yggdrasil_decision_forests")
 
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
 # Load the dependencies of YDF.
 load("//yggdrasil_decision_forests:library.bzl", ydf_load_deps = "load_dependencies")
-ydf_load_deps(exclude_repo="tensorflow")
+ydf_load_deps(exclude_repo=["tensorflow", "protobuf"])
 
 # Load the dependencies of YDF that cannot be loaded above.
-load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
-protobuf_deps()
+# Protobuf is loaded from pybind11_protobuf or Tensorflow. 
+# TODO: Unify these dependencies.
+# load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+# protobuf_deps()
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
 grpc_deps()
 load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
 grpc_extra_deps()
 
+load("@pybind11_bazel//:python_configure.bzl", "python_configure")
+python_configure(name = "local_config_python")
+
 # Emscripten
 http_archive(
   name = "emsdk",
@@ -32,3 +35,6 @@ load("@emsdk//:deps.bzl", emsdk_deps = "deps")
 emsdk_deps()
 load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps")
 emsdk_emscripten_deps()
+
+load("@pybind11_bazel//:python_configure.bzl", "python_configure")
+python_configure(name = "local_config_python")

WORKSPACE_WITH_TF

@@ -65,3 +65,7 @@ load("@emsdk//:deps.bzl", emsdk_deps = "deps")
 emsdk_deps()
 load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps")
 emsdk_emscripten_deps()
+
+load("@pybind11_bazel//:python_configure.bzl", "python_configure")
+
+python_configure(name = "local_config_python")

third_party/pybind11/BUILD

@@ -0,0 +1 @@
+licenses(["notice"])

third_party/pybind11/workspace.bzl

@@ -0,0 +1,20 @@
+"""Pybind project."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def deps():
+    # Version 2.11.1
+    PYBIND_BAZEL_COMMIT_HASH = "fd7d88857cca3d7435b06f3ac6abab77cd9983b2"
+    PYBIND_COMMIT_HASH = "8a099e44b3d5f85b20f05828d919d2332a8de841"
+    http_archive(
+        name = "pybind11_bazel",
+        strip_prefix = "pybind11_bazel-{commit}".format(commit = PYBIND_BAZEL_COMMIT_HASH),
+        urls = ["https://github.com/pybind/pybind11_bazel/archive/{commit}.tar.gz".format(commit = PYBIND_BAZEL_COMMIT_HASH)],
+    )
+
+    http_archive(
+        name = "pybind11",
+        build_file = "@pybind11_bazel//:pybind11.BUILD",
+        strip_prefix = "pybind11_bazel-{commit}".format(commit = PYBIND_COMMIT_HASH),
+        urls = ["https://github.com/pybind/pybind11/archive/{commit}.tar.gz".format(commit = PYBIND_COMMIT_HASH)],
+    )

third_party/pybind11_abseil/BUILD

@@ -0,0 +1 @@
+licenses(["notice"])

third_party/pybind11_abseil/workspace.bzl

@@ -0,0 +1,11 @@
+"""Pybind absl wrappers project."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def deps():
+    PYBIND_ABSL_COMMIT_HASH = "fcfff8502fad281b0c1197872a1e30cdab69a323"
+    http_archive(
+        name = "com_google_pybind11_abseil",
+        urls = ["https://github.com/pybind/pybind11_abseil/archive/{commit}.tar.gz".format(commit = PYBIND_ABSL_COMMIT_HASH)],
+        strip_prefix = "pybind11_abseil-{commit}".format(commit = PYBIND_ABSL_COMMIT_HASH),
+    )

third_party/pybind11_protobuf/BUILD

@@ -0,0 +1 @@
+licenses(["notice"])

third_party/pybind11_protobuf/workspace.bzl

@@ -0,0 +1,14 @@
+"""Pybind Protobuf project."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+# TODO: Update to latest commit.
+def deps():
+    http_archive(
+        name = "com_google_pybind11_protobuf",
+        sha256 = "2e1fa89c6afa23f65f5ff9835d195142a8a059bcdb96eaa3c367a195cb183f26",
+        strip_prefix = "pybind11_protobuf-10fc4b0affe36b98b5e291008ef59f36637a1f1b",
+        urls = [
+            "https://github.com/pybind/pybind11_protobuf/archive/10fc4b0affe36b98b5e291008ef59f36637a1f1b.tar.gz",
+        ],
+    )

tools/build_binary_release.sh

@@ -39,10 +39,11 @@ apt-get -y --no-install-recommends install \
   ca-certificates \
   build-essential \
   g++-10 \
-  clang-10 \
+  clang-12 \
   git \
   python3 \
   python3-pip \
+  python3-dev \
   zip \
   wget
 
@@ -59,7 +60,7 @@ if [[ ! -z ${BUILD+z} ]]; then
   cp -f WORKSPACE_WITH_TF WORKSPACE
 
   BAZEL="./bazelisk"
-  FLAGS="--config=linux_cpp17 --config=linux_avx2 --features=-fully_static_link --config=use_tensorflow_io --repo_env=CC=clang-10"
+  FLAGS="--config=linux_cpp17 --config=linux_avx2 --features=-fully_static_link --config=use_tensorflow_io --repo_env=CC=clang-12"
   ${BAZEL} build //yggdrasil_decision_forests/cli/...:all \
     //yggdrasil_decision_forests/utils/distribute/implementations/grpc:grpc_worker_main ${FLAGS}
 

tools/start_build_docker.sh

@@ -17,7 +17,7 @@
 
 set -vex
 
-DOCKER=ubuntu:18.04
+DOCKER=ubuntu:20.04
 
 # Current directory
 DIRNAME=${PWD##*/}

tools/test_bazel.sh

@@ -54,21 +54,26 @@ build_and_maybe_test () {
     # Not all tests can be run without TF support
     local testable_components=""
     local buildable_cli_components=""
+    local python_components=""
     if [ "$3" = 0 ]; then
       # No tensorflow support
       cp -f WORKSPACE_NO_TF WORKSPACE
       buildable_cli_components=":all"
       testable_components="metric/...:all"
+      # Include PYDF components
+      python_components="//yggdrasil_decision_forests/port/python/...:all"
     else 
       cp -f WORKSPACE_WITH_TF WORKSPACE
       flags="${flags} --config=use_tensorflow_io"
       buildable_cli_components="/...:all"
       testable_components="...:all"
+      # Exclude PYDF components.
+      python_components="-//yggdrasil_decision_forests/port/python/...:all"
     fi
 
-    time ${BAZEL} build //yggdrasil_decision_forests/cli${buildable_cli_components}  //examples:beginner_cc ${flags}
+    time ${BAZEL} build //yggdrasil_decision_forests/cli${buildable_cli_components} ${python_components} //examples:beginner_cc ${flags}
     if [ "$RUN_TESTS" = 1 ]; then
-      time ${BAZEL} test //yggdrasil_decision_forests/${testable_components}  //examples:beginner_cc ${flags}
+      time ${BAZEL} test //yggdrasil_decision_forests/${testable_components} ${python_components} //examples:beginner_cc ${flags}
     fi
     echo "Building and maybe testing YDF complete."
 } 

yggdrasil_decision_forests/library.bzl

@@ -9,8 +9,20 @@ load("//third_party/boost:workspace.bzl", boost = "deps")
 load("//third_party/grpc:workspace.bzl", grpc = "deps")
 load("//third_party/rapidjson:workspace.bzl", rapidjson = "deps")
 load("//third_party/eigen3:workspace.bzl", eigen = "deps")
+load("//third_party/pybind11:workspace.bzl", pybind11 = "deps")
+load("//third_party/pybind11_abseil:workspace.bzl", pybind11_abseil = "deps")
+load("//third_party/pybind11_protobuf:workspace.bzl", pybind11_protobuf = "deps")
 
 def load_dependencies(repo_name = "", exclude_repo = []):
+    if "pybind11" not in exclude_repo:
+        pybind11()
+
+    if "pybind11_abseil" not in exclude_repo:
+        pybind11_abseil()
+
+    if "pybind11_protobuf" not in exclude_repo:
+        pybind11_protobuf()
+
     if "gtest" not in exclude_repo:
         gtest()
 
@@ -40,23 +52,3 @@ def load_dependencies(repo_name = "", exclude_repo = []):
 
     if "eigen" not in exclude_repo:
         eigen()
-
-# Placeholder to use until bazel supports py_strict_binary.
-def py_strict_binary(name, **kwargs):
-    native.py_binary(name = name, **kwargs)
-
-# Placeholder to use until bazel supports py_strict_library.
-def py_strict_library(name, **kwargs):
-    native.py_library(name = name, **kwargs)
-
-# Placeholder to use until bazel supports pytype_strict_binary.
-def pytype_strict_binary(name, **kwargs):
-    native.py_binary(name = name, **kwargs)
-
-# Placeholder to use until bazel supports pytype_strict_library.
-def pytype_strict_library(name, **kwargs):
-    native.py_library(name = name, **kwargs)
-
-# Placeholder to use until bazel supports pytype_library.
-def pytype_library(name, **kwargs):
-    native.py_library(name = name, **kwargs)