From fdfdc4dd69a23c19ec9a5cc1711bd32af38134ae Mon Sep 17 00:00:00 2001 From: John Agapiou Date: Thu, 12 Dec 2024 18:31:03 +0000 Subject: [PATCH] Ensure that dmlab2d can still build wheels - add required blaze flags to keep build working in bazel 8 - ensure reproducible builds by pinning version numbers - stop supporting deprecated macos-12 - support new macos-14 (amd64) --- .bazelrc | 2 + .bazelversion | 1 + .github/workflows/make_wheel.yml | 26 ++++++--- .github/workflows/precommit.yml | 42 ++++++++------ WORKSPACE | 95 ++++++++++++++++++-------------- 5 files changed, 101 insertions(+), 65 deletions(-) create mode 100644 .bazelversion diff --git a/.bazelrc b/.bazelrc index f571e99..ef1c65b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,9 +1,11 @@ build --cxxopt=-std=c++17 build --cxxopt=-Wno-sign-compare --cxxopt=-Wno-attributes build --enable_platform_specific_config +build --enable_workspace --noenable_bzlmod build --flag_alias=lua_version=//dmlab2d/lib/lua:lua_version build --flag_alias=macos_target_arch=@luajit_archive//:target_arch build --flag_alias=luajit_external_unwinder=@luajit_archive//:use_external_unwinder +build --incompatible_disallow_empty_glob=false build:libc++ --repo_env=CXXFLAGS=-stdlib=libc++ build:libc++ --repo_env=LDFLAGS=-stdlib=libc++ diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000..c6b7980 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +8.x diff --git a/.github/workflows/make_wheel.yml b/.github/workflows/make_wheel.yml index 412365c..6326b39 100644 --- a/.github/workflows/make_wheel.yml +++ b/.github/workflows/make_wheel.yml @@ -32,29 +32,30 @@ jobs: prerelease: false build-wheel: - name: Build-wheel ${{ matrix.cfg.name }} for Python ${{ matrix.py.version }} + name: Build-wheel ${{ matrix.cfg.name }} for Python ${{ matrix.python-version }} runs-on: ${{ matrix.cfg.os }} needs: create-release strategy: + fail-fast: false matrix: cfg: - { name: 'ManyLinux 2.35 LLVM+libstdc++', os: 'ubuntu-22.04', cc: clang, cxx: clang++, config: --linkopt=-fuse-ld=lld } - { name: 'ManyLinux 2.31 LLVM+libstdc++', os: 'ubuntu-20.04', cc: clang, cxx: clang++, config: --linkopt=-fuse-ld=lld } - - { name: 'MacOS 11 x86_64 LLVM+libc++', os: 'macos-11', cc: clang, cxx: clang++, config: --config=libc++ --config=macos } - - { name: 'MacOS 12 x86_64 LLVM+libc++', os: 'macos-12', cc: clang, cxx: clang++, config: --config=libc++ --config=macos } - { name: 'MacOS 13 x86_64 LLVM+libc++', os: 'macos-13', cc: clang, cxx: clang++, config: --config=libc++ --config=macos } - - { name: 'MacOS 12 ARM64 LLVM+libc++', os: 'macos-12', cc: clang, cxx: clang++, - config: --config=libc++ --config=macos_arm64 --repo_env=PY_PLATFORM_OVERRIDE=macosx_12_0_arm64 } - { name: 'MacOS 13 ARM64 LLVM+libc++', os: 'macos-13', cc: clang, cxx: clang++, config: --config=libc++ --config=macos_arm64 --repo_env=PY_PLATFORM_OVERRIDE=macosx_13_0_arm64 } + - { name: 'MacOS 14 ARM64 LLVM+libc++', os: 'macos-14', cc: clang, cxx: clang++, + config: --config=libc++ --config=macos_arm64 --repo_env=PY_PLATFORM_OVERRIDE=macosx_14_0_arm64 } python-version: - '3.8' - '3.9' - '3.10' - '3.11' + - '3.12' + - '3.13' env: CC: ${{ matrix.cfg.cc }} @@ -69,7 +70,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Python Dependencies - run: pip3 install --upgrade pip packaging check-wheel-contents + run: | + pip3 install --upgrade setuptools packaging + which python3 + python3 --version + pip3 list + bazel --version - name: Build for Python ${{ matrix.python-version }} run: bazel --bazelrc=.bazelrc build --compilation_mode=opt --dynamic_mode=off --config=luajit ${{ matrix.cfg.config }} //dmlab2d:dmlab2d_wheel @@ -81,10 +87,12 @@ jobs: echo WHEEL_NAME="${WHEEL_NAME}" >> "${GITHUB_ENV}" - name: Check wheel contents - run: check-wheel-contents bazel-bin/dmlab2d/${{ env.WHEEL_NAME }} + run: | + pip3 install check-wheel-contents + check-wheel-contents bazel-bin/dmlab2d/${{ env.WHEEL_NAME }} - - name: Test wheel - if: (!contains(env.WHEEL_NAME, 'arm64')) + - name: Test wheel ${{ env.WHEEL_NAME }} + if: matrix.cfg.os != 'macos-13' && matrix.cfg.config != 'macos_arm64' run: | pip install bazel-bin/dmlab2d/${{ env.WHEEL_NAME }} python -I - <<'____HERE' diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 3952ef1..97e4165 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -16,46 +16,56 @@ concurrency: jobs: build-and-test: - name: Build+test ${{ matrix.name }} + name: Build+test ${{ matrix.cfg.name }} - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.cfg.os }} timeout-minutes: 30 strategy: fail-fast: ${{ github.event_name != 'workflow_dispatch' }} matrix: - include: + cfg: - { name: 'Linux GCC', os: 'ubuntu-22.04', cc: gcc-12, cxx: g++-12, config: } - { name: 'Linux LLVM+libstdc++', os: 'ubuntu-22.04', cc: clang, cxx: clang++, config: --linkopt=-fuse-ld=lld } - { name: 'Linux LLVM+libc++', os: 'ubuntu-22.04', cc: clang, cxx: clang++, config: --config=libc++ --linkopt=-fuse-ld=lld } - - { name: 'MacOS 12 x86_64 LLVM+libc++', os: 'macos-12', cc: clang, cxx: clang++, config: --config=libc++ --config=macos } - { name: 'MacOS 13 x86_64 LLVM+libc++', os: 'macos-13', cc: clang, cxx: clang++, config: --config=libc++ --config=macos } - - { name: 'MacOS 12 ARM64 LLVM+libc++', os: 'macos-12', cc: clang, cxx: clang++, config: --config=libc++ --config=macos_arm64 } - { name: 'MacOS 13 ARM64 LLVM+libc++', os: 'macos-13', cc: clang, cxx: clang++, config: --config=libc++ --config=macos_arm64 } + - { name: 'MacOS 14 ARM64 LLVM+libc++', os: 'macos-14', cc: clang, cxx: clang++, config: --config=libc++ --config=macos_arm64 } + python-version: + - '3.10' env: - CC: ${{ matrix.cc }} - CXX: ${{ matrix.cxx }} + CC: ${{ matrix.cfg.cc }} + CXX: ${{ matrix.cfg.cxx }} steps: - uses: actions/checkout@v3 - name: install infrastructure (Linux) - if: matrix.os == 'ubuntu-22.04' + if: startsWith( matrix.cfg.os, 'ubuntu-' ) run: | - sudo apt-get install python3-dev python3-numpy python3-packaging sudo ln -s /usr/lib/llvm-10/include/c++/v1 /usr/include/c++/v1 - - name: install infrastructure (MacOS) - if: matrix.os == 'macos-12' || matrix.os == 'macos-13' - run: sudo -H pip3 install numpy packaging + - name: Set up Python ${{ matrix.python-version }} + id: setup-python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies + run: | + pip3 install numpy packaging + which python3 + python3 --version + pip3 list + bazel --version - name: build-lua5.1 run: | bazel --bazelrc=.bazelrc build --config=lua5_1 ${{ matrix.config }} //... - name: test-lua5.1 - if: (!contains(matrix.config, 'arm64')) + if: matrix.cfg.os != 'macos-13' && matrix.cfg.config != 'macos_arm64' run: | bazel --bazelrc=.bazelrc test --config=lua5_1 ${{ matrix.config }} --test_output=errors //... @@ -63,7 +73,7 @@ jobs: run: | bazel --bazelrc=.bazelrc build --config=lua5_2 ${{ matrix.config }} //... - name: test-lua5.2 - if: (!contains(matrix.config, 'arm64')) + if: matrix.cfg.os != 'macos-13' && matrix.cfg.config != 'macos_arm64' run: | bazel --bazelrc=.bazelrc test --config=lua5_2 ${{ matrix.config }} --test_output=errors //... @@ -71,7 +81,7 @@ jobs: run: | bazel --bazelrc=.bazelrc build --config=luajit ${{ matrix.config }} //... - name: test-luajit - if: (!contains(matrix.config, 'arm64')) + if: matrix.cfg.os != 'macos-13' && matrix.cfg.config != 'macos_arm64' run: | bazel --bazelrc=.bazelrc test --config=luajit ${{ matrix.config }} --test_output=errors //... @@ -79,6 +89,6 @@ jobs: run: | bazel --bazelrc=.bazelrc build --config=luajit ${{ matrix.config }} --copt=-fno-asynchronous-unwind-tables --luajit_external_unwinder=False //... - name: test-luajit-internal_unwind - if: (!contains(matrix.config, 'arm64')) + if: matrix.cfg.os != 'macos-13' && matrix.cfg.config != 'macos_arm64' run: | bazel --bazelrc=.bazelrc test --config=luajit ${{ matrix.config }} --copt=-fno-asynchronous-unwind-tables --luajit_external_unwinder=False --test_output=errors //... diff --git a/WORKSPACE b/WORKSPACE index 52749bf..bd69a23 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,80 +1,86 @@ workspace(name = "org_deepmind_lab2d") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@//:python_system.bzl", "python_repo") http_archive( name = "com_google_googletest", - strip_prefix = "googletest-main", - urls = ["https://github.com/google/googletest/archive/main.zip"], + sha256 = "f179ec217f9b3b3f3c6e8b02d3e7eda997b49e4ce26d6b235c9053bec9c0bf9f", + strip_prefix = "googletest-1.15.2", + urls = ["https://github.com/google/googletest/archive/refs/tags/v1.15.2.zip"], ) http_archive( name = "com_google_benchmark", - strip_prefix = "benchmark-main", - urls = ["https://github.com/google/benchmark/archive/main.zip"], + sha256 = "8a63c9c6adf9e7ce8d0d81f251c47de83efb5e077e147d109fa2045daac8368b", + strip_prefix = "benchmark-1.9.1", + urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.9.1.zip"], ) http_archive( name = "rules_cc", - strip_prefix = "rules_cc-main", - urls = ["https://github.com/bazelbuild/rules_cc/archive/main.zip"], + sha256 = "e7c1639aa227d2cd335825d37caa2a234b40373be3ef8f87c25724ed01cab5e8", + strip_prefix = "rules_cc-0.0.17", + urls = ["https://github.com/bazelbuild/rules_cc/archive/refs/tags/0.0.17.zip"], ) http_archive( name = "rules_python", - strip_prefix = "rules_python-main", - url = "https://github.com/bazelbuild/rules_python/archive/main.zip", + sha256 = "7149fff45f7925bb6d45543ed99bfd3544ec63df82ef70cf0ce38b669c9d3bd6", + strip_prefix = "rules_python-1.0.0", + urls = ["https://github.com/bazelbuild/rules_python/archive/refs/tags/1.0.0.zip"], ) http_archive( name = "bazel_skylib", - strip_prefix = "bazel-skylib-main", - urls = ["https://github.com/bazelbuild/bazel-skylib/archive/main.zip"], + sha256 = "23ef380aa62e2f1b631d02d0c9d163e43be03e880953dafe133d754fb8d96f0a", + strip_prefix = "bazel-skylib-1.7.1", + urls = ["https://github.com/bazelbuild/bazel-skylib/archive/refs/tags/1.7.1.zip"], ) http_archive( name = "com_google_absl", - strip_prefix = "abseil-cpp-master", - urls = ["https://github.com/abseil/abseil-cpp/archive/master.zip"], + sha256 = "95e90be7c3643e658670e0dd3c1b27092349c34b632c6e795686355f67eca89f", + strip_prefix = "abseil-cpp-20240722.0", + urls = ["https://github.com/abseil/abseil-cpp/archive/refs/tags/20240722.0.zip"], ) http_archive( name = "com_google_absl_py", - strip_prefix = "abseil-py-main", - urls = ["https://github.com/abseil/abseil-py/archive/main.zip"], + sha256 = "e11083ee5a69afe5c82321ec2505061407ec315789b9b886cbfaf46560431c0b", + strip_prefix = "abseil-py-2.1.0", + urls = ["https://github.com/abseil/abseil-py/archive/refs/tags/v2.1.0.zip"], ) http_archive( name = "eigen_archive", build_file = "@//bazel:eigen.BUILD", - sha256 = "515b3c266d798f3a112efe781dda0cf1aef7bd73f6864d8f4f16129310ae1fdf", - strip_prefix = "eigen-b02c384ef4e8eba7b8bdef16f9dc6f8f4d6a6b2b", + sha256 = "8586084f71f9bde545ee7fa6d00288b264a2b7ac3607b974e54d13e7162c1c72", + strip_prefix = "eigen-3.4.0", urls = [ - "https://gitlab.com/libeigen/eigen/-/archive/b02c384ef4e8eba7b8bdef16f9dc6f8f4d6a6b2b/eigen-b02c384ef4e8eba7b8bdef16f9dc6f8f4d6a6b2b.tar.gz", - "https://storage.googleapis.com/mirror.tensorflow.org/gitlab.com/libeigen/eigen/-/archive/b02c384ef4e8eba7b8bdef16f9dc6f8f4d6a6b2b/eigen-b02c384ef4e8eba7b8bdef16f9dc6f8f4d6a6b2b.tar.gz", + "https://mirror.bazel.build/gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz", + "https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz", ], ) http_archive( name = "png_archive", build_file = "@//bazel:png.BUILD", - sha256 = "c2c50c13a727af73ecd3fc0167d78592cf5e0bca9611058ca414b6493339c784", - strip_prefix = "libpng-1.6.37", + sha256 = "0b35b4bbdf454d589bf8195bc281fefecc4b2529b42ddfbe8b6c108b986841f9", + strip_prefix = "libpng-1.6.44", urls = [ - "https://mirror.bazel.build/github.com/glennrp/libpng/archive/v1.6.37.zip", - "https://github.com/glennrp/libpng/archive/v1.6.37.zip", + "https://mirror.bazel.build/github.com/pnggroup/libpng/archive/v1.6.44.zip", + "https://github.com/pnggroup/libpng/archive/refs/tags/v1.6.44.zip", ], ) http_archive( name = "zlib_archive", build_file = "@//bazel:zlib.BUILD", - sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", - strip_prefix = "zlib-1.2.11", + sha256 = "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23", + strip_prefix = "zlib-1.3.1", urls = [ - "https://mirror.bazel.build/zlib.net/zlib-1.2.11.tar.gz", - "https://zlib.net/zlib-1.2.11.tar.gz", + "https://mirror.bazel.build/zlib.net/zlib-1.3.1.tar.gz", + "https://zlib.net/zlib-1.3.1.tar.gz", ], ) @@ -92,6 +98,7 @@ http_archive( http_archive( name = "lua5_2_archive", build_file = "@//bazel:lua5_2.BUILD", + sha256 = "b9e2e4aad6789b3b63a056d442f7b39f0ecfca3ae0f1fc0ae4e9614401b69f4b", strip_prefix = "lua-5.2.4/src", urls = [ "https://mirror.bazel.build/www.lua.org/ftp/lua-5.2.4.tar.gz", @@ -99,50 +106,58 @@ http_archive( ], ) -LJ_VER="f0ff869bc2fffa17bb765c4c773457578da125a9" - http_archive( name = "luajit_archive", build_file = "@//bazel:luajit.BUILD", - strip_prefix = "LuaJIT-{}".format(LJ_VER), - urls = ["https://github.com/LuaJIT/LuaJIT/archive/{}.tar.gz".format(LJ_VER)], + sha256 = "4af5d94608a7ab72c9a7f72833a9222a7e0a298255e058902c4a59012c53bbfc", + strip_prefix = "LuaJIT-f0ff869bc2fffa17bb765c4c773457578da125a9", + urls = ["https://github.com/LuaJIT/LuaJIT/archive/f0ff869bc2fffa17bb765c4c773457578da125a9.tar.gz"], ) http_archive( name = "dm_env_archive", build_file = "@//bazel:dm_env.BUILD", - strip_prefix = "dm_env-3c6844db2aa4ed5994b2c45dbfd9f31ad948fbb8", - urls = ["https://github.com/deepmind/dm_env/archive/3c6844db2aa4ed5994b2c45dbfd9f31ad948fbb8.zip"], + sha256 = "ed832e301bfddb6e98812567ee0217f6016c862357a0af2a94088ac6e6224af6", + strip_prefix = "dm_env-ea2a17bed7a60c82f637891c53f3a37f70d91486", + urls = ["https://github.com/deepmind/dm_env/archive/ea2a17bed7a60c82f637891c53f3a37f70d91486.zip"], ) http_archive( name = "tree_archive", build_file = "@//bazel:tree.BUILD", - strip_prefix = "tree-master", - urls = ["https://github.com/deepmind/tree/archive/master.zip"], + sha256 = "407606263ee3f049da5c49071522cf5d37142c2de04ba52c9d599da948f4ee08", + strip_prefix = "tree-0.1.8", + urls = ["https://github.com/google-deepmind/tree/archive/refs/tags/0.1.8.zip"], ) http_archive( name = "pybind11_archive", build_file = "@//bazel:pybind11.BUILD", - strip_prefix = "pybind11-master", - urls = ["https://github.com/pybind/pybind11/archive/master.zip"], + sha256 = "d0a116e91f64a4a2d8fb7590c34242df92258a61ec644b79127951e821b47be6", + strip_prefix = "pybind11-2.13.6", + urls = ["https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.zip"], ) http_archive( name = "build_bazel_apple_support", - sha256 = "9f7bb62c3ae889e0eae8c18458fd8764e2e537687d9a1d85885d6af980e4fc31", - url = "https://github.com/bazelbuild/apple_support/releases/download/1.6.0/apple_support.1.6.0.tar.gz", + sha256 = "b53f6491e742549f13866628ddffcc75d1f3b2d6987dc4f14a16b242113c890b", + urls = ["https://github.com/bazelbuild/apple_support/releases/download/1.17.1/apple_support.1.17.1.tar.gz"], ) load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") apple_support_dependencies() -load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") +load("@bazel_features//:deps.bzl", "bazel_features_deps") + +bazel_features_deps() + +load("@rules_python//python:repositories.bzl", "py_repositories") py_repositories() +load("@//:python_system.bzl", "python_repo") + python_repo( name = "python_system", )