Remove the superbuild and the external projects. (#5021)

[SC-36912](https://app.shortcut.com/tiledb-inc/story/36912/remove-cmake-variable-tiledb-vcpkg-from-the-build) [SC-36913](https://app.shortcut.com/tiledb-inc/story/36913/remove-superbuild) Historically, the Core's build system has been using CMake external projects to download and build external dependencies, and a "superbuild" architecture to ensure a build order. With the advent of vcpkg, we have stopped building the dependencies ourselves and instead rely on vcpkg (or the "system" in general) to provide them for us. The superbuild has thus became a relic of the past, consisting of only the inner `tiledb` project when the new system is enabled (formerly by specifying `-DTILEDB_VCPKG=ON`, now always). This PR removes the superbuild. TileDB became a regular CMake project, whose targets can be built directly without first building the outer project, and then building the `build/tiledb` subdirectory. The CI scripts were updated accordingly to not use the subdirectory. This is inevitably a breaking change in the build system. For starters, local development environment will certainly need to make a clean build after this change. Furthermore, there will need to be changes in build scripts to not build again on the `tiledb` subdirectory. For examples of downstream migrations, TileDB-Inc/TileDB-Go#316 uses a `make` invocation that has a similar effect both with and without the superbuild, and conda-forge/tiledb-feedstock#290 uses a semi-documented CMake option to disable the superbuild (which will have no effect after the superbuild gets removed). The majority of first-party downstreams (VCF, SOMA, MariaDB, Vector Search, the Python and Java APIs) use the `install-tiledb` target, which currently is defiend on the superbuild and builds the `install` target in the inner `tiledb` project. With this PR the `install-tiledb` target will be kept for compatibility, but alias to `install`. ~~I tried to build VCF with a TileDB external project from this branch, but it fails with an error that will be fixed with #4989. I will try again once that PR gets merged.~~ Never mind, VCF builds with the latest changes. --- TYPE: BUILD DESC: The superbuild architecture of the build system has been removed and TileDB is a regular CMake project. Build commands of the form `make && make -C tiledb <targets>` will have to be replaced by `make <targets>`.
TileDB-Inc · Jul 17, 2024 · 253d1c7 · 253d1c7
1 parent 78e2185
commit 253d1c7
Show file tree

Hide file tree

Showing 64 changed files with 264 additions and 2,472 deletions.
diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml
@@ -46,7 +46,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: tiledb_unit
-          path: ${{ github.workspace }}/build/tiledb/test/tiledb_unit
+          path: ${{ github.workspace }}/build/test/tiledb_unit
           retention-days: 1
 
   test:
@@ -69,10 +69,10 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: tiledb_unit
-          path: ${{ github.workspace }}/build/tiledb/test/
+          path: ${{ github.workspace }}/build/test/
 
       - name: Update tiledb_unit permissions
-        run: chmod +x $GITHUB_WORKSPACE/build/tiledb/test/tiledb_unit
+        run: chmod +x $GITHUB_WORKSPACE/build/test/tiledb_unit
 
       - name: Free disk space
         run: |
@@ -106,7 +106,7 @@ jobs:
 
 
           # Bypass Catch2 Framework stdout interception with awk on test output
-          $GITHUB_WORKSPACE/build/tiledb/test/tiledb_unit -d yes "[backwards-compat]"| awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          $GITHUB_WORKSPACE/build/test/tiledb_unit -d yes "[backwards-compat]"| awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
       - name: 'Test status check'
         run: |

diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
@@ -242,11 +242,11 @@ jobs:
             # & "$env:BUILD_SOURCESDIRECTORY\scripts\install-minio.ps1"
 
             # update CMake to disable S3 for the test configuration, see minio note above
-            cmake -B $env:BUILD_BUILDDIRECTORY\tiledb -DTILEDB_S3=0 $env:BUILD_SOURCESDIRECTORY
+            cmake -B $env:BUILD_BUILDDIRECTORY -DTILEDB_S3=OFF $env:BUILD_SOURCESDIRECTORY
           }
 
           # CMake exits with non-0 status if there are any warnings during the build, so
-          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit unit_vfs tiledb_regression all_link_complete --config $CMakeBuildType
+          cmake --build $env:BUILD_BUILDDIRECTORY -j --target tiledb_unit unit_vfs tiledb_regression all_link_complete --config $CMakeBuildType
 
           if ($env:TILEDB_AZURE -eq "ON") {
             if($env.TILEDB_USE_CUSTOM_NODE_JS) {
@@ -277,23 +277,23 @@ jobs:
 
           # Actually run tests
 
-          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\tiledb_unit.exe -d=yes"
+          $cmds = "$env:BUILD_BUILDDIRECTORY\test\tiledb_unit.exe -d=yes"
           Write-Host "cmds: '$cmds'"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
              Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde
              $host.SetShouldExit($LastExitCode)
           }
           
-          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\unit_vfs -d=yes"
+          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\sm\filesystem\test\unit_vfs -d=yes"
           Write-Host "cmds: '$cmds'"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
              Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde
              $host.SetShouldExit($LastExitCode)
           }
 
-          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\test_assert.exe -d=yes"
+          $cmds = "$env:BUILD_BUILDDIRECTORY\test\ci\test_assert.exe -d=yes"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
              Write-Host "Tests failed. test_assert exit status: " $LastExitCocde
@@ -313,9 +313,9 @@ jobs:
 
           $env:Path += ";$env:BUILD_BUILDDIRECTORY\dist\bin;$env:BUILD_BUILDDIRECTORY\externals\install\bin"
 
-          $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api")
-          $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\test_app_data")
-          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\") -Filter *.exe |
+          $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "examples\c_api")
+          $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "examples\c_api\test_app_data")
+          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "examples\c_api\") -Filter *.exe |
           Foreach-Object {
             try {
               Set-Location -path $TestAppDir
@@ -346,9 +346,9 @@ jobs:
           Set-Location -path $TestAppDir
           Remove-Item -Path $TestAppDataDir -recurse -force -ErrorAction SilentlyContinue
 
-          $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api")
-          $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\test_app_data")
-          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\") -Filter *.exe |
+          $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "examples\cpp_api")
+          $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "examples\cpp_api\test_app_data")
+          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "examples\cpp_api\") -Filter *.exe |
           Foreach-Object {
             try {
               Set-Location -path $TestAppDir
@@ -405,7 +405,7 @@ jobs:
       - name: Packaging test
         shell: pwsh
         run: |
-          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb --target check-package
+          cmake --build $env:BUILD_BUILDDIRECTORY --target check-package
 
       - name: 'process crashdumps'
         shell: cmd

diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml
@@ -213,10 +213,10 @@ jobs:
           # Run tests
 
           # Bypass Catch2 Framework stdout interception with awk on test output
-          ./tiledb/test/regression/tiledb_regression -d yes
-          ./tiledb/test/ci/test_assert -d yes
-          ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
-          ./tiledb/tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./test/regression/tiledb_regression -d yes
+          ./test/ci/test_assert -d yes
+          ./test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
           
           ctest -R tiledb_timing_unit | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
@@ -244,7 +244,7 @@ jobs:
           ###################################################
           # Perform package test
 
-          cmake --build $GITHUB_WORKSPACE/build/tiledb --target check-package
+          cmake --build $GITHUB_WORKSPACE/build --target check-package
 
       - name: 'Dump core stacks on failure'
         if: ${{ failure() && startsWith(matrix.os, 'ubuntu-') == true }} # only run this job if the build step failed

diff --git a/.github/workflows/mingw-w64-tiledb/PKGBUILD b/.github/workflows/mingw-w64-tiledb/PKGBUILD
@@ -34,24 +34,22 @@ build() {
     -DTILEDB_TESTS=OFF \
     -DTILEDB_S3=ON \
     -DCOMPILER_SUPPORTS_AVX2=OFF \
-    -DTILEDB_SKIP_S3AWSSDK_DIR_LENGTH_CHECK=ON \
     -DTILEDB_WERROR=OFF \
     -DVCPKG_TARGET_TRIPLET=${vcpkg_triplet} \
     ..
   make
-  make -C tiledb
 
 # in a local environ, can build to demo/check for previously seen (windows event) handle leakge
 # apparently due to issues with mingw std library implementation.
 # note that with S3 enabled above, and minio not running, the aws sdk may fatally error on exit.
-#  make -C tiledb/test tiledb_unit # need the support lib built
-#  make -C tiledb/test/performance tiledb_explore_msys_handle_leakage
+#  make -C test tiledb_unit # need the support lib built
+#  make -C test/performance tiledb_explore_msys_handle_leakage
 #  . ../test/performance/msys_handle_leakage/trials.sh
 }
 
 package() {
   cd ${source_dir}/build-${MINGW_CHOST}
-  make DESTDIR="${pkgdir}" -C tiledb install
+  make DESTDIR="${pkgdir}" install
   # Copy vcpkg_installed to the mingw package.
   cp -r vcpkg_installed/${vcpkg_triplet}/. ${pkgdir}/${MINGW_ARCH}
 }

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -125,8 +125,8 @@ jobs:
         with:
           name: release
           path: |
-            build/tiledb/tiledb-*.tar.gz*
-            build/tiledb/tiledb-*.zip*
+            build/tiledb-*.tar.gz*
+            build/tiledb-*.zip*
       - name: "Print log files (failed build only)"
         run: |
           source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh

diff --git a/.github/workflows/test-cloud-e2e.yml b/.github/workflows/test-cloud-e2e.yml
@@ -90,7 +90,7 @@ jobs:
           # Run tests
 
           # Bypass Catch2 Framework stdout interception with awk on test output
-          ./tiledb/test/tiledb_unit -d yes "[vfs-e2e]" | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./test/tiledb_unit -d yes "[vfs-e2e]" | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
       - name: 'Test status check'
         run: |

diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml
@@ -61,14 +61,13 @@ jobs:
             -DCMAKE_BUILD_TYPE=Debug \
             -DTILEDB_SERIALIZATION=ON \
             -DTILEDB_ASSERTIONS=ON
-          cmake --build build -j4
           # Build all unit tests
-          cmake --build build/tiledb --target tests -j4
+          cmake --build build --target tests -j4
 
       - name: 'Run standalone unit tests'
         run: |
-          ctest --test-dir build/tiledb -R '(^unit_|test_assert)' --no-tests=error
-          ctest --test-dir build/tiledb -R 'test_ci_asserts'
+          ctest --test-dir build -R '(^unit_|test_assert)' --no-tests=error
+          ctest --test-dir build -R 'test_ci_asserts'
 
       - name: "Print log files (failed build only)"
         run: |

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -112,56 +112,9 @@ set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 # Disable warnings from Boost
 set(Boost_NO_WARN_NEW_VERSIONS ON)
 
-############################################################
-# Superbuild setup
-############################################################
-
-# Set the variable used when calling find_package(), find_file() etc.
-# to determine if NO_DEFAULT_PATH should be passed.
-if (TILEDB_FORCE_ALL_DEPS)
-  set(TILEDB_DEPS_NO_DEFAULT_PATH NO_DEFAULT_PATH)
-else()
-  set(TILEDB_DEPS_NO_DEFAULT_PATH)
-endif()
-
-# If this is an in-IDE build, we need to disable the superbuild and explicitly
-# set the EP base dir. The normal 'cmake && make' process won't need this step,
-# it is for better CLion support of this superbuild architecture.
-if (TILEDB_CMAKE_IDE)
-  set(TILEDB_SUPERBUILD OFF)
-  set(TILEDB_EP_BASE "${CMAKE_CURRENT_BINARY_DIR}/externals")
-endif()
-
-# Set main TileDB Source
-set(TILEDB_BASE_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/tiledb")
-# Set experimental TileDB Source
-set(TILEDB_EXPERIMENTAL_BASE_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/experimental")
-
-# Perform superbuild config and exit.
-if (TILEDB_SUPERBUILD)
-  project(TileDB-Superbuild)
-
-  # Override default install prefix if not set
-  if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-    message(STATUS "Setting CMAKE_INSTALL_PREFIX to: '${PROJECT_BINARY_DIR}/dist'")
-    set(CMAKE_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/dist" CACHE PATH "..." FORCE)
-  endif()
-
-  message(STATUS "Starting TileDB superbuild.")
-  include("cmake/TileDB-Superbuild.cmake")
-  message(STATUS "Install prefix is ${CMAKE_INSTALL_PREFIX}")
-  # The superbuild file incorporates this file as an external project with the
-  # superbuild off. Thus we stop processing this file here, knowing that we'll
-  # come back later.
-  return()
-endif()
-
 project(TileDB)
 message(STATUS "Starting TileDB regular build.")
 message(STATUS "  CMake version: ${CMAKE_VERSION}")
-# Paths to locate the installed external projects.
-set(TILEDB_EP_SOURCE_DIR "${TILEDB_EP_BASE}/src")
-set(TILEDB_EP_INSTALL_PREFIX "${TILEDB_EP_BASE}/install")
 
 ############################################################
 # Compile options/definitions for all targets
@@ -532,9 +485,6 @@ add_subdirectory(experimental/experimental_examples)
 
 # Build tools
 if (TILEDB_TOOLS)
-  if (TILEDB_AZURE AND NOT TILEDB_VCPKG)
-    message(FATAL_ERROR "Building tools with Azure enabled requires enabling TILEDB_VCPKG")
-  endif()
   add_subdirectory(tools)
 endif()
 

diff --git a/bootstrap b/bootstrap
@@ -39,8 +39,6 @@ Configuration:
                                     ['"${default_dependency}"']
     --linkage                       specify the linkage of tiledb. Defaults to shared.
                                     [static|shared]
-    --force-build-all-deps          force building of all dependencies, even those
-                                    already installed at system-level
     --remove-deprecations           build TileDB without any deprecated APIs
     --disable-werror                disables use of -Werror during build
     --disable-cpp-api               disables building of the TileDB C++ API
@@ -99,7 +97,6 @@ tiledb_gcs="OFF"
 tiledb_werror="ON"
 tiledb_tests="ON"
 tiledb_cpp_api="ON"
-tiledb_force_all_deps="OFF"
 tiledb_remove_deprecations="OFF"
 tiledb_stats="ON"
 build_shared_libs="OFF"
@@ -122,7 +119,7 @@ while test $# != 0; do
     --dependency=*) dir=`arg "$1"`
                 dependency_dir="$dir";;
     --linkage=*) linkage=`arg "$1"`;;
-    --force-build-all-deps) tiledb_force_all_deps="ON";;
+    --force-build-all-deps) echo "Argument '--force-build-all-deps' has no effect and will be removed in a future version. Vcpkg builds all dependencies by default, please consult the guide in doc/dev/BUILD.md or vcpkg's documentation to see how to provide your own dependencies.";;
     --remove-deprecations) tiledb_remove_deprecations="ON";;
     --disable-werror) tiledb_werror="OFF";;
     --disable-tests) tiledb_tests="OFF";;
@@ -246,7 +243,6 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \
     -DTILEDB_CCACHE=${tiledb_ccache} \
     -DTILEDB_ARROW_TESTS=${tiledb_arrow_tests} \
     -DTILEDB_WEBP=${tiledb_build_webp} \
-    -DTILEDB_FORCE_ALL_DEPS=${tiledb_force_all_deps} \
     -DTILEDB_REMOVE_DEPRECATIONS=${tiledb_remove_deprecations} \
     -DTILEDB_SANITIZER="${sanitizer}" \
     -DTILEDB_EXPERIMENTAL_FEATURES=${tiledb_experimental_features} \
@@ -255,4 +251,4 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \
     ${vcpkg_base_triplet} \
     "${source_dir}" || die "failed to configure the project"
 
-echo 'bootstrap success. Run "make" to build, "make check" to test, or "make -C tiledb install" to install.'
+echo 'bootstrap success. Run "make" to build, "make check" to test, or "make install" to install.'
diff --git a/bootstrap.ps1 b/bootstrap.ps1
@@ -71,7 +71,7 @@ Enables building TileDB as a static library.
 Deprecated, use -Linkage static instead.
 
 .PARAMETER EnableBuildDeps
-Enables building TileDB dependencies from source (superbuild)
+Unused, kept for compatibility.
 
 .PARAMETER EnableTools
 Enables building TileDB CLI tools (experimental)
@@ -277,9 +277,8 @@ if ($EnableExperimentalFeatures.IsPresent) {
     $TileDBExperimentalFeatures = "ON"
 }
 
-$TileDBBuildDeps = "OFF";
 if ($EnableBuildDeps.IsPresent) {
-    $TileDBBuildDeps = "ON"
+    Write-Warning "-EnableBuildDeps has no effect and will be removed in a future version. Vcpkg builds all dependencies by default, please consult the guide in doc/dev/BUILD.md or vcpkg's documentation to see how to provide your own dependencies."
 }
 
 $ArrowTests="OFF"
@@ -330,7 +329,7 @@ if ($CMakeGenerator -eq $null) {
 
 # Run CMake.
 # We use Invoke-Expression so we can echo the command to the user.
-$CommandString = "cmake $ArchFlag -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory"""
+$CommandString = "cmake $ArchFlag -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory"""
 Write-Host $CommandString
 Write-Host
 Invoke-Expression "$CommandString"