Fix and enable vcpkg for all dependencies (#4055)

Fix and enable vcpkg build for all supported dependencies.

---
TYPE: BUILD
DESC: Fix and enable vcpkg for all dependencies

.dockerignore

@@ -1,4 +1,5 @@
 build
 builds
+**/buildtrees/*
 scripts
 !scripts/capnpupdate.sh

.github/workflows/build-windows.yml

@@ -11,7 +11,7 @@ env:
   TILEDB_SERIALIZATION: OFF
   TILEDB_STATIC: OFF
   TILEDB_TOOLS: ON
-  TILEDB_BASE_BOOTSTRAP_OPTIONS: '-EnableVerbose -EnableStaticTileDB -EnableBuildDeps'
+  TILEDB_BASE_BOOTSTRAP_OPTIONS: '-EnableVcpkg -EnableVerbose -EnableStaticTileDB'
 
 jobs:
   build:
@@ -41,6 +41,7 @@ jobs:
             TILEDB_AZURE: OFF
             TILEDB_ARROW_TESTS: OFF
             TILEDB_SERIALIZATION: ON
+            TILEDB_WEBP: ON
     env:
       TILEDB_HOME: ${{ github.workspace }}
       TILEDB_GA_IMAGE_NAME: ${{ matrix.os }}
@@ -49,7 +50,8 @@ jobs:
       TILEDB_SERIALIZATION: ${{ matrix.TILEDB_SERIALIZATION }} #serialization }}
       TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }}
       TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }}
-      TILEDB_CMAKE_BUILD_TYPE: 'RelWithDebInfo'
+      TILEDB_CMAKE_BUILD_TYPE: 'Release'
+      VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
     steps:
       - name: 'tiledb env prep'
         run: |
@@ -66,6 +68,14 @@ jobs:
           write-host "BUILD_BUILDDIRECTORY=$env:BUILD_BUILDDIRECTORY" *>> $env:GITHUB_ENV
           write-host "BUILD_SOURCESDIRECTORY=$env:BUILD_SOURCESDIRECTORY" *>> $env:GITHUB_ENV
 
+      # Configure required environment variables for vcpkg to use
+      # GitHub's Action Cache
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
       - name: 'Print env'
         shell: cmd
         run: |
@@ -99,12 +109,37 @@ jobs:
           if [[ "$TILEDB_ARROW_TESTS" == "ON" ]]; then
             pip install pyarrow pybind11 numpy
           fi
+
       - name: Prepare git
         run: git config --global core.autocrlf false
+
       - uses: actions/checkout@v3
         with:
           submodules: true
           fetch-depth: 0
+
+      - name: Prevent vpckg from building debug variants
+        shell: python
+        run: |
+          import os
+
+          workspace = os.environ["GITHUB_WORKSPACE"]
+          triplets = os.path.join(workspace, "external", "vcpkg", "triplets")
+
+          for path, dnames, fnames in os.walk(triplets):
+            for fname in fnames:
+              fname = os.path.join(path, fname)
+              print(fname)
+              with open(fname, "rb") as handle:
+                contents = handle.read()
+              # If a file is already patched we skip patching it again because
+              # this affects the hashes calculated by vcpkg for caching
+              if b"VCPKG_BUILD_TYPE release" in contents:
+                continue
+              contents += b"\nset(VCPKG_BUILD_TYPE release)\n"
+              with open(fname, "wb") as handle:
+                handle.write(contents)
+
       - name: core tiledb windows build
         run: |
           $ErrorView = "NormalView"
@@ -288,7 +323,7 @@ jobs:
             }
           }
           # Build the examples
-          cmake --build $env:BUILD_BUILDDIRECTORY --target examples --config $CMakeBuildType -- /verbosity:minimal
+          cmake --build $env:BUILD_BUILDDIRECTORY --target examples --config $CMakeBuildType -v
 
           if ($LastExitCode -ne 0) {
             Write-Host "Examples failed to build."
@@ -371,9 +406,9 @@ jobs:
           cd build
 
           # Build zip artifact
-          cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist" ..
+          cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILD_DIRECTORY\vcpkg_installed\x64-windows-mod" ..
 
-          cmake --build . --config $CMakeBuildType -- /verbosity:minimal
+          cmake --build . --config $CMakeBuildType -v
 
           #.\$CMakeBuildType\ExampleExe.exe
           $cmd = ".\$CMakeBuildType\ExampleExe.exe"
@@ -426,3 +461,13 @@ jobs:
             echo TILEDB_CI_SUCCESS != 1, exiting.
             exit 1;
           fi
+
+      - name: "Print log files (failed build only)"
+        shell: bash
+        run: |
+          source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh
+          echo "# ------------------------------------"
+          find $BUILD_BUILDDIRECTORY/dist
+          echo "# --------vcpkg----------------------------"
+          find $BUILD_BUILDDIRECTORY/vcpkg_installed
+        if: ${{ failure() }} # only run this job if the build step failed

.github/workflows/ci-linux_mac.yml

@@ -40,7 +40,7 @@ env:
   TILEDB_CI_OS: runner.os
   CXX: ${{ inputs.matrix_compiler_cxx }}
   CC: ${{ inputs.matrix_compiler_cc }}
-  bootstrap_args: "--enable-ccache ${{ inputs.bootstrap_args }}"
+  bootstrap_args: "--enable-vcpkg --enable-ccache ${{ inputs.bootstrap_args }}"
   VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
   SCCACHE_GHA_ENABLED: "true"
 

.github/workflows/unit-test-runs.yml

@@ -46,6 +46,7 @@ jobs:
           pushd $GITHUB_WORKSPACE/build
 
           cmake .. \
+            -DTILEDB_VCPKG=ON \
             -DTILEDB_AZURE=${TILEDB_AZURE}\
             -DTILEDB_GCS=${TILEDB_GCS} \
             -DTILEDB_S3=${TILEDB_S3} \

CMakeLists.txt

@@ -559,10 +559,6 @@ if (TILEDB_TESTS)
     add_dependencies(tests unit_link_webp)
   endif()
 
-  if (TILEDB_ABSEIL)
-    add_dependencies(tests unit_link_absl)
-  endif()
-
   if (ENABLE_MAGIC_TEST)
     add_dependencies(tests unit_mgc_dict)
   endif()

azure-pipelines.yml

@@ -4,6 +4,7 @@ trigger:
     - dev
     - release-*
     - refs/tags/*
+    - build-
 pr:
   branches:
     include:
@@ -18,7 +19,9 @@ pr:
 
 variables:
   - name: MANYLINUX_IMAGE
-    value: quay.io/pypa/manylinux2014_x86_64:2022-11-06-7be974c
+    #value: quay.io/pypa/manylinux2014_x86_64:2022-11-06-7be974c
+    # modified from above tag to add linux-headers
+    value: ghcr.io/ihnorton/tiledb-manylinux2014_x86_64:2023-04-02
 
 stages:
   - stage: CI
@@ -92,6 +95,7 @@ stages:
              TILEDB_GCS: OFF
              TILEDB_WERROR: OFF
              CMAKE_OSX_ARCHITECTURES: "arm64"
+             BUILD_MAGIC_MACOS_UNIVERSAL: "ON"
              MACOSX_DEPLOYMENT_TARGET: 11
        pool:
          vmImage: $(imageName)

bootstrap

@@ -180,6 +180,10 @@ for en in "${enables[@]}"; do
   esac
 done
 
+if [ "${tiledb_build_abseil}" = "ON" ]; then
+    echo "Argument '--build-abseil' is obsolete and will be removed in TileDB 2.18"
+fi
+
 if [ "${source_dir}" = "${binary_dir}" ]; then
     die "cannot build the project in the source directory! Out-of-source build is enforced!"
 fi
@@ -233,7 +237,6 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \
     -DTILEDB_ARROW_TESTS=${tiledb_arrow_tests} \
     -DTILEDB_CRC32=${tiledb_build_crc32} \
     -DTILEDB_WEBP=${tiledb_build_webp} \
-    -DTILEDB_ABSEIL=${tiledb_build_abseil} \
     -DTILEDB_FORCE_ALL_DEPS=${tiledb_force_all_deps} \
     -DTILEDB_REMOVE_DEPRECATIONS=${tiledb_remove_deprecations} \
     -DSANITIZER="${sanitizer}" \

bootstrap.ps1

@@ -36,6 +36,9 @@ Enable build with code coverage support.
 .PARAMETER EnableVerbose
 Enable verbose status messages.
 
+.PARAMETER EnableVcpkg
+Enables building dependencies with vcpkg.
+
 .PARAMETER EnableAzure
 Enables building with the Azure storage backend.
 
@@ -58,7 +61,7 @@ Enables building TileDB CLI tools (experimental)
 Enables building TileDB Experimental features
 
 .PARAMETER EnableAbseil
-Enables building of Abseil and simple linkage test
+(Obsolete) Enables building of Abseil and simple linkage test
 
 .PARAMETER DisableWebP
 Disables building of WebP and simple linkage test
@@ -96,6 +99,7 @@ Param(
     [switch]$EnableReleaseSymbols,
     [switch]$EnableCoverage,
     [switch]$EnableVerbose,
+    [switch]$EnableVcpkg,
     [switch]$EnableAzure,
     [switch]$EnableS3,
     [switch]$EnableSerialization,
@@ -156,6 +160,12 @@ if ($EnableVerbose.IsPresent) {
     $Verbosity = "ON"
 }
 
+# Set vcpkg flag
+$UseVcpkg = "OFF"
+if ($EnableVcpkg.IsPresent) {
+    $UseVcpkg = "ON"
+}
+
 # Set TileDB Azure flag
 $UseAzure = "OFF"
 if ($EnableAzure.IsPresent) {
@@ -213,9 +223,9 @@ if ($EnableExperimentalFeatures.IsPresent) {
     $TileDBExperimentalFeatures = "ON"
 }
 
-$BuildAbseil="OFF"
 if ($EnableAbseil.IsPresent) {
-  $BuildAbseil="ON"
+  # remove in 2.18
+  Write-Host "EnableAbseil is deprecated and will be removed"
 }
 
 $TileDBBuildDeps = "OFF";
@@ -261,7 +271,7 @@ if ($CMakeGenerator -eq $null) {
 
 # Run CMake.
 # We use Invoke-Expression so we can echo the command to the user.
-$CommandString = "cmake -A X64 -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_ABSEIL=$BuildAbseil -DTILEDB_CRC32=$BuildCrc32 $GeneratorFlag ""$SourceDirectory"""
+$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 $GeneratorFlag ""$SourceDirectory"""
 Write-Host $CommandString
 Write-Host
 Invoke-Expression "$CommandString"

cmake/Modules/FindAWSSDK_EP.cmake

@@ -32,42 +32,22 @@
 include(TileDBCommon)
 
 if(TILEDB_VCPKG)
-  find_package(AWSSDK QUIET CONFIG COMPONENTS core s3 REQUIRED)
-  set(AWS_SERVICES s3)
-  AWSSDK_DETERMINE_LIBS_TO_LINK(AWS_SERVICES AWS_LINKED_LIBS)
-  list(APPEND AWS_LINKED_LIBS aws-c-cal
-                              aws-c-io
-                              aws-cpp-sdk-identity-management
-                              aws-cpp-sdk-sts)
-
-  foreach (LIB ${AWS_LINKED_LIBS})
-    if (NOT ${LIB} MATCHES "aws-*")
-      continue()
-    endif()
-
-    find_library("AWS_FOUND_${LIB}"
-      NAMES ${LIB}
-      PATHS ${AWSSDK_LIB_DIR}
-      NO_DEFAULT_PATH
-    )
-    message(STATUS "Found AWS lib: ${LIB} (${AWS_FOUND_${LIB}})")
-    if (NOT TARGET AWSSDK::${LIB})
-      add_library(AWSSDK::${LIB} UNKNOWN IMPORTED)
-      set_target_properties(AWSSDK::${LIB} PROPERTIES
-        INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}"
-        IMPORTED_LOCATION "${AWS_FOUND_${LIB}}"
-      )
-    endif()
-  endforeach ()
-
-  # Add missing link directives here rather than adding
-  # conditional logic in tiledb/CMakeLists.txt
-  target_link_libraries(AWSSDK::aws-cpp-sdk-s3 INTERFACE AWSSDK::aws-c-cal)
-  target_link_libraries(AWSSDK::aws-cpp-sdk-s3 INTERFACE AWSSDK::aws-c-io)
-
+  set(AWS_SERVICES identity-management sts s3)
+
+  # Provides:  ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}
+  # TODO: We may need to conditionally use ${AWSSDK_PLATFORM_DEPS} here for dynamically linked deps, but
+  #       it lists bare "pthread;curl" which leads to linkage of system versions. For static linkage, we
+  #       handle those elsewhere at the moment.
+  find_package(AWSSDK REQUIRED QUIET COMPONENTS ${AWS_SERVICES})
+  install_all_target_libs("${AWSSDK_LINK_LIBRARIES}")
   return()
 endif()
 
+
+###############################################################################
+# Start superbuild/unmanaged/legacy version
+###############################################################################
+
 ##-----------------------------------
 # early WIN32 audit of path length for aws sdk build where
 # insufficient available path length causes sdk build failure.

cmake/Modules/FindCapnp_EP.cmake

@@ -32,6 +32,13 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
+if (TILEDB_VCPKG AND TILEDB_SERIALIZATION)
+  find_package(CapnProto REQUIRED)
+  install_all_target_libs("CapnProto::capnp;CapnProto::capnp-json;CapnProto::kj")
+  return()
+endif()
+
+
 # If the EP was built, it will install the CapnProtoConfig.cmake file, which we
 # can use with find_package.
 
@@ -65,8 +72,8 @@ if (NOT CAPNP_FOUND)
 
     if (WIN32)
       find_package(Git REQUIRED)
-      set(CONDITIONAL_PATCH 
-           cd ${CMAKE_SOURCE_DIR} && 
+      set(CONDITIONAL_PATCH
+           cd ${CMAKE_SOURCE_DIR} &&
            ${GIT_EXECUTABLE} apply --ignore-whitespace -p1 --unsafe-paths --verbose --directory=${TILEDB_EP_SOURCE_DIR}/ep_capnp < ${TILEDB_CMAKE_INPUTS_DIR}/patches/ep_capnp/capnp_CMakeLists.txt.patch &&
            ${GIT_EXECUTABLE} apply --ignore-whitespace -p1 --unsafe-paths --verbose --directory=${TILEDB_EP_SOURCE_DIR}/ep_capnp < ${TILEDB_CMAKE_INPUTS_DIR}/patches/ep_capnp/windows-sanity.h.patch)
     else()

cmake/Modules/FindCurl_EP.cmake

@@ -24,15 +24,18 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #
-# Finds the Curl library, installing with an ExternalProject as necessary.
-# This module defines:
-#   - CURL_INCLUDE_DIR, directory containing headers
-#   - CURL_LIBRARIES, the Curl library path
-#   - CURL_FOUND, whether Curl has been found
-#   - The Curl::Curl imported target
-
-# Include some common helper functions.
-include(TileDBCommon)
+
+if (TARGET CURL OR NOT (TILEDB_SERIALIZATION OR TILEDB_GCS OR TILEDB_S3 OR TILEDB_AZURE))
+  return()
+endif()
+
+if (TILEDB_VCPKG)
+  find_package(CURL REQUIRED ${TILEDB_DEPS_NO_DEFAULT_PATH})
+  install_target_libs(CURL::libcurl)
+  return()
+endif()
+
+# === Superbuild
 
 # Search the path set during the superbuild for the EP.
 set(CURL_PATHS ${TILEDB_EP_INSTALL_PREFIX})

cmake/Modules/FindMagic_EP.cmake

@@ -34,7 +34,7 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
-if(TILEDB_VCPKG)
+if(FALSE) #NOT WINDOWS AND TILEDB_VCPKG AND NOT TILEDB_VCPKG_DISABLE_MAGIC)
   find_path(libmagic_INCLUDE_DIR NAMES magic.h)
   find_library(libmagic_LIBRARIES magic)
   find_file(libmagic_DICTIONARY magic.mgc
@@ -136,6 +136,7 @@ if(NOT TILEDB_LIBMAGIC_EP_BUILT)
         -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
         "-DCMAKE_C_FLAGS=${CFLAGS_DEF}"
         -Dlibmagic_STATIC_LIB=ON
+        -DBUILD_MAGIC_MACOS_UNIVERSAL=$ENV{BUILD_MAGIC_MACOS_UNIVERSAL}
       LOG_DOWNLOAD TRUE
       LOG_CONFIGURE TRUE
       LOG_BUILD TRUE

cmake/Modules/FindOpenSSL_EP.cmake

@@ -33,6 +33,13 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
+if (TILEDB_VCPKG)
+  find_package(OpenSSL REQUIRED)
+  target_link_libraries(OpenSSL::SSL)
+  install_all_target_libs(OpenSSL::SSL)
+  return()
+endif()
+
 # Search the path set during the superbuild for the EP.
 set(OPENSSL_PATHS ${TILEDB_EP_INSTALL_PREFIX})