Use vcpkg to build macOS packages (#465)

apache · Dec 30, 2024 · 1e8e4d2 · 1e8e4d2
1 parent bdf6854
commit 1e8e4d2
Show file tree

Hide file tree

Showing 7 changed files with 109 additions and 212 deletions.
diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml
@@ -208,6 +208,9 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: recursive
 
       - name: Install dependencies
         run: |

diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml
@@ -311,54 +311,20 @@ jobs:
         run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest
 
   cpp-build-macos:
-    timeout-minutes: 120
-    name: Build CPP Client on macOS
-    needs: formatting-check
-    runs-on: macos-14
-    steps:
-      - name: checkout
-        uses: actions/checkout@v3
-
-      - name: Install dependencies
-        run: brew install openssl protobuf boost zstd snappy googletest
-
-      - name: Configure (default)
-        shell: bash
-        run: cmake -B ./build-macos -S .
-
-      - name: Compile
-        shell: bash
-        run: |
-          cmake --build ./build-macos --parallel --config Release
-
-      - name: Build with C++20
-        shell: bash
-        run: |
-          cmake -B build-macos-cpp20 -DCMAKE_CXX_STANDARD=20
-          cmake --build build-macos-cpp20 -j8
-
-  cpp-build-macos-static:
     timeout-minutes: 120
     name: Build CPP Client on macOS with static dependencies
     runs-on: macos-14
-    needs: unit-tests
+    needs: formatting-check
     steps:
       - name: checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: recursive
 
       - name: Build libraries
         run: ./pkg/mac/build-static-library.sh
 
-      - name: Test static libraries
-        run: |
-          export PULSAR_DIR=$PWD/pkg/mac/.install
-          echo "Build with static library"
-          clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a
-          ./static.out
-          echo "Build with dynamic library"
-          clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar
-          ./dynamic.out
-
   # Job that will be required to complete and depends on all the other jobs
   check-completion:
     name: Check Completion

diff --git a/build-support/merge_archives_vcpkg.sh b/build-support/merge_archives_vcpkg.sh
@@ -27,6 +27,12 @@ if [[ $# -lt 1 ]]; then
 fi
 
 CMAKE_BUILD_DIRECTORY=$1
-./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \
-    $CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \
-    $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug)
+if [[ $VCPKG_TRIPLET ]]; then
+    ./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \
+        $CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \
+        $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed/$VCPKG_TRIPLET" -name "*.a" | grep -v debug)
+else
+    ./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \
+        $CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \
+        $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug)
+fi
diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh
@@ -18,182 +18,59 @@
 # under the License.
 #
 
-set -ex
-cd `dirname $0`
+set -e
+cd `dirname $0`/../..
 
-python3 -m venv venv
-source venv/bin/activate
-python3 -m pip install pyyaml
-
-MACOSX_DEPLOYMENT_TARGET=10.15
-if [[ -z ${ARCH} ]]; then
-    ARCH=`uname -m`
-fi
-
-BUILD_DIR=$PWD/.build
-INSTALL_DIR=$PWD/.install
-PREFIX=$BUILD_DIR/install
-mkdir -p $BUILD_DIR
-cp -f ../../build-support/dep-version.py $BUILD_DIR/
-cp -f ../../dependencies.yaml $BUILD_DIR/
-
-pushd $BUILD_DIR
-
-BOOST_VERSION=$(./dep-version.py boost)
-ZLIB_VERSION=$(./dep-version.py zlib)
-OPENSSL_VERSION=$(./dep-version.py openssl)
-PROTOBUF_VERSION=$(./dep-version.py protobuf)
-ZSTD_VERSION=$(./dep-version.py zstd)
-SNAPPY_VERSION=$(./dep-version.py snappy)
-CURL_VERSION=$(./dep-version.py curl)
-
-if [ ! -f boost/.done ]; then
-    echo "Building Boost $BOOST_VERSION"
-    curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz
-    tar zxf boost-${BOOST_VERSION}.tar.gz
-    mkdir -p $PREFIX/include
-    pushd boost-${BOOST_VERSION}
-      ./bootstrap.sh
-      ./b2 headers
-      cp -rf boost $PREFIX/include/
-    popd
-    mkdir -p boost
-    touch boost/.done
-else
-    echo "Using cached Boost $BOOST_VERSION"
+if [[ -z $ARCH ]]; then
+    ARCH=$(uname -m)
+    echo "Use default ARCH: $ARCH"
 fi
-
-if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then
-    echo "Building ZLib $ZLIB_VERSION"
-    curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz
-    tar zxf zlib-${ZLIB_VERSION}.tar.gz
-    pushd zlib-$ZLIB_VERSION
-      CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
-      make -j16
-      make install
-      touch .done
-    popd
+if [[ $ARCH == "x86_64" ]]; then
+    export VCPKG_TRIPLET=x64-osx
+elif [[ $ARCH == "arm64" ]]; then
+    export VCPKG_TRIPLET=arm64-osx
 else
-    echo "Using cached ZLib $ZLIB_VERSION"
+    echo "Invalid ARCH: $ARCH"
+    exit 1
 fi
 
-OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
-if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then
-    echo "Building OpenSSL $OPENSSL_VERSION"
-    curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz
-    tar zxf OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz
-
-    pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
-      if [[ $ARCH = 'arm64' ]]; then
-          PLATFORM=darwin64-arm64-cc
-      else
-          PLATFORM=darwin64-x86_64-cc
-      fi
-      CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
-          ./Configure --prefix=$PREFIX no-shared no-unit-test $PLATFORM
-      make -j8 >/dev/null
-      make install_sw >/dev/null
-    popd
-
-    touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done
-else
-    echo "Using cached OpenSSL $OPENSSL_VERSION"
-fi
-
-if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then
-    echo "Building Protobuf $PROTOBUF_VERSION"
-    curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
-    tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
-    pushd protobuf-${PROTOBUF_VERSION}
-      pushd cmake/
-        # Build protoc that can run on both x86 and arm architectures
-        cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch x86_64 -arch arm64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
-            -Dprotobuf_BUILD_TESTS=OFF \
-            -DCMAKE_INSTALL_PREFIX=$PREFIX
-        cmake --build build -j16 --target install
-      popd
-
-      # Retain the library for one architecture so that `ar` can work on the library
-      pushd $PREFIX/lib
-        mv libprotobuf.a libprotobuf_universal.a
-        lipo libprotobuf_universal.a -thin ${ARCH} -output libprotobuf.a
-      popd
-      touch .done
-    popd
-else
-    echo "Using cached Protobuf $PROTOBUF_VERSION"
-fi
-
-if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then
-    echo "Building ZStd $ZSTD_VERSION"
-    curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
-    tar zxf zstd-${ZSTD_VERSION}.tar.gz
-    pushd zstd-${ZSTD_VERSION}
-      CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \
-            make -j16 -C lib install-static install-includes
-      touch .done
-    popd
-else
-    echo "Using cached ZStd $ZSTD_VERSION"
-fi
-
-if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then
-    echo "Building Snappy $SNAPPY_VERSION"
-    curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz
-    tar zxf ${SNAPPY_VERSION}.tar.gz
-    pushd snappy-${SNAPPY_VERSION}
-      # Without this patch, snappy 1.10 will report a sign-compare error, which cannot be suppressed with the -Wno-sign-compare option in CI
-      curl -O -L https://raw.githubusercontent.com/microsoft/vcpkg/2024.02.14/ports/snappy/no-werror.patch
-      patch <no-werror.patch
-      CXXFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
-          cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF
-      make -j16
-      make install
-      touch .done
-    popd
-else
-    echo "Using cached Snappy $SNAPPY_VERSION"
-fi
-
-if [ ! -f curl-${CURL_VERSION}/.done ]; then
-    echo "Building LibCurl $CURL_VERSION"
-    CURL_VERSION_=${CURL_VERSION//./_}
-    curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz
-    tar zxf curl-${CURL_VERSION}.tar.gz
-    pushd curl-${CURL_VERSION}
-      # Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl.
-      cp -rf $PREFIX/include/openssl include/
-      CFLAGS="-I$PREFIX/include -fPIC -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
-            ./configure --with-ssl=$PREFIX \
-              --without-nghttp2 \
-              --without-libidn2 \
-              --disable-ldap \
-              --without-brotli \
-              --without-secure-transport \
-              --without-librtmp \
-              --disable-ipv6 \
-              --without-libpsl \
-              --host=$ARCH-apple-darwin \
-              --prefix=$PREFIX
-      make -j16 install
-      touch .done
-    popd
-else
-    echo "Using cached LibCurl $CURL_VERSION"
-fi
-
-popd # pkg/mac
-cd ../../ # project root
-
-cmake -B build-static -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \
-    -DLINK_STATIC=ON \
+# Apply the patch to support building libcurl with IPv6 disabled
+COMMIT_ID=$(grep "builtin-baseline" vcpkg.json | sed 's/"//g' | sed 's/,//' | awk '{print $2}')
+cd vcpkg
+git reset --hard $COMMIT_ID
+git apply ../pkg/mac/vcpkg-curl-patch.diff
+git add ports/curl
+git commit -m "Disable IPv6 for macOS in curl"
+./bootstrap-vcpkg.sh
+./vcpkg x-add-version --all
+git add versions/
+git commit -m "Update version"
+COMMIT_ID=$(git log --pretty=oneline | head -n 1 | awk '{print $1}')
+cd ..
+sed -i.bak "s/.*builtin-baseline.*/  \"builtin-baseline\": \"$COMMIT_ID\",/" vcpkg.json
+sed -i.bak "s/\"version>=\": \"8\.4\.0\"/\"version>=\": \"8.4.0#1\"/" vcpkg.json
+
+INSTALL_DIR=$PWD/pkg/mac/.install
+set -x
+cmake -B build-osx \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \
+    -DINTEGRATE_VCPKG=ON \
+    -DVCPKG_OVERLAY_TRIPLETS=$PWD/vcpkg-triplets \
+    -DVCPKG_TARGET_TRIPLET=$VCPKG_TRIPLET \
+    -DCMAKE_OSX_ARCHITECTURES=$ARCH \
+    -DCMAKE_BUILD_TYPE=Release \
     -DBUILD_TESTS=OFF \
+    -DBUILD_PERF_TOOLS=OFF \
     -DBUILD_DYNAMIC_LIB=ON \
     -DBUILD_STATIC_LIB=ON \
-    -DCMAKE_OSX_ARCHITECTURES=${ARCH} \
-    -DCMAKE_PREFIX_PATH=$PREFIX \
-    -DOPENSSL_ROOT_DIR=$PREFIX \
-    -DPROTOC_PATH=$PREFIX/bin/protoc \
-    -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
-    -DCMAKE_BUILD_TYPE=Release
-cmake --build build-static -j16 --target install
+    -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
+cmake --build build-osx -j16 --target install
+
+./build-support/merge_archives_vcpkg.sh $PWD/build-osx
+cp ./build-osx/libpulsarwithdeps.a $INSTALL_DIR/lib/
+
+# Test the libraries
+clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch $ARCH -I $INSTALL_DIR/include -L $INSTALL_DIR/lib -Wl,-rpath $INSTALL_DIR/lib -lpulsar
+./dynamic.out
+clang++ win-examples/example.cc -o static.out -std=c++11 -arch $ARCH -I $INSTALL_DIR/include $INSTALL_DIR/lib/libpulsarwithdeps.a
+./static.out
diff --git a/pkg/mac/vcpkg-curl-patch.diff b/pkg/mac/vcpkg-curl-patch.diff
@@ -0,0 +1,27 @@
+diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
+index bdc544e9e..340d93865 100644
+--- a/ports/curl/portfile.cmake
++++ b/ports/curl/portfile.cmake
+@@ -64,6 +64,10 @@ if(VCPKG_TARGET_IS_WINDOWS)
+     list(APPEND OPTIONS -DENABLE_UNICODE=ON)
+ endif()
+
++if(VCPKG_TARGET_IS_OSX)
++    list(APPEND OPTIONS -DENABLE_IPV6=OFF)
++endif()
++
+ vcpkg_cmake_configure(
+     SOURCE_PATH "${SOURCE_PATH}"
+     OPTIONS 
+diff --git a/ports/curl/vcpkg.json b/ports/curl/vcpkg.json
+index e028d3897..a63858e34 100644
+--- a/ports/curl/vcpkg.json
++++ b/ports/curl/vcpkg.json
+@@ -1,6 +1,7 @@
+ {
+   "name": "curl",
+   "version": "8.4.0",
++  "port-version": 1,
+   "description": "A library for transferring data with URLs",
+   "homepage": "https://curl.se/",
+   "license": "curl AND ISC AND BSD-3-Clause",
diff --git a/vcpkg-triplets/arm64-osx.cmake b/vcpkg-triplets/arm64-osx.cmake
@@ -0,0 +1,9 @@
+set(VCPKG_TARGET_ARCHITECTURE arm64)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE static)
+
+set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
+set(VCPKG_OSX_ARCHITECTURES arm64)
+set(VCPKG_OSX_DEPLOYMENT_TARGET 13.0)
+
+set(VCPKG_BUILD_TYPE release)
diff --git a/vcpkg-triplets/x64-osx.cmake b/vcpkg-triplets/x64-osx.cmake
@@ -0,0 +1,9 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE static)
+
+set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
+set(VCPKG_OSX_ARCHITECTURES x86_64)
+set(VCPKG_OSX_DEPLOYMENT_TARGET 13.0)
+
+set(VCPKG_BUILD_TYPE release)