From 42cd2846a165e984dbcac421b793eb57fc250a5e Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 16 Dec 2024 12:28:18 +0800 Subject: [PATCH 01/23] Test macOS build --- .github/workflows/ci-pr-validation.yaml | 334 ++---------------------- .github/workflows/codeql-analysis.yml | 73 ------ pkg/mac/build-static-library.sh | 4 + 3 files changed, 21 insertions(+), 390 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 0d229818..d19b51f3 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -29,341 +29,41 @@ concurrency: jobs: - formatting-check: - name: Formatting Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Run clang-format style check for C/C++/Protobuf programs. - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: '11' - exclude-regex: '.*\.(proto|hpp)' - - wireshark-dissector-build: - name: Build the Wireshark dissector - needs: formatting-check - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - strategy: - matrix: - # TODO: support build on macos-14 - os: [ubuntu-20.04] - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Install deps (Ubuntu) - if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: | - sudo apt-get update -y - sudo apt-get install -y protobuf-compiler libprotobuf-dev wireshark-dev - - - name: Install deps (macOS) - if: ${{ startsWith(matrix.os, 'macos') }} - run: | - # See https://github.com/Homebrew/homebrew-core/issues/157142 - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - cd /usr/local/bin - rm -f 2to3* idle3* pydoc* python3* - rm -f /usr/local/share/man/man1/python3.1 /usr/local/lib/pkgconfig/python3* - cd /usr/local/Frameworks/Python.framework - rm -rf Headers Python Resources Versions/Current - brew update - brew install pkg-config wireshark protobuf - - name: Build wireshark plugin - run: | - cmake -S wireshark -B build-wireshark - cmake --build build-wireshark - - unit-tests: - name: Run unit tests - needs: formatting-check - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - - name: Build core libraries - run: | - cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=OFF - cmake --build . -j8 - - - name: Check formatting - run: | - ./vcpkg/vcpkg format-manifest vcpkg.json - if [[ $(git diff | wc -l) -gt 0 ]]; then - echo "Please run `./vcpkg/vcpkg format-manifest vcpkg.json` to reformat vcpkg.json" - exit 1 - fi - - - name: Build tests - run: | - cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON - cmake --build . -j8 - - - name: Install gtest-parallel - run: | - sudo curl -o /gtest-parallel https://raw.githubusercontent.com/google/gtest-parallel/master/gtest_parallel.py - - - name: Run unit tests - run: RETRY_FAILED=3 ./run-unit-tests.sh - - - name: Build perf tools - run: | - cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DBUILD_PERF_TOOLS=ON - cmake --build . -j8 - - - name: Verify custom vcpkg installation - run: | - mv vcpkg /tmp/ - cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_TOOLCHAIN_FILE="/tmp/vcpkg/scripts/buildsystems/vcpkg.cmake" - - cpp20-build: - name: Build with the C++20 standard - needs: formatting-check - runs-on: ubuntu-22.04 - timeout-minutes: 60 - - steps: - - name: checkout - uses: actions/checkout@v3 - - name: Install deps - run: | - sudo apt-get update -y - sudo apt-get install -y libcurl4-openssl-dev libssl-dev \ - protobuf-compiler libprotobuf-dev libboost-dev \ - libboost-dev libboost-program-options-dev \ - libzstd-dev libsnappy-dev libgmock-dev libgtest-dev - - name: CMake - run: cmake -B build -DBUILD_PERF_TOOLS=ON -DCMAKE_CXX_STANDARD=20 - - name: Build - run: | - cmake --build build -j8 --target pulsarShared pulsarStatic - cmake --build build -j8 - - cpp-build-windows: - timeout-minutes: 120 - name: Build CPP Client on ${{ matrix.name }} - needs: unit-tests - runs-on: ${{ matrix.os }} - env: - VCPKG_ROOT: '${{ github.workspace }}/vcpkg' - INSTALL_DIR: 'C:\\pulsar-cpp' - strategy: - fail-fast: false - matrix: - include: - - name: 'Windows x64' - os: windows-2019 - triplet: x64-windows-static - suffix: 'windows-win64' - generator: 'Visual Studio 16 2019' - arch: '-A x64' - - name: 'Windows x86' - os: windows-2019 - triplet: x86-windows-static - suffix: 'windows-win32' - generator: 'Visual Studio 16 2019' - arch: '-A Win32' - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Restore vcpkg and its artifacts. - uses: actions/cache@v3 - id: vcpkg-cache - with: - path: | - ${{ env.VCPKG_ROOT }} - vcpkg_installed - !${{ env.VCPKG_ROOT }}/.git - !${{ env.VCPKG_ROOT }}/buildtrees - !${{ env.VCPKG_ROOT }}/packages - !${{ env.VCPKG_ROOT }}/downloads - key: | - ${{ runner.os }}-${{ matrix.triplet}}-${{ hashFiles( 'vcpkg.json' ) }} - - - name: Get vcpkg(windows) - if: ${{ runner.os == 'Windows' && steps.vcpkg-cache.outputs.cache-hit != 'true' }} - run: | - cd ${{ github.workspace }} - mkdir build -force - git clone https://github.com/Microsoft/vcpkg.git - cd vcpkg - .\bootstrap-vcpkg.bat - - - name: remove system vcpkg(windows) - if: runner.os == 'Windows' - run: rm -rf "$VCPKG_INSTALLATION_ROOT" - shell: bash - - - name: Install vcpkg packages - run: | - ${{ env.VCPKG_ROOT }}\vcpkg.exe install --triplet ${{ matrix.triplet }} - - - name: Configure - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake \ - -B ./build-1 \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DUSE_ASIO=ON \ - -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ - -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ - -S . - fi - - - name: Install - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake --build ./build-1 --parallel --config Release - cmake --install ./build-1 - fi - - - name: Test examples - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cd win-examples - cmake \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DLINK_STATIC=OFF \ - -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \ - -B build-dynamic - cmake --build build-dynamic --config Release - cmake \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DLINK_STATIC=ON \ - -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \ - -B build-static - cmake --build build-static --config Release - ./build-static/Release/win-example.exe - fi - - - name: Build (Debug) - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake \ - -B ./build-2 \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DUSE_ASIO=ON \ - -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ - -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ - -DCMAKE_BUILD_TYPE=Debug \ - -S . - cmake --build ./build-2 --parallel --config Debug - fi - - package: - name: Build ${{matrix.pkg.name}} ${{matrix.cpu.platform}} - runs-on: ubuntu-22.04 - needs: unit-tests + package-macos: + name: Build macOS libraries + runs-on: macos-14 timeout-minutes: 500 strategy: - fail-fast: true + fail-fast: false matrix: - pkg: - - { name: 'RPM', type: 'rpm', path: 'pkg/rpm/RPMS' } - - { name: 'Deb', type: 'deb', path: 'pkg/deb/BUILD/DEB' } - - { name: 'Alpine', type: 'apk', path: 'pkg/apk/build' } - cpu: - - { arch: 'x86_64', platform: 'x86_64' } + arch: [x86_64, arm64] - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/${{matrix.pkg.type}} - load: true - tags: build:latest - platforms: linux/${{matrix.cpu.platform}} - build-args: PLATFORM=${{matrix.cpu.arch}} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build packages - run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest - - cpp-build-macos: - timeout-minutes: 120 - name: Build CPP Client on macOS - needs: formatting-check - runs-on: macos-14 steps: - name: checkout uses: actions/checkout@v3 - name: Install dependencies - run: brew install openssl protobuf boost zstd snappy googletest - - - name: Configure (default) - shell: bash - run: cmake -B ./build-macos -S . - - - name: Compile - shell: bash run: | - cmake --build ./build-macos --parallel --config Release + export ARCH=${{ matrix.arch }} + ./pkg/mac/build-static-library.sh - - name: Build with C++20 - shell: bash + - name: Zip artifact run: | - cmake -B build-macos-cpp20 -DCMAKE_CXX_STANDARD=20 - cmake --build build-macos-cpp20 -j8 - - cpp-build-macos-static: - timeout-minutes: 120 - name: Build CPP Client on macOS with static dependencies - runs-on: macos-14 - needs: unit-tests - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Build libraries - run: ./pkg/mac/build-static-library.sh + cd ./pkg/mac/.install + zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/* + cp macos-${{ matrix.arch }}.zip ../../../ - - name: Test static libraries - run: | - export PULSAR_DIR=$PWD/pkg/mac/.install - echo "Build with static library" - clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a - ./static.out - echo "Build with dynamic library" - clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar - ./dynamic.out + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: macos-${{ matrix.arch }}.zip + path: macos-${{ matrix.arch }}.zip - # Job that will be required to complete and depends on all the other jobs check-completion: name: Check Completion runs-on: ubuntu-latest - needs: [formatting-check, wireshark-dissector-build, unit-tests, cpp20-build, cpp-build-windows, package, cpp-build-macos] + needs: [package-macos] steps: - run: true diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index c877c642..00000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "CodeQL" - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: '16 21 * * 2' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-22.04 - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'cpp' ] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - - - name: Install deps - run: | - sudo apt-get update -y - sudo apt-get install -y libcurl4-openssl-dev libssl-dev \ - protobuf-compiler libprotobuf-dev libboost-dev \ - libboost-dev libboost-program-options-dev \ - libzstd-dev libsnappy-dev - - - name: Build - run: | - cmake . -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DBUILD_PERF_TOOLS=ON - cmake --build . -j8 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.language}}" diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index d787ca90..b161ee87 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -185,6 +185,10 @@ fi popd # pkg/mac cd ../../ # project root +# TODO: add it only for debugging +ls $PREFIX/lib +for file in $(ls $PREFIX/lib/*.a); do file $file; done + cmake -B build-static -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ -DLINK_STATIC=ON \ -DBUILD_TESTS=OFF \ From bcfec3fc0ba15b2a714dc92222d6ceb94dc4f401 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 16 Dec 2024 16:07:32 +0800 Subject: [PATCH 02/23] Build macOS packages via vcpkg --- pkg/mac/build-static-library.sh | 189 ++------------------------------ 1 file changed, 11 insertions(+), 178 deletions(-) diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index b161ee87..f4c32397 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -19,185 +19,18 @@ # set -ex -cd `dirname $0` +cd `dirname $0`/../.. -python3 -m venv venv -source venv/bin/activate -python3 -m pip install pyyaml - -MACOSX_DEPLOYMENT_TARGET=10.15 -if [[ -z ${ARCH} ]]; then - ARCH=`uname -m` -fi - -BUILD_DIR=$PWD/.build -INSTALL_DIR=$PWD/.install -PREFIX=$BUILD_DIR/install -mkdir -p $BUILD_DIR -cp -f ../../build-support/dep-version.py $BUILD_DIR/ -cp -f ../../dependencies.yaml $BUILD_DIR/ - -pushd $BUILD_DIR - -BOOST_VERSION=$(./dep-version.py boost) -ZLIB_VERSION=$(./dep-version.py zlib) -OPENSSL_VERSION=$(./dep-version.py openssl) -PROTOBUF_VERSION=$(./dep-version.py protobuf) -ZSTD_VERSION=$(./dep-version.py zstd) -SNAPPY_VERSION=$(./dep-version.py snappy) -CURL_VERSION=$(./dep-version.py curl) - -if [ ! -f boost/.done ]; then - echo "Building Boost $BOOST_VERSION" - curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz - tar zxf boost-${BOOST_VERSION}.tar.gz - mkdir -p $PREFIX/include - pushd boost-${BOOST_VERSION} - ./bootstrap.sh - ./b2 headers - cp -rf boost $PREFIX/include/ - popd - mkdir -p boost - touch boost/.done -else - echo "Using cached Boost $BOOST_VERSION" -fi - -if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then - echo "Building ZLib $ZLIB_VERSION" - curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz - tar zxf zlib-${ZLIB_VERSION}.tar.gz - pushd zlib-$ZLIB_VERSION - CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX - make -j16 - make install - touch .done - popd -else - echo "Using cached ZLib $ZLIB_VERSION" -fi - -OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') -if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then - echo "Building OpenSSL $OPENSSL_VERSION" - curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz - tar zxf OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz - - pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - if [[ $ARCH = 'arm64' ]]; then - PLATFORM=darwin64-arm64-cc - else - PLATFORM=darwin64-x86_64-cc - fi - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./Configure --prefix=$PREFIX no-shared no-unit-test $PLATFORM - make -j8 >/dev/null - make install_sw >/dev/null - popd - - touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done -else - echo "Using cached OpenSSL $OPENSSL_VERSION" -fi - -if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then - echo "Building Protobuf $PROTOBUF_VERSION" - curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz - tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz - pushd protobuf-${PROTOBUF_VERSION} - pushd cmake/ - # Build protoc that can run on both x86 and arm architectures - cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch x86_64 -arch arm64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - -Dprotobuf_BUILD_TESTS=OFF \ - -DCMAKE_INSTALL_PREFIX=$PREFIX - cmake --build build -j16 --target install - popd - - # Retain the library for one architecture so that `ar` can work on the library - pushd $PREFIX/lib - mv libprotobuf.a libprotobuf_universal.a - lipo libprotobuf_universal.a -thin ${ARCH} -output libprotobuf.a - popd - touch .done - popd -else - echo "Using cached Protobuf $PROTOBUF_VERSION" -fi - -if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then - echo "Building ZStd $ZSTD_VERSION" - curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz - tar zxf zstd-${ZSTD_VERSION}.tar.gz - pushd zstd-${ZSTD_VERSION} - CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ - make -j16 -C lib install-static install-includes - touch .done - popd -else - echo "Using cached ZStd $ZSTD_VERSION" -fi - -if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then - echo "Building Snappy $SNAPPY_VERSION" - curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz - tar zxf ${SNAPPY_VERSION}.tar.gz - pushd snappy-${SNAPPY_VERSION} - # Without this patch, snappy 1.10 will report a sign-compare error, which cannot be suppressed with the -Wno-sign-compare option in CI - curl -O -L https://raw.githubusercontent.com/microsoft/vcpkg/2024.02.14/ports/snappy/no-werror.patch - patch Date: Mon, 16 Dec 2024 16:09:33 +0800 Subject: [PATCH 03/23] Add tests for static libraries --- .github/workflows/ci-pr-validation.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index d19b51f3..c4437857 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -48,6 +48,16 @@ jobs: export ARCH=${{ matrix.arch }} ./pkg/mac/build-static-library.sh + - name: Test static libraries + run: | + export PULSAR_DIR=$PWD/pkg/mac/.install + echo "Build with static library" + clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreFoundation + ./static.out + echo "Build with dynamic library" + clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar + ./dynamic.out + - name: Zip artifact run: | cd ./pkg/mac/.install From 7c3f5f5d96aaa60bc54a24f4bf8e2f45841152a2 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 16 Dec 2024 16:10:43 +0800 Subject: [PATCH 04/23] Fetch vcpkg submodules --- .github/workflows/ci-pr-validation.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index c4437857..c6c1d721 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -42,6 +42,9 @@ jobs: steps: - name: checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive - name: Install dependencies run: | From 80621f00dbfbd7c3777d68e6db7fb071627f996f Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 16 Dec 2024 16:20:56 +0800 Subject: [PATCH 05/23] Install the libraries --- pkg/mac/build-static-library.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index f4c32397..0b95d31e 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -30,7 +30,7 @@ cmake -B build-osx -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 \ -DBUILD_DYNAMIC_LIB=ON \ -DBUILD_STATIC_LIB=ON \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -cmake --build build-osx -j16 +cmake --build build-osx -j16 --target install ./build-support/merge_archives_vcpkg.sh $PWD/build-osx cp ./build-osx/libpulsarwithdeps.a $INSTALL_DIR/lib/ From 092c3b2bf525e72bd2c3d9cc978ea264ce6cf9d6 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 16 Dec 2024 16:56:01 +0800 Subject: [PATCH 06/23] Specify OSX architecture --- .github/workflows/ci-pr-validation.yaml | 4 ++-- build-support/merge_archives_vcpkg.sh | 12 +++++++++--- pkg/mac/build-static-library.sh | 15 ++++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index c6c1d721..f7575623 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -55,10 +55,10 @@ jobs: run: | export PULSAR_DIR=$PWD/pkg/mac/.install echo "Build with static library" - clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreFoundation + clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreFoundation ./static.out echo "Build with dynamic library" - clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar + clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar ./dynamic.out - name: Zip artifact diff --git a/build-support/merge_archives_vcpkg.sh b/build-support/merge_archives_vcpkg.sh index de1756e5..73c445fb 100755 --- a/build-support/merge_archives_vcpkg.sh +++ b/build-support/merge_archives_vcpkg.sh @@ -27,6 +27,12 @@ if [[ $# -lt 1 ]]; then fi CMAKE_BUILD_DIRECTORY=$1 -./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \ - $CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \ - $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug) +if [[ $VCPKG_TRIPLET ]]; then + ./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \ + $CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \ + $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed/$VCPKG_TRIPLET" -name "*.a" | grep -v debug) +else + ./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \ + $CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \ + $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug) +fi diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index 0b95d31e..74ae0be9 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -18,12 +18,25 @@ # under the License. # -set -ex +set -e cd `dirname $0`/../.. +if [[ $ARCH == "x86_64" ]]; then + export VCPKG_TRIPLET=x64-osx +elif [[ $ARCH == "arm64" ]]; then + export VCPKG_TRIPLET=arm64-osx +else + echo "Invalid ARCH: $ARCH" + exit 1 +fi +CMAKE_OSX_ARCHITECTURES=$ARCH + INSTALL_DIR=$PWD/pkg/mac/.install +set -x cmake -B build-osx -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 \ -DINTEGRATE_VCPKG=ON \ + -DVCPKG_TARGET_TRIPLET=$VCPKG_TRIPLET \ + -DCMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_TESTS=OFF \ -DBUILD_PERF_TOOLS=OFF \ From de2f80e15af5fb3aa577be4f2237b8e09fe3a2c0 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 16 Dec 2024 17:54:10 +0800 Subject: [PATCH 07/23] Upload artifacts before verifications --- .github/workflows/ci-pr-validation.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index f7575623..14dad854 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -51,16 +51,6 @@ jobs: export ARCH=${{ matrix.arch }} ./pkg/mac/build-static-library.sh - - name: Test static libraries - run: | - export PULSAR_DIR=$PWD/pkg/mac/.install - echo "Build with static library" - clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreFoundation - ./static.out - echo "Build with dynamic library" - clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar - ./dynamic.out - - name: Zip artifact run: | cd ./pkg/mac/.install @@ -73,6 +63,16 @@ jobs: name: macos-${{ matrix.arch }}.zip path: macos-${{ matrix.arch }}.zip + - name: Test static libraries + run: | + echo "Build with dynamic library" + clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar + ./dynamic.out + export PULSAR_DIR=$PWD/pkg/mac/.install + echo "Build with static library" + clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreFoundation + ./static.out + check-completion: name: Check Completion runs-on: ubuntu-latest From 812bc984d2739d941cd3e08dab055747f70f91ba Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 16 Dec 2024 20:48:00 +0800 Subject: [PATCH 08/23] Fix the PULSAR_DIR not set issue --- .github/workflows/ci-pr-validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 14dad854..0da32096 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -65,10 +65,10 @@ jobs: - name: Test static libraries run: | + export PULSAR_DIR=$PWD/pkg/mac/.install echo "Build with dynamic library" clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar ./dynamic.out - export PULSAR_DIR=$PWD/pkg/mac/.install echo "Build with static library" clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreFoundation ./static.out From b92cf38e8640b6909ef89640da815f59ce333349 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 16 Dec 2024 23:07:53 +0800 Subject: [PATCH 09/23] Set VCPKG_OSX_DEPLOYMENT_TARGET with 10.15 in triplets --- arm64-osx.cmake | 7 +++++++ pkg/mac/build-static-library.sh | 2 ++ x64-osx.cmake | 7 +++++++ 3 files changed, 16 insertions(+) create mode 100644 arm64-osx.cmake create mode 100644 x64-osx.cmake diff --git a/arm64-osx.cmake b/arm64-osx.cmake new file mode 100644 index 00000000..caae5524 --- /dev/null +++ b/arm64-osx.cmake @@ -0,0 +1,7 @@ +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES arm64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 10.15) diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index 74ae0be9..ab86a3e0 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -23,8 +23,10 @@ cd `dirname $0`/../.. if [[ $ARCH == "x86_64" ]]; then export VCPKG_TRIPLET=x64-osx + cp -f x64-osx.cmake vcpkg/triplets/x64-osx.cmake elif [[ $ARCH == "arm64" ]]; then export VCPKG_TRIPLET=arm64-osx + cp -f arm64-osx.cmake vcpkg/triplets/community/arm64-osx.cmake else echo "Invalid ARCH: $ARCH" exit 1 diff --git a/x64-osx.cmake b/x64-osx.cmake new file mode 100644 index 00000000..3e1730f1 --- /dev/null +++ b/x64-osx.cmake @@ -0,0 +1,7 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES x86_64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 10.15) From b63fb20669493acf618bde153e09665bc853c001 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 00:01:51 +0800 Subject: [PATCH 10/23] Upgrade VCPKG_OSX_DEPLOYMENT_TARGET to 13.7 --- arm64-osx.cmake | 2 +- pkg/mac/build-static-library.sh | 2 +- x64-osx.cmake | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arm64-osx.cmake b/arm64-osx.cmake index caae5524..e15ff52f 100644 --- a/arm64-osx.cmake +++ b/arm64-osx.cmake @@ -4,4 +4,4 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) -set(VCPKG_OSX_DEPLOYMENT_TARGET 10.15) +set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index ab86a3e0..ecdbb115 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -35,7 +35,7 @@ CMAKE_OSX_ARCHITECTURES=$ARCH INSTALL_DIR=$PWD/pkg/mac/.install set -x -cmake -B build-osx -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 \ +cmake -B build-osx \ -DINTEGRATE_VCPKG=ON \ -DVCPKG_TARGET_TRIPLET=$VCPKG_TRIPLET \ -DCMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \ diff --git a/x64-osx.cmake b/x64-osx.cmake index 3e1730f1..f9864fe5 100644 --- a/x64-osx.cmake +++ b/x64-osx.cmake @@ -4,4 +4,4 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES x86_64) -set(VCPKG_OSX_DEPLOYMENT_TARGET 10.15) +set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) From e0b2ecf61c8f251891d843f9a0c8eb41f68deb05 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 00:12:39 +0800 Subject: [PATCH 11/23] Speed up dependencies build by only building release libraries --- pkg/mac/build-static-library.sh | 4 ++-- arm64-osx.cmake => vcpkg-triplets/arm64-osx.cmake | 2 ++ x64-osx.cmake => vcpkg-triplets/x64-osx.cmake | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) rename arm64-osx.cmake => vcpkg-triplets/arm64-osx.cmake (87%) rename x64-osx.cmake => vcpkg-triplets/x64-osx.cmake (87%) diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index ecdbb115..0f45c438 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -23,10 +23,10 @@ cd `dirname $0`/../.. if [[ $ARCH == "x86_64" ]]; then export VCPKG_TRIPLET=x64-osx - cp -f x64-osx.cmake vcpkg/triplets/x64-osx.cmake + cp -f vcpkg-triplets/x64-osx.cmake vcpkg/triplets/x64-osx.cmake elif [[ $ARCH == "arm64" ]]; then export VCPKG_TRIPLET=arm64-osx - cp -f arm64-osx.cmake vcpkg/triplets/community/arm64-osx.cmake + cp -f vcpkg-triplets/arm64-osx.cmake vcpkg/triplets/community/arm64-osx.cmake else echo "Invalid ARCH: $ARCH" exit 1 diff --git a/arm64-osx.cmake b/vcpkg-triplets/arm64-osx.cmake similarity index 87% rename from arm64-osx.cmake rename to vcpkg-triplets/arm64-osx.cmake index e15ff52f..189776c6 100644 --- a/arm64-osx.cmake +++ b/vcpkg-triplets/arm64-osx.cmake @@ -5,3 +5,5 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) + +set(VCPKG_BUILD_TYPE release) diff --git a/x64-osx.cmake b/vcpkg-triplets/x64-osx.cmake similarity index 87% rename from x64-osx.cmake rename to vcpkg-triplets/x64-osx.cmake index f9864fe5..09daa871 100644 --- a/x64-osx.cmake +++ b/vcpkg-triplets/x64-osx.cmake @@ -5,3 +5,5 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES x86_64) set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) + +set(VCPKG_BUILD_TYPE release) From af8fd74e5c55d769b7d86ce1911cab310054aa85 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 00:34:05 +0800 Subject: [PATCH 12/23] Remove VCPKG_OSX_DEPLOYMENT_TARGET --- vcpkg-triplets/arm64-osx.cmake | 1 - vcpkg-triplets/x64-osx.cmake | 1 - 2 files changed, 2 deletions(-) diff --git a/vcpkg-triplets/arm64-osx.cmake b/vcpkg-triplets/arm64-osx.cmake index 189776c6..94529079 100644 --- a/vcpkg-triplets/arm64-osx.cmake +++ b/vcpkg-triplets/arm64-osx.cmake @@ -4,6 +4,5 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) -set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) set(VCPKG_BUILD_TYPE release) diff --git a/vcpkg-triplets/x64-osx.cmake b/vcpkg-triplets/x64-osx.cmake index 09daa871..61132d23 100644 --- a/vcpkg-triplets/x64-osx.cmake +++ b/vcpkg-triplets/x64-osx.cmake @@ -4,6 +4,5 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES x86_64) -set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) set(VCPKG_BUILD_TYPE release) From c4c1215a899572f589154a4faee15b2a197a1670 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 00:48:50 +0800 Subject: [PATCH 13/23] Revert "Remove VCPKG_OSX_DEPLOYMENT_TARGET" This reverts commit af8fd74e5c55d769b7d86ce1911cab310054aa85. --- vcpkg-triplets/arm64-osx.cmake | 1 + vcpkg-triplets/x64-osx.cmake | 1 + 2 files changed, 2 insertions(+) diff --git a/vcpkg-triplets/arm64-osx.cmake b/vcpkg-triplets/arm64-osx.cmake index 94529079..189776c6 100644 --- a/vcpkg-triplets/arm64-osx.cmake +++ b/vcpkg-triplets/arm64-osx.cmake @@ -4,5 +4,6 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) set(VCPKG_BUILD_TYPE release) diff --git a/vcpkg-triplets/x64-osx.cmake b/vcpkg-triplets/x64-osx.cmake index 61132d23..09daa871 100644 --- a/vcpkg-triplets/x64-osx.cmake +++ b/vcpkg-triplets/x64-osx.cmake @@ -4,5 +4,6 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES x86_64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) set(VCPKG_BUILD_TYPE release) From 796759eb8cf85ce5008380cafa65b0c0d27dc3e3 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 00:56:38 +0800 Subject: [PATCH 14/23] Set minimum osx version to 13.0 and link CoreServices --- .github/workflows/ci-pr-validation.yaml | 2 +- vcpkg-triplets/arm64-osx.cmake | 2 +- vcpkg-triplets/x64-osx.cmake | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 0da32096..8debdca9 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -70,7 +70,7 @@ jobs: clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar ./dynamic.out echo "Build with static library" - clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreFoundation + clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreServices -framework CoreFoundation ./static.out check-completion: diff --git a/vcpkg-triplets/arm64-osx.cmake b/vcpkg-triplets/arm64-osx.cmake index 189776c6..10ba57c0 100644 --- a/vcpkg-triplets/arm64-osx.cmake +++ b/vcpkg-triplets/arm64-osx.cmake @@ -4,6 +4,6 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) -set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) +set(VCPKG_OSX_DEPLOYMENT_TARGET 13.0) set(VCPKG_BUILD_TYPE release) diff --git a/vcpkg-triplets/x64-osx.cmake b/vcpkg-triplets/x64-osx.cmake index 09daa871..4406d939 100644 --- a/vcpkg-triplets/x64-osx.cmake +++ b/vcpkg-triplets/x64-osx.cmake @@ -4,6 +4,6 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES x86_64) -set(VCPKG_OSX_DEPLOYMENT_TARGET 13.7) +set(VCPKG_OSX_DEPLOYMENT_TARGET 13.0) set(VCPKG_BUILD_TYPE release) From 7908cc37fc5ada918d096f14f3a4f900ec28cefe Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 18:45:22 +0800 Subject: [PATCH 15/23] Set CMAKE_OSX_DEPLOYMENT_TARGET for pulsar --- pkg/mac/build-static-library.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index 0f45c438..84065855 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -36,6 +36,7 @@ CMAKE_OSX_ARCHITECTURES=$ARCH INSTALL_DIR=$PWD/pkg/mac/.install set -x cmake -B build-osx \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ -DINTEGRATE_VCPKG=ON \ -DVCPKG_TARGET_TRIPLET=$VCPKG_TRIPLET \ -DCMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \ From 382c9d9a5fcc3212987685d416b06983e9af82c6 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 19:03:36 +0800 Subject: [PATCH 16/23] Apply the patch for vcpkg to disable IPv6 for curl's macOS build --- .github/workflows/ci-pr-validation.yaml | 5 ++- build-support/apply-vcpkg-macos-curl-patch.sh | 38 +++++++++++++++++++ build-support/vcpkg-curl-patch.diff | 27 +++++++++++++ pkg/mac/build-static-library.sh | 10 +++-- 4 files changed, 75 insertions(+), 5 deletions(-) create mode 100755 build-support/apply-vcpkg-macos-curl-patch.sh create mode 100644 build-support/vcpkg-curl-patch.diff diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 8debdca9..30f16020 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -46,6 +46,9 @@ jobs: fetch-depth: 0 submodules: recursive + - name: Apply the patch for vcpkg + run: ./build-support/apply-vcpkg-macos-curl-patch.sh + - name: Install dependencies run: | export ARCH=${{ matrix.arch }} @@ -70,7 +73,7 @@ jobs: clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar ./dynamic.out echo "Build with static library" - clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreServices -framework CoreFoundation + clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a ./static.out check-completion: diff --git a/build-support/apply-vcpkg-macos-curl-patch.sh b/build-support/apply-vcpkg-macos-curl-patch.sh new file mode 100755 index 00000000..27556a18 --- /dev/null +++ b/build-support/apply-vcpkg-macos-curl-patch.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +cd `dirname $0`/.. +set -ex + +COMMIT_ID=$(grep "builtin-baseline" vcpkg.json | sed 's/"//g' | sed 's/,//' | awk '{print $2}') +cd vcpkg +git reset --hard $COMMIT_ID +git apply ../build-support/vcpkg-curl-patch.diff +git add ports/curl +git commit -m "Disable IPv6 for macOS in curl" +./bootstrap-vcpkg.sh +./vcpkg x-add-version --all +git add versions/ +git commit -m "Update version" +COMMIT_ID=$(git log --pretty=oneline | head -n 1 | awk '{print $1}') + +cd .. +sed -i.bak "s/.*builtin-baseline.*/ \"builtin-baseline\": \"$COMMIT_ID\",/" vcpkg.json +sed -i.bak "s/\"version>=\": \"8\.4\.0\"/\"version>=\": \"8.4.0#1\"/" vcpkg.json diff --git a/build-support/vcpkg-curl-patch.diff b/build-support/vcpkg-curl-patch.diff new file mode 100644 index 00000000..e619e29b --- /dev/null +++ b/build-support/vcpkg-curl-patch.diff @@ -0,0 +1,27 @@ +diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake +index bdc544e9e..340d93865 100644 +--- a/ports/curl/portfile.cmake ++++ b/ports/curl/portfile.cmake +@@ -64,6 +64,10 @@ if(VCPKG_TARGET_IS_WINDOWS) + list(APPEND OPTIONS -DENABLE_UNICODE=ON) + endif() + ++if(VCPKG_TARGET_IS_OSX) ++ list(APPEND OPTIONS -DENABLE_IPV6=OFF) ++endif() ++ + vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS +diff --git a/ports/curl/vcpkg.json b/ports/curl/vcpkg.json +index e028d3897..a63858e34 100644 +--- a/ports/curl/vcpkg.json ++++ b/ports/curl/vcpkg.json +@@ -1,6 +1,7 @@ + { + "name": "curl", + "version": "8.4.0", ++ "port-version": 1, + "description": "A library for transferring data with URLs", + "homepage": "https://curl.se/", + "license": "curl AND ISC AND BSD-3-Clause", diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index 84065855..e483b467 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -21,25 +21,27 @@ set -e cd `dirname $0`/../.. +if [[ -z $ARCH ]]; then + ARCH=$(uname -m) + echo "Use default ARCH: $ARCH" +fi if [[ $ARCH == "x86_64" ]]; then export VCPKG_TRIPLET=x64-osx - cp -f vcpkg-triplets/x64-osx.cmake vcpkg/triplets/x64-osx.cmake elif [[ $ARCH == "arm64" ]]; then export VCPKG_TRIPLET=arm64-osx - cp -f vcpkg-triplets/arm64-osx.cmake vcpkg/triplets/community/arm64-osx.cmake else echo "Invalid ARCH: $ARCH" exit 1 fi -CMAKE_OSX_ARCHITECTURES=$ARCH INSTALL_DIR=$PWD/pkg/mac/.install set -x cmake -B build-osx \ -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ -DINTEGRATE_VCPKG=ON \ + -DVCPKG_OVERLAY_TRIPLETS=$PWD/vcpkg-triplets \ -DVCPKG_TARGET_TRIPLET=$VCPKG_TRIPLET \ - -DCMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \ + -DCMAKE_OSX_ARCHITECTURES=$ARCH \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_TESTS=OFF \ -DBUILD_PERF_TOOLS=OFF \ From 9ed491c8075a524f2e6a144df923fb6933a0b322 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 20:38:36 +0800 Subject: [PATCH 17/23] Switch to bewaremypower/vcpkg to build curl without frameworks linked --- .github/workflows/ci-pr-validation.yaml | 23 +++++------ .gitmodules | 2 +- build-support/apply-vcpkg-macos-curl-patch.sh | 38 ------------------- build-support/vcpkg-curl-patch.diff | 27 ------------- vcpkg | 2 +- vcpkg.json | 4 +- 6 files changed, 14 insertions(+), 82 deletions(-) delete mode 100755 build-support/apply-vcpkg-macos-curl-patch.sh delete mode 100644 build-support/vcpkg-curl-patch.diff diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 30f16020..fb5bd751 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -46,14 +46,21 @@ jobs: fetch-depth: 0 submodules: recursive - - name: Apply the patch for vcpkg - run: ./build-support/apply-vcpkg-macos-curl-patch.sh - - name: Install dependencies run: | export ARCH=${{ matrix.arch }} ./pkg/mac/build-static-library.sh + - name: Test static libraries + run: | + export PULSAR_DIR=$PWD/pkg/mac/.install + echo "Build with dynamic library" + clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar + ./dynamic.out + echo "Build with static library" + clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a + ./static.out + - name: Zip artifact run: | cd ./pkg/mac/.install @@ -66,16 +73,6 @@ jobs: name: macos-${{ matrix.arch }}.zip path: macos-${{ matrix.arch }}.zip - - name: Test static libraries - run: | - export PULSAR_DIR=$PWD/pkg/mac/.install - echo "Build with dynamic library" - clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar - ./dynamic.out - echo "Build with static library" - clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a - ./static.out - check-completion: name: Check Completion runs-on: ubuntu-latest diff --git a/.gitmodules b/.gitmodules index a0a57f3d..8e838b6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "vcpkg"] path = vcpkg - url = https://github.com/microsoft/vcpkg.git + url = https://github.com/bewaremypower/vcpkg.git diff --git a/build-support/apply-vcpkg-macos-curl-patch.sh b/build-support/apply-vcpkg-macos-curl-patch.sh deleted file mode 100755 index 27556a18..00000000 --- a/build-support/apply-vcpkg-macos-curl-patch.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cd `dirname $0`/.. -set -ex - -COMMIT_ID=$(grep "builtin-baseline" vcpkg.json | sed 's/"//g' | sed 's/,//' | awk '{print $2}') -cd vcpkg -git reset --hard $COMMIT_ID -git apply ../build-support/vcpkg-curl-patch.diff -git add ports/curl -git commit -m "Disable IPv6 for macOS in curl" -./bootstrap-vcpkg.sh -./vcpkg x-add-version --all -git add versions/ -git commit -m "Update version" -COMMIT_ID=$(git log --pretty=oneline | head -n 1 | awk '{print $1}') - -cd .. -sed -i.bak "s/.*builtin-baseline.*/ \"builtin-baseline\": \"$COMMIT_ID\",/" vcpkg.json -sed -i.bak "s/\"version>=\": \"8\.4\.0\"/\"version>=\": \"8.4.0#1\"/" vcpkg.json diff --git a/build-support/vcpkg-curl-patch.diff b/build-support/vcpkg-curl-patch.diff deleted file mode 100644 index e619e29b..00000000 --- a/build-support/vcpkg-curl-patch.diff +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake -index bdc544e9e..340d93865 100644 ---- a/ports/curl/portfile.cmake -+++ b/ports/curl/portfile.cmake -@@ -64,6 +64,10 @@ if(VCPKG_TARGET_IS_WINDOWS) - list(APPEND OPTIONS -DENABLE_UNICODE=ON) - endif() - -+if(VCPKG_TARGET_IS_OSX) -+ list(APPEND OPTIONS -DENABLE_IPV6=OFF) -+endif() -+ - vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS -diff --git a/ports/curl/vcpkg.json b/ports/curl/vcpkg.json -index e028d3897..a63858e34 100644 ---- a/ports/curl/vcpkg.json -+++ b/ports/curl/vcpkg.json -@@ -1,6 +1,7 @@ - { - "name": "curl", - "version": "8.4.0", -+ "port-version": 1, - "description": "A library for transferring data with URLs", - "homepage": "https://curl.se/", - "license": "curl AND ISC AND BSD-3-Clause", diff --git a/vcpkg b/vcpkg index 97dd2672..b03173a5 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 97dd26728e3856ed1ab62ee74ee3a391d9c81d19 +Subproject commit b03173a5d0cbd2f7c3c5e85a576e98fba3560835 diff --git a/vcpkg.json b/vcpkg.json index 5ff44100..e117563c 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -2,7 +2,7 @@ "name": "pulsar-cpp", "version": "3.5.0", "description": "Pulsar C++ SDK", - "builtin-baseline": "b051745c68faa6f65c493371d564c4eb8af34dad", + "builtin-baseline": "b03173a5d0cbd2f7c3c5e85a576e98fba3560835", "dependencies": [ { "name": "asio", @@ -25,7 +25,7 @@ "features": [ "openssl" ], - "version>=": "8.4.0" + "version>=": "8.4.0#1" }, { "name": "dlfcn-win32", From 0cfe3ae937f4b5e5dff8ad7a055a53b5d4e8d5a5 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 20:54:04 +0800 Subject: [PATCH 18/23] Restore the deleted workflows --- .github/workflows/ci-pr-validation.yaml | 329 ++++++++++++++++++++++-- .github/workflows/codeql-analysis.yml | 73 ++++++ pkg/mac/build-static-library.sh | 6 + vcpkg | 2 +- 4 files changed, 383 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index fb5bd751..b40fa3f8 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -29,15 +29,59 @@ concurrency: jobs: - package-macos: - name: Build macOS libraries - runs-on: macos-14 - timeout-minutes: 500 + formatting-check: + name: Formatting Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run clang-format style check for C/C++/Protobuf programs. + uses: jidicula/clang-format-action@v4.11.0 + with: + clang-format-version: '11' + exclude-regex: '.*\.(proto|hpp)' + wireshark-dissector-build: + name: Build the Wireshark dissector + needs: formatting-check + runs-on: ${{ matrix.os }} + timeout-minutes: 60 strategy: - fail-fast: false matrix: - arch: [x86_64, arm64] + # TODO: support build on macos-14 + os: [ubuntu-20.04] + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Install deps (Ubuntu) + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt-get update -y + sudo apt-get install -y protobuf-compiler libprotobuf-dev wireshark-dev + + - name: Install deps (macOS) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + # See https://github.com/Homebrew/homebrew-core/issues/157142 + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + cd /usr/local/bin + rm -f 2to3* idle3* pydoc* python3* + rm -f /usr/local/share/man/man1/python3.1 /usr/local/lib/pkgconfig/python3* + cd /usr/local/Frameworks/Python.framework + rm -rf Headers Python Resources Versions/Current + brew update + brew install pkg-config wireshark protobuf + - name: Build wireshark plugin + run: | + cmake -S wireshark -B build-wireshark + cmake --build build-wireshark + + unit-tests: + name: Run unit tests + needs: formatting-check + runs-on: ubuntu-22.04 + timeout-minutes: 120 steps: - name: checkout @@ -46,37 +90,270 @@ jobs: fetch-depth: 0 submodules: recursive - - name: Install dependencies + - name: Build core libraries + run: | + cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=OFF + cmake --build . -j8 + + - name: Check formatting + run: | + ./vcpkg/vcpkg format-manifest vcpkg.json + if [[ $(git diff | wc -l) -gt 0 ]]; then + echo "Please run `./vcpkg/vcpkg format-manifest vcpkg.json` to reformat vcpkg.json" + exit 1 + fi + + - name: Build tests + run: | + cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON + cmake --build . -j8 + + - name: Install gtest-parallel + run: | + sudo curl -o /gtest-parallel https://raw.githubusercontent.com/google/gtest-parallel/master/gtest_parallel.py + + - name: Run unit tests + run: RETRY_FAILED=3 ./run-unit-tests.sh + + - name: Build perf tools + run: | + cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DBUILD_PERF_TOOLS=ON + cmake --build . -j8 + + - name: Verify custom vcpkg installation + run: | + mv vcpkg /tmp/ + cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_TOOLCHAIN_FILE="/tmp/vcpkg/scripts/buildsystems/vcpkg.cmake" + + cpp20-build: + name: Build with the C++20 standard + needs: formatting-check + runs-on: ubuntu-22.04 + timeout-minutes: 60 + + steps: + - name: checkout + uses: actions/checkout@v3 + - name: Install deps + run: | + sudo apt-get update -y + sudo apt-get install -y libcurl4-openssl-dev libssl-dev \ + protobuf-compiler libprotobuf-dev libboost-dev \ + libboost-dev libboost-program-options-dev \ + libzstd-dev libsnappy-dev libgmock-dev libgtest-dev + - name: CMake + run: cmake -B build -DBUILD_PERF_TOOLS=ON -DCMAKE_CXX_STANDARD=20 + - name: Build + run: | + cmake --build build -j8 --target pulsarShared pulsarStatic + cmake --build build -j8 + + cpp-build-windows: + timeout-minutes: 120 + name: Build CPP Client on ${{ matrix.name }} + needs: unit-tests + runs-on: ${{ matrix.os }} + env: + VCPKG_ROOT: '${{ github.workspace }}/vcpkg' + INSTALL_DIR: 'C:\\pulsar-cpp' + strategy: + fail-fast: false + matrix: + include: + - name: 'Windows x64' + os: windows-2019 + triplet: x64-windows-static + suffix: 'windows-win64' + generator: 'Visual Studio 16 2019' + arch: '-A x64' + - name: 'Windows x86' + os: windows-2019 + triplet: x86-windows-static + suffix: 'windows-win32' + generator: 'Visual Studio 16 2019' + arch: '-A Win32' + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Restore vcpkg and its artifacts. + uses: actions/cache@v3 + id: vcpkg-cache + with: + path: | + ${{ env.VCPKG_ROOT }} + vcpkg_installed + !${{ env.VCPKG_ROOT }}/.git + !${{ env.VCPKG_ROOT }}/buildtrees + !${{ env.VCPKG_ROOT }}/packages + !${{ env.VCPKG_ROOT }}/downloads + key: | + ${{ runner.os }}-${{ matrix.triplet}}-${{ hashFiles( 'vcpkg.json' ) }} + + - name: Get vcpkg(windows) + if: ${{ runner.os == 'Windows' && steps.vcpkg-cache.outputs.cache-hit != 'true' }} + run: | + cd ${{ github.workspace }} + mkdir build -force + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + .\bootstrap-vcpkg.bat + + - name: remove system vcpkg(windows) + if: runner.os == 'Windows' + run: rm -rf "$VCPKG_INSTALLATION_ROOT" + shell: bash + + - name: Install vcpkg packages + run: | + ${{ env.VCPKG_ROOT }}\vcpkg.exe install --triplet ${{ matrix.triplet }} + + - name: Configure + shell: bash + run: | + if [ "$RUNNER_OS" == "Windows" ]; then + cmake \ + -B ./build-1 \ + -G "${{ matrix.generator }}" ${{ matrix.arch }} \ + -DUSE_ASIO=ON \ + -DBUILD_TESTS=OFF \ + -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ + -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ + -S . + fi + + - name: Install + shell: bash run: | - export ARCH=${{ matrix.arch }} - ./pkg/mac/build-static-library.sh + if [ "$RUNNER_OS" == "Windows" ]; then + cmake --build ./build-1 --parallel --config Release + cmake --install ./build-1 + fi - - name: Test static libraries + - name: Test examples + shell: bash run: | - export PULSAR_DIR=$PWD/pkg/mac/.install - echo "Build with dynamic library" - clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar - ./dynamic.out - echo "Build with static library" - clang++ win-examples/example.cc -o static.out -std=c++11 -arch ${{ matrix.arch }} -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a - ./static.out + if [ "$RUNNER_OS" == "Windows" ]; then + cd win-examples + cmake \ + -G "${{ matrix.generator }}" ${{ matrix.arch }} \ + -DLINK_STATIC=OFF \ + -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \ + -B build-dynamic + cmake --build build-dynamic --config Release + cmake \ + -G "${{ matrix.generator }}" ${{ matrix.arch }} \ + -DLINK_STATIC=ON \ + -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \ + -B build-static + cmake --build build-static --config Release + ./build-static/Release/win-example.exe + fi - - name: Zip artifact + - name: Build (Debug) + shell: bash run: | - cd ./pkg/mac/.install - zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/* - cp macos-${{ matrix.arch }}.zip ../../../ + if [ "$RUNNER_OS" == "Windows" ]; then + cmake \ + -B ./build-2 \ + -G "${{ matrix.generator }}" ${{ matrix.arch }} \ + -DUSE_ASIO=ON \ + -DBUILD_TESTS=OFF \ + -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ + -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ + -DCMAKE_BUILD_TYPE=Debug \ + -S . + cmake --build ./build-2 --parallel --config Debug + fi + + package: + name: Build ${{matrix.pkg.name}} ${{matrix.cpu.platform}} + runs-on: ubuntu-22.04 + needs: unit-tests + timeout-minutes: 500 + + strategy: + fail-fast: true + matrix: + pkg: + - { name: 'RPM', type: 'rpm', path: 'pkg/rpm/RPMS' } + - { name: 'Deb', type: 'deb', path: 'pkg/deb/BUILD/DEB' } + - { name: 'Alpine', type: 'apk', path: 'pkg/apk/build' } + cpu: + - { arch: 'x86_64', platform: 'x86_64' } + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - - name: Upload artifacts - uses: actions/upload-artifact@v3 + - name: Package Pulsar source + run: build-support/generate-source-archive.sh + + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 with: - name: macos-${{ matrix.arch }}.zip - path: macos-${{ matrix.arch }}.zip + context: ./pkg/${{matrix.pkg.type}} + load: true + tags: build:latest + platforms: linux/${{matrix.cpu.platform}} + build-args: PLATFORM=${{matrix.cpu.arch}} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build packages + run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest + + cpp-build-macos: + timeout-minutes: 120 + name: Build CPP Client on macOS + needs: formatting-check + runs-on: macos-14 + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Install dependencies + run: brew install openssl protobuf boost zstd snappy googletest + + - name: Configure (default) + shell: bash + run: cmake -B ./build-macos -S . + + - name: Compile + shell: bash + run: | + cmake --build ./build-macos --parallel --config Release + + - name: Build with C++20 + shell: bash + run: | + cmake -B build-macos-cpp20 -DCMAKE_CXX_STANDARD=20 + cmake --build build-macos-cpp20 -j8 + + cpp-build-macos-static: + timeout-minutes: 120 + name: Build CPP Client on macOS with static dependencies + runs-on: macos-14 + needs: unit-tests + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Build libraries + run: ./pkg/mac/build-static-library.sh + # Job that will be required to complete and depends on all the other jobs check-completion: name: Check Completion runs-on: ubuntu-latest - needs: [package-macos] + needs: [formatting-check, wireshark-dissector-build, unit-tests, cpp20-build, cpp-build-windows, package, cpp-build-macos] steps: - run: true diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..c877c642 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '16 21 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-22.04 + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'cpp' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Install deps + run: | + sudo apt-get update -y + sudo apt-get install -y libcurl4-openssl-dev libssl-dev \ + protobuf-compiler libprotobuf-dev libboost-dev \ + libboost-dev libboost-program-options-dev \ + libzstd-dev libsnappy-dev + + - name: Build + run: | + cmake . -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DBUILD_PERF_TOOLS=ON + cmake --build . -j8 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index e483b467..9524ad47 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -52,3 +52,9 @@ cmake --build build-osx -j16 --target install ./build-support/merge_archives_vcpkg.sh $PWD/build-osx cp ./build-osx/libpulsarwithdeps.a $INSTALL_DIR/lib/ + +# Test the libraries +clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch $ARCH -I $INSTALL_DIR/include -L $INSTALL_DIR/lib -Wl,-rpath $INSTALL_DIR/lib -lpulsar +./dynamic.out +clang++ win-examples/example.cc -o static.out -std=c++11 -arch $ARCH -I $INSTALL_DIR/include $INSTALL_DIR/lib/libpulsarwithdeps.a +./static.out diff --git a/vcpkg b/vcpkg index b03173a5..b322364f 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit b03173a5d0cbd2f7c3c5e85a576e98fba3560835 +Subproject commit b322364f06308bdd24823f9d8f03fe0cc86fd46f From 8e3eb8ddc09df225af34dd32ed5c07805aa34a4c Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 21:18:39 +0800 Subject: [PATCH 19/23] Update the commit id --- .github/workflows/ci-pr-validation.yaml | 2 +- vcpkg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index b40fa3f8..0cc2f7ae 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -341,7 +341,7 @@ jobs: timeout-minutes: 120 name: Build CPP Client on macOS with static dependencies runs-on: macos-14 - needs: unit-tests + needs: formatting-check steps: - name: checkout uses: actions/checkout@v3 diff --git a/vcpkg b/vcpkg index b322364f..b03173a5 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit b322364f06308bdd24823f9d8f03fe0cc86fd46f +Subproject commit b03173a5d0cbd2f7c3c5e85a576e98fba3560835 From 92bb3957a0366a0f070f2b4f8495bd1238a886a5 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 17 Dec 2024 21:40:26 +0800 Subject: [PATCH 20/23] Only use forked vcpkg for macOS build --- .gitmodules | 2 +- pkg/mac/build-static-library.sh | 5 +++++ vcpkg | 2 +- vcpkg-osx-json.diff | 22 ++++++++++++++++++++++ vcpkg.json | 4 ++-- 5 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 vcpkg-osx-json.diff diff --git a/.gitmodules b/.gitmodules index 8e838b6f..a0a57f3d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "vcpkg"] path = vcpkg - url = https://github.com/bewaremypower/vcpkg.git + url = https://github.com/microsoft/vcpkg.git diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index 9524ad47..62440f4c 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -34,6 +34,11 @@ else exit 1 fi +# Use a forked version of vcpkg to support building libcurl with IPv6 disabled +rm -rf vcpkg +git clone https://github.com/BewareMyPower/vcpkg.git -b curl-8.4.0-osx-patch +git apply vcpkg-osx-json.diff + INSTALL_DIR=$PWD/pkg/mac/.install set -x cmake -B build-osx \ diff --git a/vcpkg b/vcpkg index b03173a5..97dd2672 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit b03173a5d0cbd2f7c3c5e85a576e98fba3560835 +Subproject commit 97dd26728e3856ed1ab62ee74ee3a391d9c81d19 diff --git a/vcpkg-osx-json.diff b/vcpkg-osx-json.diff new file mode 100644 index 00000000..53c453a9 --- /dev/null +++ b/vcpkg-osx-json.diff @@ -0,0 +1,22 @@ +diff --git a/vcpkg.json b/vcpkg.json +index 5ff4410..e117563 100644 +--- a/vcpkg.json ++++ b/vcpkg.json +@@ -2,7 +2,7 @@ + "name": "pulsar-cpp", + "version": "3.5.0", + "description": "Pulsar C++ SDK", +- "builtin-baseline": "b051745c68faa6f65c493371d564c4eb8af34dad", ++ "builtin-baseline": "b03173a5d0cbd2f7c3c5e85a576e98fba3560835", + "dependencies": [ + { + "name": "asio", +@@ -25,7 +25,7 @@ + "features": [ + "openssl" + ], +- "version>=": "8.4.0" ++ "version>=": "8.4.0#1" + }, + { + "name": "dlfcn-win32", diff --git a/vcpkg.json b/vcpkg.json index e117563c..5ff44100 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -2,7 +2,7 @@ "name": "pulsar-cpp", "version": "3.5.0", "description": "Pulsar C++ SDK", - "builtin-baseline": "b03173a5d0cbd2f7c3c5e85a576e98fba3560835", + "builtin-baseline": "b051745c68faa6f65c493371d564c4eb8af34dad", "dependencies": [ { "name": "asio", @@ -25,7 +25,7 @@ "features": [ "openssl" ], - "version>=": "8.4.0#1" + "version>=": "8.4.0" }, { "name": "dlfcn-win32", From d29abe08726c6fd04683af489b8113ef250dfde3 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 27 Dec 2024 10:50:49 +0800 Subject: [PATCH 21/23] Apply the patch instead of using a forked vcpkg repo --- pkg/mac/build-static-library.sh | 19 +++++++++++++++---- pkg/mac/vcpkg-curl-patch.diff | 27 +++++++++++++++++++++++++++ vcpkg-osx-json.diff | 22 ---------------------- 3 files changed, 42 insertions(+), 26 deletions(-) create mode 100644 pkg/mac/vcpkg-curl-patch.diff delete mode 100644 vcpkg-osx-json.diff diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index 62440f4c..88b674b4 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -34,10 +34,21 @@ else exit 1 fi -# Use a forked version of vcpkg to support building libcurl with IPv6 disabled -rm -rf vcpkg -git clone https://github.com/BewareMyPower/vcpkg.git -b curl-8.4.0-osx-patch -git apply vcpkg-osx-json.diff +# Apply the patch to support building libcurl with IPv6 disabled +COMMIT_ID=$(grep "builtin-baseline" vcpkg.json | sed 's/"//g' | sed 's/,//' | awk '{print $2}') +cd vcpkg +git reset --hard $COMMIT_ID +git apply ../pkg/mac/vcpkg-curl-patch.diff +git add ports/curl +git commit -m "Disable IPv6 for macOS in curl" +./bootstrap-vcpkg.sh +./vcpkg x-add-version --all +git add versions/ +git commit -m "Update version" +COMMIT_ID=$(git log --pretty=oneline | head -n 1 | awk '{print $1}') +cd .. +sed -i.bak "s/.*builtin-baseline.*/ \"builtin-baseline\": \"$COMMIT_ID\",/" vcpkg.json +sed -i.bak "s/\"version>=\": \"8\.4\.0\"/\"version>=\": \"8.4.0#1\"/" vcpkg.json INSTALL_DIR=$PWD/pkg/mac/.install set -x diff --git a/pkg/mac/vcpkg-curl-patch.diff b/pkg/mac/vcpkg-curl-patch.diff new file mode 100644 index 00000000..e619e29b --- /dev/null +++ b/pkg/mac/vcpkg-curl-patch.diff @@ -0,0 +1,27 @@ +diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake +index bdc544e9e..340d93865 100644 +--- a/ports/curl/portfile.cmake ++++ b/ports/curl/portfile.cmake +@@ -64,6 +64,10 @@ if(VCPKG_TARGET_IS_WINDOWS) + list(APPEND OPTIONS -DENABLE_UNICODE=ON) + endif() + ++if(VCPKG_TARGET_IS_OSX) ++ list(APPEND OPTIONS -DENABLE_IPV6=OFF) ++endif() ++ + vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS +diff --git a/ports/curl/vcpkg.json b/ports/curl/vcpkg.json +index e028d3897..a63858e34 100644 +--- a/ports/curl/vcpkg.json ++++ b/ports/curl/vcpkg.json +@@ -1,6 +1,7 @@ + { + "name": "curl", + "version": "8.4.0", ++ "port-version": 1, + "description": "A library for transferring data with URLs", + "homepage": "https://curl.se/", + "license": "curl AND ISC AND BSD-3-Clause", diff --git a/vcpkg-osx-json.diff b/vcpkg-osx-json.diff deleted file mode 100644 index 53c453a9..00000000 --- a/vcpkg-osx-json.diff +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/vcpkg.json b/vcpkg.json -index 5ff4410..e117563 100644 ---- a/vcpkg.json -+++ b/vcpkg.json -@@ -2,7 +2,7 @@ - "name": "pulsar-cpp", - "version": "3.5.0", - "description": "Pulsar C++ SDK", -- "builtin-baseline": "b051745c68faa6f65c493371d564c4eb8af34dad", -+ "builtin-baseline": "b03173a5d0cbd2f7c3c5e85a576e98fba3560835", - "dependencies": [ - { - "name": "asio", -@@ -25,7 +25,7 @@ - "features": [ - "openssl" - ], -- "version>=": "8.4.0" -+ "version>=": "8.4.0#1" - }, - { - "name": "dlfcn-win32", From 8833d1fcdfa1a7411f93ea79ddf51b8308543d36 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 27 Dec 2024 10:52:58 +0800 Subject: [PATCH 22/23] Remove the workflow with brew dependencies --- .github/workflows/ci-pr-validation.yaml | 27 ------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 0cc2f7ae..fac5ab69 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -311,33 +311,6 @@ jobs: run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest cpp-build-macos: - timeout-minutes: 120 - name: Build CPP Client on macOS - needs: formatting-check - runs-on: macos-14 - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Install dependencies - run: brew install openssl protobuf boost zstd snappy googletest - - - name: Configure (default) - shell: bash - run: cmake -B ./build-macos -S . - - - name: Compile - shell: bash - run: | - cmake --build ./build-macos --parallel --config Release - - - name: Build with C++20 - shell: bash - run: | - cmake -B build-macos-cpp20 -DCMAKE_CXX_STANDARD=20 - cmake --build build-macos-cpp20 -j8 - - cpp-build-macos-static: timeout-minutes: 120 name: Build CPP Client on macOS with static dependencies runs-on: macos-14 From b5a2a0ddbbe58d9ac9be0229698ea87d968134b6 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 27 Dec 2024 11:03:25 +0800 Subject: [PATCH 23/23] Fetch the submodule in CI --- .github/workflows/ci-build-binary-artifacts.yaml | 3 +++ .github/workflows/ci-pr-validation.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 5b9dd52f..23df0805 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -208,6 +208,9 @@ jobs: steps: - name: checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive - name: Install dependencies run: | diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index fac5ab69..5cb18df9 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -318,6 +318,9 @@ jobs: steps: - name: checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive - name: Build libraries run: ./pkg/mac/build-static-library.sh