From c5f0449146179aa676765efcdad77169778f1b30 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 26 Jan 2024 17:08:24 -0500 Subject: [PATCH 1/2] feat: first-class musl support --- .cross_rubies | 48 +++++++++++------ Gemfile | 2 +- rakelib/extensions.rake | 112 +++++++++++++++------------------------- 3 files changed, 76 insertions(+), 86 deletions(-) diff --git a/.cross_rubies b/.cross_rubies index fa47675a4f..4244f7b1d5 100644 --- a/.cross_rubies +++ b/.cross_rubies @@ -1,32 +1,48 @@ -3.0.0:aarch64-linux -3.0.0:arm-linux +3.0.0:aarch64-linux-gnu +3.0.0:aarch64-linux-musl +3.0.0:arm-linux-gnu +3.0.0:arm-linux-musl 3.0.0:arm64-darwin 3.0.0:x64-mingw32 -3.0.0:x86-linux +3.0.0:x86-linux-gnu +3.0.0:x86-linux-musl 3.0.0:x86-mingw32 3.0.0:x86_64-darwin -3.0.0:x86_64-linux -3.1.0:aarch64-linux -3.1.0:arm-linux +3.0.0:x86_64-linux-gnu +3.0.0:x86_64-linux-musl +3.1.0:aarch64-linux-gnu +3.1.0:aarch64-linux-musl +3.1.0:arm-linux-gnu +3.1.0:arm-linux-musl 3.1.0:arm64-darwin 3.1.0:x64-mingw-ucrt -3.1.0:x86-linux +3.1.0:x86-linux-gnu +3.1.0:x86-linux-musl 3.1.0:x86-mingw32 3.1.0:x86_64-darwin -3.1.0:x86_64-linux -3.2.0:aarch64-linux -3.2.0:arm-linux +3.1.0:x86_64-linux-gnu +3.1.0:x86_64-linux-musl +3.2.0:aarch64-linux-gnu +3.2.0:aarch64-linux-musl +3.2.0:arm-linux-gnu +3.2.0:arm-linux-musl 3.2.0:arm64-darwin 3.2.0:x64-mingw-ucrt -3.2.0:x86-linux +3.2.0:x86-linux-gnu +3.2.0:x86-linux-musl 3.2.0:x86-mingw32 3.2.0:x86_64-darwin -3.2.0:x86_64-linux -3.3.0:aarch64-linux -3.3.0:arm-linux +3.2.0:x86_64-linux-gnu +3.2.0:x86_64-linux-musl +3.3.0:aarch64-linux-gnu +3.3.0:aarch64-linux-musl +3.3.0:arm-linux-gnu +3.3.0:arm-linux-musl 3.3.0:arm64-darwin 3.3.0:x64-mingw-ucrt -3.3.0:x86-linux +3.3.0:x86-linux-gnu +3.3.0:x86-linux-musl 3.3.0:x86-mingw32 3.3.0:x86_64-darwin -3.3.0:x86_64-linux +3.3.0:x86_64-linux-gnu +3.3.0:x86_64-linux-musl diff --git a/Gemfile b/Gemfile index e63c8740fe..b8d197976a 100644 --- a/Gemfile +++ b/Gemfile @@ -17,7 +17,7 @@ group :development do # building extensions gem "rake-compiler", "1.2.7" - gem "rake-compiler-dock", "1.4.0" + gem "rake-compiler-dock", "1.5.0.rc1" # parser generator gem "rexical", "= 1.0.7" diff --git a/rakelib/extensions.rake b/rakelib/extensions.rake index cb2881436a..d11527c8e5 100644 --- a/rakelib/extensions.rake +++ b/rakelib/extensions.rake @@ -8,9 +8,12 @@ CrossRuby = Struct.new(:version, :platform) do MINGWUCRT_PLATFORM_REGEX = /mingw-ucrt/ MINGW32_PLATFORM_REGEX = /mingw32/ LINUX_PLATFORM_REGEX = /linux/ - X86_LINUX_PLATFORM_REGEX = /x86.*linux/ - AARCH_LINUX_PLATFORM_REGEX = /aarch.*linux/ - ARM_LINUX_PLATFORM_REGEX = /arm-linux/ + X86_LINUX_GNU_PLATFORM_REGEX = /x86.*linux-gnu$/ + X86_LINUX_MUSL_PLATFORM_REGEX = /x86.*linux-musl$/ + AARCH_LINUX_GNU_PLATFORM_REGEX = /aarch.*linux-gnu$/ + AARCH_LINUX_MUSL_PLATFORM_REGEX = /aarch.*linux-musl$/ + ARM_LINUX_GNU_PLATFORM_REGEX = /arm-linux-gnu$/ + ARM_LINUX_MUSL_PLATFORM_REGEX = /arm-linux-musl$/ DARWIN_PLATFORM_REGEX = /darwin/ def windows? @@ -42,47 +45,20 @@ CrossRuby = Struct.new(:version, :platform) do end end - def host - @host ||= case platform - when "x64-mingw-ucrt" - "x86_64-w64-mingw32" - when "x64-mingw32" - "x86_64-w64-mingw32" - when "x86-mingw32" - "i686-w64-mingw32" - when "x86_64-linux" - "x86_64-linux-gnu" - when "x86-linux" - "i686-linux-gnu" - when "aarch64-linux" - "aarch64-linux" - when "x86_64-darwin" - "x86_64-darwin" - when "arm64-darwin" - "aarch64-darwin" - else - raise "CrossRuby.platform: unsupported platform: #{platform}" - end - end - def tool(name) (@binutils_prefix ||= case platform - when "x64-mingw-ucrt", "x64-mingw32" - "x86_64-w64-mingw32-" - when "x86-mingw32" - "i686-w64-mingw32-" - when "x86_64-linux" - "x86_64-redhat-linux-" - when "x86-linux" - "i686-redhat-linux-" - when "aarch64-linux" - "aarch64-linux-gnu-" - when "x86_64-darwin" - "x86_64-apple-darwin-" - when "arm64-darwin" - "aarch64-apple-darwin-" - when "arm-linux" - "arm-linux-gnueabihf-" + when "aarch64-linux-gnu" then "aarch64-linux-gnu-" + when "aarch64-linux-musl" then "aarch64-linux-musl-" + when "arm-linux-gnu" then "arm-linux-gnueabihf-" + when "arm-linux-musl" then "arm-linux-musleabihf-" + when "arm64-darwin" then "aarch64-apple-darwin-" + when "x64-mingw-ucrt", "x64-mingw32" then "x86_64-w64-mingw32-" + when "x86-linux-gnu" then "i686-redhat-linux-gnu-" + when "x86-linux-musl" then "i686-unknown-linux-musl-" + when "x86-mingw32" then "i686-w64-mingw32-" + when "x86_64-darwin" then "x86_64-apple-darwin-" + when "x86_64-linux-gnu" then "x86_64-redhat-linux-gnu-" + when "x86_64-linux-musl" then "x86_64-unknown-linux-musl-" else raise "CrossRuby.tool: unmatched platform: #{platform}" end) + name @@ -90,22 +66,14 @@ CrossRuby = Struct.new(:version, :platform) do def target_file_format case platform - when "x64-mingw-ucrt", "x64-mingw32" - "pei-x86-64" - when "x86-mingw32" - "pei-i386" - when "x86_64-linux" - "elf64-x86-64" - when "x86-linux" - "elf32-i386" - when "aarch64-linux" - "elf64-littleaarch64" - when "x86_64-darwin" - "Mach-O 64-bit x86-64" # hmm - when "arm64-darwin" - "Mach-O arm64" - when "arm-linux" - "elf32-littlearm" + when "aarch64-linux-gnu", "aarch64-linux-musl" then "elf64-littleaarch64" + when "arm-linux-gnu", "arm-linux-musl" then "elf32-littlearm" + when "arm64-darwin" then "Mach-O arm64" + when "x64-mingw-ucrt", "x64-mingw32" then "pei-x86-64" + when "x86-linux-gnu", "x86-linux-musl" then "elf32-i386" + when "x86-mingw32" then "pei-i386" + when "x86_64-darwin" then "Mach-O 64-bit x86-64" # hmm + when "x86_64-linux-gnu", "x86_64-linux-musl" then "elf64-x86-64" else raise "CrossRuby.target_file_format: unmatched platform: #{platform}" end @@ -162,7 +130,17 @@ CrossRuby = Struct.new(:version, :platform) do "api-ms-win-crt-utility-l1-1-0.dll", libruby_dll, ] - when X86_LINUX_PLATFORM_REGEX + when DARWIN_PLATFORM_REGEX + [ + "/usr/lib/libSystem.B.dylib", + "/usr/lib/liblzma.5.dylib", + "/usr/lib/libobjc.A.dylib", + ] + when X86_LINUX_MUSL_PLATFORM_REGEX, ARM_LINUX_MUSL_PLATFORM_REGEX, AARCH_LINUX_MUSL_PLATFORM_REGEX + [ + "libc.so", + ] + when X86_LINUX_GNU_PLATFORM_REGEX [ "libm.so.6", "libc.so.6", @@ -170,7 +148,7 @@ CrossRuby = Struct.new(:version, :platform) do ].tap do |dlls| dlls << "libpthread.so.0" if ver >= "3.2.0" end - when AARCH_LINUX_PLATFORM_REGEX + when AARCH_LINUX_GNU_PLATFORM_REGEX [ "libm.so.6", "libc.so.6", @@ -179,13 +157,7 @@ CrossRuby = Struct.new(:version, :platform) do ].tap do |dlls| dlls << "libpthread.so.0" if ver >= "3.2.0" end - when DARWIN_PLATFORM_REGEX - [ - "/usr/lib/libSystem.B.dylib", - "/usr/lib/liblzma.5.dylib", - "/usr/lib/libobjc.A.dylib", - ] - when ARM_LINUX_PLATFORM_REGEX + when ARM_LINUX_GNU_PLATFORM_REGEX [ "libm.so.6", "libdl.so.2", @@ -201,10 +173,12 @@ CrossRuby = Struct.new(:version, :platform) do def dll_ref_versions case platform - when X86_LINUX_PLATFORM_REGEX + when X86_LINUX_GNU_PLATFORM_REGEX { "GLIBC" => "2.17" } - when AARCH_LINUX_PLATFORM_REGEX, ARM_LINUX_PLATFORM_REGEX + when AARCH_LINUX_GNU_PLATFORM_REGEX, ARM_LINUX_GNU_PLATFORM_REGEX { "GLIBC" => "2.29" } + when X86_LINUX_MUSL_PLATFORM_REGEX, AARCH_LINUX_MUSL_PLATFORM_REGEX, ARM_LINUX_MUSL_PLATFORM_REGEX + {} else raise "CrossRuby.dll_ref_versions: unmatched platform: #{platform}" end From 94592d2753097ef4eb09a4a992cab7853b7c3e8a Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 26 Jan 2024 17:28:35 -0500 Subject: [PATCH 2/2] ci: update pipeline to test linux-musl --- .github/workflows/ci.yml | 162 ++++++++++++++++----------------------- 1 file changed, 66 insertions(+), 96 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0203838e5c..d8f1594007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -440,12 +440,10 @@ jobs: - uses: actions/cache@v4 with: path: ports/archives - key: tarballs-ubuntu-${{hashFiles('dependencies.yml', 'patches/**/*.patch')}} - - env: - DOCKER_IMAGE: "ghcr.io/rake-compiler/rake-compiler-dock-image:${{needs.rcd_image_version.outputs.rcd_image_version}}-mri-x86_64-linux" - run: | - docker run --rm -v "$(pwd):/nokogiri" -w /nokogiri \ - ${DOCKER_IMAGE} \ + key: tarballs-ubuntu-${{ hashFiles('dependencies.yml', 'patches/**/*.patch') }} + - run: | + docker run --rm -v $PWD:/nokogiri -w /nokogiri \ + ghcr.io/rake-compiler/rake-compiler-dock-image:${{ needs.rcd_image_version.outputs.rcd_image_version }}-mri-x86_64-linux-gnu \ ./scripts/test-gem-build gems ruby - uses: actions/upload-artifact@v4 with: @@ -549,15 +547,19 @@ jobs: fail-fast: false matrix: plat: - - "aarch64-linux" - - "arm-linux" + - "aarch64-linux-gnu" + - "aarch64-linux-musl" + - "arm-linux-gnu" + - "arm-linux-musl" - "arm64-darwin" # github actions does not support this runtime as of 2022-12, but let's build anyway - "x64-mingw-ucrt" - "x64-mingw32" - - "x86-linux" + - "x86-linux-gnu" + - "x86-linux-musl" - "x86-mingw32" # github actions does not support this runtime as of 2022-12, but let's build anyway - "x86_64-darwin" - - "x86_64-linux" + - "x86_64-linux-gnu" + - "x86_64-linux-musl" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -567,68 +569,70 @@ jobs: with: path: ports/archives key: tarballs-ubuntu-${{hashFiles('dependencies.yml', 'patches/**/*.patch')}} - - env: - DOCKER_IMAGE: "ghcr.io/rake-compiler/rake-compiler-dock-image:${{needs.rcd_image_version.outputs.rcd_image_version}}-mri-${{matrix.plat}}" - run: | - docker run --rm -v "$(pwd):/nokogiri" -w /nokogiri \ - ${DOCKER_IMAGE} \ - ./scripts/test-gem-build gems ${{matrix.plat}} + - run: | + docker run --rm -v $PWD:/nokogiri -w /nokogiri \ + ghcr.io/rake-compiler/rake-compiler-dock-image:${{ needs.rcd_image_version.outputs.rcd_image_version }}-mri-${{ matrix.plat }} \ + ./scripts/test-gem-build gems ${{ matrix.plat }} - uses: actions/upload-artifact@v4 with: - name: "cruby-${{matrix.plat}}-gem" + name: "cruby-${{ matrix.plat }}-gem" path: gems retention-days: 1 - cruby-x86-linux-install: + test_native_gem_with_setup_ruby: needs: ["cruby-package"] + name: "${{ matrix.platform }} setup-ruby(${{ matrix.ruby }})" strategy: fail-fast: false matrix: + platform: ["x86_64-linux-gnu"] ruby: ["3.0", "3.1", "3.2", "3.3"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: true - - uses: actions/download-artifact@v4 - with: - name: cruby-x86-linux-gem - path: gems - - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --rm -v "$(pwd):/nokogiri" -w /nokogiri \ - --platform=linux/386 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install gems - - cruby-aarch64-linux-install: - needs: ["cruby-package"] - strategy: - fail-fast: false - matrix: - ruby: ["3.0", "3.1", "3.2", "3.3"] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 with: - submodules: true + ruby-version: "${{ matrix.ruby }}" + rubygems: latest - uses: actions/download-artifact@v4 with: - name: cruby-aarch64-linux-gem + name: cruby-${{ matrix.platform }}-gem path: gems - - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --rm -v "$(pwd):/nokogiri" -w /nokogiri \ - --platform=linux/arm64/v8 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install gems + - run: ./scripts/test-gem-install gems - cruby-arm-linux-install: + test_native_gem_matrix: needs: ["cruby-package"] + name: "${{ matrix.platform }} ${{ matrix.ruby }}" strategy: fail-fast: false matrix: + platform: + - aarch64-linux-gnu + - aarch64-linux-musl + - arm-linux-gnu + - arm-linux-musl + - x86-linux-gnu + - x86-linux-musl + - x86_64-linux-gnu + - x86_64-linux-musl ruby: ["3.0", "3.1", "3.2", "3.3"] + include: + # declare docker image for each platform + - { platform: aarch64-linux-musl, docker_tag: "-alpine", bootstrap: "apk add bash build-base &&" } + - { platform: arm-linux-musl, docker_tag: "-alpine", bootstrap: "apk add bash build-base &&" } + - { platform: x86-linux-musl, docker_tag: "-alpine", bootstrap: "apk add bash build-base &&" } + - { platform: x86_64-linux-musl, docker_tag: "-alpine", bootstrap: "apk add bash build-base &&" } + # declare docker platform for each platform + - { platform: aarch64-linux-gnu, docker_platform: "--platform=linux/arm64/v8" } + - { platform: aarch64-linux-musl, docker_platform: "--platform=linux/arm64/v8" } + - { platform: arm-linux-gnu, docker_platform: "--platform=linux/arm/v7" } + - { platform: arm-linux-musl, docker_platform: "--platform=linux/arm/v7" } + - { platform: x86-linux-gnu, docker_platform: "--platform=linux/386" } + - { platform: x86-linux-musl, docker_platform: "--platform=linux/386" } + # make sure we have a compatible version of rubygems + - { ruby: "3.0", rubyenv: "gem update --system &&" } runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -636,51 +640,17 @@ jobs: submodules: true - uses: actions/download-artifact@v4 with: - name: cruby-arm-linux-gem + name: cruby-${{ matrix.platform }}-gem path: gems - run: | docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --rm -v "$(pwd):/nokogiri" -w /nokogiri \ - --platform=linux/arm/v7 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install gems - - cruby-x86_64-linux-install: - needs: ["cruby-package"] - strategy: - fail-fast: false - matrix: - ruby: ["3.0", "3.1", "3.2", "3.3"] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "${{matrix.ruby}}" - - uses: actions/download-artifact@v4 - with: - name: cruby-x86_64-linux-gem - path: gems - - run: ./scripts/test-gem-install gems - - cruby-x86_64-musl-install: - needs: ["cruby-package"] - strategy: - fail-fast: false - runs-on: ubuntu-latest - container: - image: ghcr.io/sparklemotion/nokogiri-test:alpine - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: actions/download-artifact@v4 - with: - name: cruby-x86_64-linux-gem - path: gems - - run: ./scripts/test-gem-install gems + docker run --rm -v $PWD:/nokogiri -w /nokogiri \ + ${{ matrix.docker_platform }} ruby:${{ matrix.ruby }}${{ matrix.docker_tag }} \ + sh -c " + ${{ matrix.bootstrap }} + ${{ matrix.rubyenv }} + ./scripts/test-gem-install gems + " cruby-x86_64-darwin-install: needs: ["cruby-package"] @@ -695,7 +665,7 @@ jobs: submodules: true - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" - uses: actions/download-artifact@v4 with: name: cruby-x86_64-darwin-gem @@ -715,7 +685,7 @@ jobs: submodules: true - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" - uses: actions/download-artifact@v4 with: name: cruby-x64-mingw32-gem @@ -736,7 +706,7 @@ jobs: submodules: true - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" - uses: actions/download-artifact@v4 with: name: cruby-x64-mingw-ucrt-gem @@ -748,7 +718,7 @@ jobs: needs: ["rcd_image_version"] runs-on: ubuntu-latest container: - image: "ghcr.io/rake-compiler/rake-compiler-dock-image:${{needs.rcd_image_version.outputs.rcd_image_version}}-jruby" + image: "ghcr.io/rake-compiler/rake-compiler-dock-image:${{ needs.rcd_image_version.outputs.rcd_image_version }}-jruby" steps: - uses: actions/checkout@v4 with: @@ -767,7 +737,7 @@ jobs: matrix: jruby: ["9.4"] jre: ["8", "11"] - name: "jruby-${{matrix.jruby}}-jre${{matrix.jre}}-install" + name: "jruby-${{ matrix.jruby }}-jre${{ matrix.jre }}-install" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -778,6 +748,6 @@ jobs: name: jruby-gem path: gems - run: | - docker run --rm -v "$(pwd):/nokogiri" -w /nokogiri \ - jruby:${{matrix.jruby}}-jre${{matrix.jre}} \ + docker run --rm -v $PWD:/nokogiri -w /nokogiri \ + jruby:${{ matrix.jruby }}-jre${{ matrix.jre }} \ ./scripts/test-gem-install gems