(feat) add support for musl using cross-rs (#2536)

* add cross configuration for musl Signed-off-by: Jorge Prendes <[email protected]> * use cargo.sh wapper in build.sh Signed-off-by: Jorge Prendes <[email protected]> * make tests build with musl Signed-off-by: Jorge Prendes <[email protected]> * add targets to run musl tests Signed-off-by: Jorge Prendes <[email protected]> * Use cargo.sh wrapper and cross in CI Signed-off-by: Jorge Prendes <[email protected]> * Update scripts/cargo.sh Co-Authored-By: adrianalin <[email protected]> Signed-off-by: Jorge Prendes <[email protected]> * Use glibc as cross default target --------- Signed-off-by: Jorge Prendes <[email protected]> Co-authored-by: adrianalin <[email protected]>
youki-dev · Dec 21, 2023 · ff79c54 · ff79c54
1 parent 72c56f2
commit ff79c54
Show file tree

Hide file tree

Showing 13 changed files with 240 additions and 138 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -15,33 +15,38 @@ jobs:
     strategy:
       matrix:
         arch: [ "x86_64", "aarch64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/[email protected]
         env:
-          RUST_CACHE_KEY_OS: rust-cache-${{ matrix.arch }}
+          RUST_CACHE_KEY_OS: rust-cache-${{ matrix.arch }}-${{ matrix.libc }}
       - name: Install just
         uses: taiki-e/install-action@just
-      - name: Install requirements
-        if: ${{ matrix.arch == 'x86_64' }}
-        run: sudo env PATH=$PATH arch=${{ matrix.arch }} just ci-prepare
-      - name: Install requirements for aarch64
-        if: ${{ matrix.arch == 'aarch64' }}
+      - name: Install cross-rs
         run: cargo install cross --git https://github.com/cross-rs/cross
+      - name: Setup target
+        run: |
+          echo "CARGO=cross" >> ${GITHUB_ENV}
+          echo "TARGET=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}" >> ${GITHUB_ENV}
       - name: Build youki
-        run: arch=${{ matrix.arch }} just youki-release
+        run: just youki-release
       - name: Upload youki binary
         if: ${{ matrix.arch == 'x86_64' }}
         uses: actions/upload-artifact@v3
         with:
-          name: youki
+          name: youki-${{ matrix.arch }}-${{ matrix.libc }}
           path: youki
 
   containerd-integration-tests:
     runs-on: ubuntu-22.04
     needs: [youki-build]
     timeout-minutes: 40
+    strategy:
+      matrix:
+        arch: [ "x86_64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -60,7 +65,7 @@ jobs:
       - name: Download youki binary
         uses: actions/download-artifact@v3
         with:
-          name: youki
+          name: youki-${{ matrix.arch }}-${{ matrix.libc }}
       - name: Replace runc to youki
         run: |
           sudo rm -f /usr/bin/runc /usr/local/bin/runc /usr/sbin/runc
@@ -74,12 +79,16 @@ jobs:
     runs-on: ubuntu-22.04
     needs: [youki-build]
     timeout-minutes: 40
+    strategy:
+      matrix:
+        arch: [ "x86_64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
       - name: Download youki binary
         uses: actions/download-artifact@v3
         with:
-          name: youki
+          name: youki-${{ matrix.arch }}-${{ matrix.libc }}
       - name: Add the permission to run
         run: chmod +x ./youki
       - name: Install just
@@ -91,6 +100,10 @@ jobs:
     runs-on: ubuntu-22.04
     needs: [youki-build]
     timeout-minutes: 15
+    strategy:
+      matrix:
+        arch: [ "x86_64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -107,7 +120,7 @@ jobs:
       - name: Download youki binary
         uses: actions/download-artifact@v3
         with:
-          name: youki
+          name: youki-${{ matrix.arch }}-${{ matrix.libc }}
       - name: Add the permission to run
         run: chmod +x ./youki
       - name: Run integration tests
@@ -117,6 +130,10 @@ jobs:
     runs-on: ubuntu-22.04
     needs: [youki-build]
     timeout-minutes: 20
+    strategy:
+      matrix:
+        arch: [ "x86_64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -130,7 +147,7 @@ jobs:
       - name: Download youki binary
         uses: actions/download-artifact@v3
         with:
-          name: youki
+          name: youki-${{ matrix.arch }}-${{ matrix.libc }}
       - name: Add the permission to run
         run: chmod +x ./youki
       - name: Build
@@ -142,6 +159,10 @@ jobs:
     runs-on: ubuntu-22.04
     needs: [youki-build]
     timeout-minutes: 20
+    strategy:
+      matrix:
+        arch: [ "x86_64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -155,7 +176,7 @@ jobs:
       - name: Download youki binary
         uses: actions/download-artifact@v3
         with:
-          name: youki
+          name: youki-${{ matrix.arch }}-${{ matrix.libc }}
       - name: Add the permission to run
         run: chmod +x ./youki
       - name: Run tests

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -35,21 +35,26 @@ jobs:
     if: needs.changes.outputs.any_modified == 'true'
     runs-on: ubuntu-22.04
     timeout-minutes: 15
+    strategy:
+      matrix:
+        arch: [ "x86_64", "aarch64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           components: rustfmt, clippy
-
       - name: typos-action
         uses: crate-ci/[email protected]
-
-      - uses: taiki-e/install-action@just
-
-      - name: Install requirements
-        run: sudo env PATH=$PATH just ci-prepare
-
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install cross-rs
+        run: cargo install cross --git https://github.com/cross-rs/cross
+      - name: Setup target
+        run: |
+          echo "CARGO=cross" >> ${GITHUB_ENV}
+          echo "TARGET=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}" >> ${GITHUB_ENV}
       - name: Check formatting and lints
         run: just lint
 
@@ -58,46 +63,27 @@ jobs:
     if: needs.changes.outputs.any_modified == 'true'
     runs-on: ubuntu-22.04
     timeout-minutes: 20
+    strategy:
+      matrix:
+        arch: [ "x86_64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1
-      - uses: taiki-e/install-action@just
-      - name: Install requirements
-        run: sudo env PATH=$PATH just ci-prepare
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install cross-rs
+        run: cargo install cross --git https://github.com/cross-rs/cross
+      - name: Setup target
+        run: |
+          echo "CARGO=cross" >> ${GITHUB_ENV}
+          echo "TARGET=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}" >> ${GITHUB_ENV}
       - name: Run tests
         run: just test-basic
       - name: Run feature tests
         run: just test-features
 
-  # TODO: musl testing is flaky. Turn it back on when we can resolve it.
-  # # musl target testing is religated to a separate job because our current build
-  # # for musl requires nightly. It is difficult to mix stable and nightly
-  # # toolchains in the same job.
-  # musl:
-  #   needs: [changes]
-  #   if: needs.changes.outputs.any_modified == 'true'
-  #   runs-on: ubuntu-22.04
-  #   timeout-minutes: 20
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     # We do not use `actions-rust-lang/setup-rust-toolchain` here because we
-  #     # want to override the default toolchain file to use nightly toolchains.
-  #     # The `setup-rust-toolchain` action will always choose toolchain file with
-  #     # no way to override. 
-  #     - name: Setup Rust toolchain and cache
-  #       uses: dtolnay/rust-toolchain@v1
-  #       with:
-  #         toolchain: nightly
-  #         target: x86_64-unknown-linux-musl
-  #         components: rust-src
-  #     - uses: taiki-e/install-action@just
-  #     - name: Install requirements
-  #       run: sudo env PATH=$PATH just ci-musl-prepare
-  #     - name: Run test against musl target
-  #       run: just test-musl
-
-
   coverage:
     needs: [changes]
     if: needs.changes.outputs.any_modified == 'true'

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -24,25 +24,25 @@ jobs:
     strategy:
       matrix:
         arch: [ "x86_64", "aarch64" ]
+        libc: [ "gnu", "musl" ]
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/[email protected]
         env:
-          RUST_CACHE_KEY_OS: rust-cache-${{ matrix.arch }}
+          RUST_CACHE_KEY_OS: rust-cache-${{ matrix.arch }}-${{ matrix.libc }}
       - name: Install just
         uses: taiki-e/install-action@just
-      - name: Install requirements
-        if: ${{ matrix.arch == 'x86_64' }}
-        run: | 
-          sudo env PATH=$PATH arch=${{ matrix.arch }} just ci-prepare
-      - name: Install requirements for aarch64
-        if: ${{ matrix.arch == 'aarch64' }}
+      - name: Install cross-rs
         run: cargo install cross --git https://github.com/cross-rs/cross
+      - name: Setup target
+        run: |
+          echo "CARGO=cross" >> ${GITHUB_ENV}
+          echo "TARGET=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}" >> ${GITHUB_ENV}
       - name: Release build
-        run: arch=${{ matrix.arch }} just youki-release
+        run: just youki-release
       - name: test
         # TODO(utam0k): The feature test needs nightly
         # run: just test-basic featuretest test-oci
@@ -51,12 +51,12 @@ jobs:
       - name: Create output directory
         run: mkdir output
       - name: Create artifact
-        run: tar -zcvf youki-${{ needs.parse.outputs.version }}-${{ matrix.arch }}.tar.gz youki README.md LICENSE
+        run: tar -zcvf youki-${{ needs.parse.outputs.version }}-${{ matrix.arch }}-${{ matrix.libc }}.tar.gz youki README.md LICENSE
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
-          name: youki-${{ matrix.arch }}
-          path: youki-${{ needs.parse.outputs.version }}-${{ matrix.arch }}.tar.gz
+          name: youki-${{ matrix.arch }}-${{ matrix.libc }}
+          path: youki-${{ needs.parse.outputs.version }}-${{ matrix.arch }}-${{ matrix.libc }}.tar.gz
 
   release:
     name: Create Draft Release
@@ -74,6 +74,8 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           path: artifacts
+      - name: Show artifacts
+        run: ls -alhR artifacts
       - name: Create release draft
         shell: bash
         run: |

diff --git a/Cross.toml b/Cross.toml
@@ -6,3 +6,9 @@ dockerfile = "cross/Dockerfile.gnu"
 
 [target.x86_64-unknown-linux-gnu]
 dockerfile = "cross/Dockerfile.gnu"
+
+[target.aarch64-unknown-linux-musl]
+dockerfile = "cross/Dockerfile.musl"
+
+[target.x86_64-unknown-linux-musl]
+dockerfile = "cross/Dockerfile.musl"
diff --git a/crates/libcgroups/src/v2/devices/mocks.rs b/crates/libcgroups/src/v2/devices/mocks.rs
@@ -4,10 +4,15 @@
 
 #[cfg_attr(test, automock())]
 pub mod libc {
-    pub fn setrlimit(
-        _resource: libc::__rlimit_resource_t,
-        _rlim: *const libc::rlimit,
-    ) -> libc::c_int {
+    #[cfg(target_env = "musl")]
+    #[allow(non_camel_case_types)]
+    pub type __rlimit_resource_t = libc::c_int;
+
+    #[cfg(not(target_env = "musl"))]
+    #[allow(non_camel_case_types)]
+    pub type __rlimit_resource_t = libc::__rlimit_resource_t;
+
+    pub fn setrlimit(_resource: __rlimit_resource_t, _rlim: *const libc::rlimit) -> libc::c_int {
         unimplemented!();
     }
 }

diff --git a/cross/Dockerfile.gnu b/cross/Dockerfile.gnu
@@ -5,4 +5,11 @@ FROM $CROSS_BASE_IMAGE
 ARG CROSS_DEB_ARCH
 RUN dpkg --add-architecture ${CROSS_DEB_ARCH} && \
     apt-get -y update && \
-    apt-get install -y pkg-config libseccomp-dev:${CROSS_DEB_ARCH}
+    apt-get install -y pkg-config \
+    # dependencies required to build libsecccomp-rs
+    libseccomp-dev:${CROSS_DEB_ARCH} \
+    # dependencies required to build libbpf-sys
+    libelf-dev:${CROSS_DEB_ARCH} \
+    zlib1g-dev:${CROSS_DEB_ARCH} \
+    # dependencies to build wasmedge-sys
+    libzstd-dev:${CROSS_DEB_ARCH}
diff --git a/cross/Dockerfile.musl b/cross/Dockerfile.musl
@@ -0,0 +1,32 @@
+ARG CROSS_BASE_IMAGE
+FROM $CROSS_BASE_IMAGE
+
+COPY --from=jorgeprendes420/apk-anywhere / /
+ENV MARCH=${CROSS_CMAKE_SYSTEM_PROCESSOR}
+RUN apk-init ${MARCH} ${CROSS_SYSROOT}
+
+RUN apk-${MARCH} add \
+    # dependencies required to build libsecccomp-rs
+    libseccomp-static libseccomp-dev \
+    # dependencies required to build libbpf-sys
+    libelf-static elfutils-dev \
+    zlib-dev zlib-static \
+    # dependencies to build wasmedge-sys
+    g++ zstd-static
+
+# configure libsecccomp-rs to use static linking
+ENV LIBSECCOMP_LINK_TYPE="static"
+ENV LIBSECCOMP_LIB_PATH="${CROSS_SYSROOT}/lib"    
+
+# configure wasmedge-sys to link stdc++ statically
+ENV WASMEDGE_DEP_STDCXX_LINK_TYPE="static"
+ENV WASMEDGE_DEP_STDCXX_LIB_PATH="${CROSS_SYSROOT}/lib"
+
+# wasmedge-sys (through llvm) needs some symbols defined in libgcc
+RUN mkdir /.cargo && cat <<'EOF' > /.cargo/config.toml
+[target.'cfg(target_env = "musl")']
+rustflags = ["-Clink-arg=-lgcc"]
+EOF
+
+RUN apt-get -y update && \
+    apt-get install -y pkg-config