Merge pull request #56 from nebius/build_pyxis_only

Workflow for building pyxis only

.github/workflows/slurm_packages_and_pyxis.yml → .github/workflows/pyxis.yml

@@ -1,4 +1,4 @@
-name: Build slurm packages
+name: Build pyxis
 
 on:
   workflow_dispatch:
@@ -17,15 +17,17 @@ jobs:
       matrix:
         slurm:
           - version: 24.05.5
+        enroot:
+          - version: 3.5.0
+        pyxis:
+          - vesion: 0.21.0
         image:
-          - context: .
+          - file: pyxis/Dockerfile
             push: false
             platforms: linux/amd64
             load: true
-            ubuntu_version: jammy
-            cuda_version: 12.4.1
             build_args:
-              BASE_IMAGE=nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+              BASE_IMAGE=ubuntu:22.04
 
     steps:
       - name: Harden Runner
@@ -42,42 +44,42 @@ jobs:
       - name: Build docker images
         uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
         with:
-          context: ${{ matrix.image.context }}
+          file: ${{ matrix.image.file }}
           push: ${{ matrix.image.push }}
-          tags: slurm_builder:${{ matrix.image.cuda_version }}-${{ matrix.image.ubuntu_version }}-slurm${{ matrix.slurm.version }}
+          tags: pyxis_builder_slurm_${{ matrix.slurm.version }}_enroot_${{ matrix.enroot.version }}:${{ matrix.pyxis.vesion }}
           platforms: ${{ matrix.image.platforms }}
           load: ${{ matrix.image.load }}
           build-args: |
             ${{ matrix.image.build_args }}
             SLURM_VERSION=${{ matrix.slurm.version }}
+            ENROOT_VERSION=${{ matrix.enroot.version }}
+            PYXIS_VERSION=${{ matrix.pyxis.version }}
           cache-from: type=local,src=.buildx-cache
           cache-to: type=local,dest=.buildx-cache,mode=max
 
-      - name: Create slurm_build_output directory
-        run: mkdir -p slurm_build_output
+      - name: Create pyxis_build_output directory
+        run: mkdir -p pyxis_build_output
 
       - name: Run Docker container and copy files
         run: |
-          container_id=$(docker create slurm_builder:${{ matrix.image.cuda_version }}-${{ matrix.image.ubuntu_version }}-slurm${{ matrix.slurm.version }})
+          container_id=$(docker pyxis_builder_slurm_${{ matrix.slurm.version }}_enroot_${{ matrix.enroot.version }}:${{ matrix.pyxis.vesion }})
           docker start $container_id
-          docker cp $container_id:/usr/src/debs/ ./slurm_build_output/
-          docker cp $container_id:/usr/src/nccl-tests/build/nccl-tests-perf.tar.gz ./slurm_build_output/
+          docker cp $container_id:/usr/src/debs/ ./pyxis_build_output/
           docker rm $container_id
 
       - name: Create GitHub Release and Upload DEB packages
         uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v2.1.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          tag_name: ${{ matrix.image.cuda_version }}-${{ matrix.image.ubuntu_version }}-slurm${{ matrix.slurm.version }}
-          name: Cuda ${{ matrix.image.cuda_version }}, Ubuntu ${{ matrix.image.ubuntu_version }}, Slurm ${{ matrix.slurm.version }}
-          body: "Release based on Ubuntu ${{ matrix.image.ubuntu_version }} for Slurm ${{ matrix.slurm.version }} with cuda ${{ matrix.image.cuda_version }}, nccl-tests and pyxis"
+          tag_name: pyxis-${{ matrix.pyxis.version }}_slurm_${{ matrix.slurm.version }}_enroot_${{ matrix.enroot.version }}
+          name: Pyxis ${{ matrix.pyxis.version }}
+          body: Pyxis ${{ matrix.pyxis.version }} for Slurm ${{ matrix.slurm.version }} and Enroot ${{ matrix.enroot.version }}
           draft: false
           prerelease: false
           files: |
-            slurm_build_output/debs/*.deb
-            slurm_build_output/nccl-tests-perf.tar.gz
+            pyxis_build_output/debs/*.deb
             
 
       - name: Cleanup
-        run: rm -rf slurm_build_output/*.deb
+        run: rm -rf pyxis_build_output/*.deb

pyxis/Dockerfile

@@ -0,0 +1,72 @@
+ARG BASE_IMAGE=ubuntu:22.04
+
+FROM $BASE_IMAGE
+
+ARG SLURM_VERSION=24.05.5
+ARG ENROOT_VERSION=3.5.0
+ARG PYXIS_VERSION=0.21.0
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install dependencies
+RUN apt-get update && \
+    apt -y install \
+        wget \
+        curl \
+        git \
+        build-essential \
+        devscripts \
+        debhelper \
+        fakeroot \
+        autoconf \
+        pkg-config \
+        libssl-dev \
+        libpam0g-dev \
+        libtool \
+        libjansson-dev \
+        libjson-c-dev \
+        libmunge-dev \
+        libhwloc-dev \
+        liblz4-dev \
+        flex \
+        libevent-dev \
+        squashfs-tools \
+        zstd \
+        libpmix2 \
+        libpmix-dev && \
+    apt clean
+
+# Install enroot (required for pyxis)
+RUN curl -fSsL -o /tmp/enroot_${ENROOT_VERSION}-1_amd64.deb https://github.com/NVIDIA/enroot/releases/download/v${ENROOT_VERSION}/enroot_${ENROOT_VERSION}-1_amd64.deb && \
+    curl -fSsL -o /tmp/enroot+caps_${ENROOT_VERSION}-1_amd64.deb https://github.com/NVIDIA/enroot/releases/download/v${ENROOT_VERSION}/enroot+caps_${ENROOT_VERSION}-1_amd64.deb && \
+    apt install -y /tmp/*.deb && rm -rf /tmp/*.deb && \
+    mkdir -m 777 /usr/share/enroot/enroot-data && \
+    mkdir -m 755 /run/enroot && \
+    setcap cap_sys_admin+pe /usr/bin/enroot-mksquashovlfs && \
+    setcap cap_sys_admin,cap_mknod+pe /usr/bin/enroot-aufs2ovlfs
+
+
+# Download and build pyxis deb
+# TODO @itechdima: keep watching this PR https://github.com/NVIDIA/pyxis/pull/155
+RUN cd /usr/src && \
+    wget https://github.com/nebius/slurm-deb-packages/releases/download/slurm-packages-${SLURM_VERSION}/slurm-smd_${SLURM_VERSION}-1_amd64.deb && \
+    wget https://github.com/nebius/slurm-deb-packages/releases/download/slurm-packages-${SLURM_VERSION}/slurm-smd-dev_${SLURM_VERSION}-1_amd64.deb && \
+    apt install -y /usr/src/*.deb && \
+    wget https://github.com/itechdima/pyxis/archive/refs/tags/v"$PYXIS_VERSION".tar.gz && \
+    tar -xzvf v"$PYXIS_VERSION".tar.gz && \
+    rm v"$PYXIS_VERSION".tar.gz && \
+    cd pyxis-"$PYXIS_VERSION" && \
+    sed -i 's|dh_auto_install -- prefix= libdir=/usr/lib/$(DEB_HOST_MULTIARCH) datarootdir=/usr/share|dh_auto_install -- prefix=/usr libdir=/usr/lib/x86_64-linux-gnu datarootdir=/usr/share|' debian/rules && \
+    make orig && \
+    make deb
+
+################################################################
+# RESULT
+################################################################
+# ls -la ../nvslurm-plugin-pyxis*.deb
+# /usr/src/nvslurm-plugin-pyxis_0.20.0-1_amd64.deb
+################################################################
+
+# Move deb files
+RUN mkdir /usr/src/debs && \
+    mv /usr/src/nvslurm-plugin-pyxis*.deb /usr/src/debs/