Merge pull request #132 from bedroge/riscv

Build containers for riscv64

.github/workflows/build-publish-containers.yml

@@ -12,7 +12,7 @@ on:
       - main
     paths:
       - containers/Dockerfile.EESSI-*
-      - containers/build-or-download-cvmfs-rpm.sh
+      - containers/build-or-download-cvmfs-*.sh
 
   # also rebuild the containers for new releases of filesystem-layer:
   # this often means that we have a new CVMFS configuration package
@@ -32,8 +32,14 @@ jobs:
       contents: read
     strategy:
       matrix:
-        tag: ['client:centos7', 'build-node:debian11']
-        platform: [amd64, arm64, ppc64le]
+        tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid']
+        platform: [amd64, arm64, riscv64]
+        exclude:
+          # exclude images that don't support RISC-V
+          - tag: client:centos7
+            platform: riscv64
+          - tag: build-node:debian11
+            platform: riscv64
 
     steps:
       - name: Check out the repo
@@ -91,7 +97,7 @@ jobs:
       contents: read
     strategy:
       matrix:
-        tag: ['client:centos7', 'build-node:debian11']
+        tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid']
     steps:
       - name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
@@ -106,12 +112,19 @@ jobs:
           echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
 
       - name: Create the final image using  "docker buildx imagetools create"
-        if: github.event_name != 'pull_request'
+        if: github.event_name != 'pull_request' && matrix.tag == 'build-node:debian-sid'
         run: |
           docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \
             ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \
             ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 \
-            ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-ppc64le
+            ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-riscv64
+
+      - name: Create the final image using  "docker buildx imagetools create"
+        if: github.event_name != 'pull_request' && matrix.tag != 'build-node:debian-sid'
+        run: |
+          docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \
+            ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \
+            ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64
 
       - name: Inspect the final image
         if: github.event_name != 'pull_request'

containers/Dockerfile.EESSI-build-node-debian-sid

@@ -0,0 +1,44 @@
+ARG cvmfsversion=2.11.2
+ARG archspecversion=0.2.2
+ARG awscliversion=1.32.22
+ARG fuseoverlayfsversion=1.10
+
+FROM debian:sid-20240330-slim AS prepare-deb
+ARG cvmfsversion
+COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-debs.sh
+RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion}
+
+
+FROM debian:sid-20240330-slim
+ARG cvmfsversion
+ARG archspecversion
+ARG awscliversion
+ARG fuseoverlayfsversion
+
+COPY --from=prepare-deb /root/deb /root/deb
+
+RUN apt-get update
+RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace
+# python3 and jq are required for eessi-upload-to-staging script (next to awscli)
+RUN apt-get install -y python3-pip python3-venv jq
+RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \
+            /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \
+            /root/deb/cvmfs-libs_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \
+            /root/deb/cvmfs-config-default_latest_all.deb \
+            /root/deb/cvmfs-config-eessi_latest_all.deb
+
+# download binary for specific version of fuse-overlayfs
+RUN curl -L -o /usr/local/bin/fuse-overlayfs https://github.com/containers/fuse-overlayfs/releases/download/v${fuseoverlayfsversion}/fuse-overlayfs-$(uname -m) \
+  && chmod +x /usr/local/bin/fuse-overlayfs
+
+RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local \
+  && echo 'CVMFS_HIDE_MAGIC_XATTRS=yes' >> /etc/cvmfs/default.local
+
+RUN mkdir -p /cvmfs/software.eessi.io
+
+RUN useradd -ms /bin/bash eessi
+
+RUN python3 -m venv /opt/archspec && . /opt/archspec/bin/activate && pip3 install archspec==${archspecversion} && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec
+# stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947)
+RUN python3 -m venv /opt/awscli && . /opt/awscli/bin/activate && pip3 install awscli==${awscliversion} && deactivate && ln -s /opt/awscli/bin/aws /usr/local/bin/aws

containers/Dockerfile.EESSI-build-node-debian11

@@ -39,7 +39,3 @@ RUN useradd -ms /bin/bash eessi
 
 # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947)
 RUN pip3 install archspec awscli==${awscliversion}
-
-RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \
-  && mv eessi-upload-to-staging /usr/bin \
-  && chmod a+x /usr/bin/eessi-upload-to-staging

containers/build-or-download-cvmfs-debs.sh

@@ -1,22 +1,52 @@
 #/bin/bash
 cvmfsversion=$1
 arch=$(dpkg --print-architecture)
-os=debian11
 
 apt-get update
-apt-get install -y wget
-if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
+apt-get install -y wget lsb-release
+
+distro=$(lsb_release -si | tr [:upper:] [:lower:])
+release=$(lsb_release -sr)
+
+# lsb_release -sr prints n/a for debian sid, replace it by 13
+if [ "${distro}" = "debian" ] && [ "${release}" = "n/a" ]
+then
+    release=13
+fi
+
+os="${distro}${release}"
+
+if [ "$arch" = "arm64" ] || [ "$arch" = "riscv64" ] || [ "${os}" = "debian13" ]
 then
-    apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools python3-dev python3-setuptools uuid-dev valgrind libz-dev lsb-release
-    # Set Python 2 as default Python
-    update-alternatives --install /usr/bin/python python /usr/bin/python2 1
-    update-alternatives --install /usr/bin/python python /usr/bin/python3 2
+    apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python3-dev python3-setuptools python3-dev python3-setuptools uuid-dev libz-dev lsb-release
     cd /tmp
     wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
     tar xzf cvmfs-${cvmfsversion}.tar.gz
-    cd cvmfs-cvmfs-${cvmfsversion}/ci/cvmfs
+    cd cvmfs-cvmfs-${cvmfsversion}
     mkdir /root/deb
-    sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 ppc64el/' ../../packaging/debian/cvmfs/control.in
+    sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 riscv64/' packaging/debian/cvmfs/control.in
+    sed -i 's/python-dev/python3-dev/' packaging/debian/cvmfs/control.in
+    sed -i 's/python-setuptools/python3-setuptools/' packaging/debian/cvmfs/control.in
+    if [ "$arch" = "riscv64" ]
+    then
+        # valgrind is not available (yet) for RISC-V
+        sed -i 's/, valgrind//' packaging/debian/cvmfs/control.in
+        # for RISC-V we need to run autoreconf, see:
+        # https://github.com/cvmfs/cvmfs/pull/3446
+        wget https://github.com/cvmfs/cvmfs/pull/3446.patch
+        patch -p 1 -i ./3446.patch
+        rm 3446.patch
+        # QEMU shows the host CPU in /proc/cpuinfo, so we need to tweak the CPU detection for some packages and use uname -m instead
+        sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/libcrypto/src/configureHook.sh
+        sed -i "s/rv64/riscv64/" externals/libcrypto/src/configureHook.sh
+        sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/protobuf/src/configureHook.sh
+        sed -i "s/rv64/riscv64/" externals/protobuf/src/configureHook.sh
+    else
+        apt-get install -y valgrind
+    fi
+    cd ci/cvmfs
+    # make sure the cvmfs package also uses debian 13 for debian sid
+    [ $release = "13" ] && sed -i "s@\$(lsb_release -sr)@13@" ./deb.sh && sed -i "s/focal/trixie/" ./deb.sh
     ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb
 else
     mkdir -p /root/deb

containers/build-or-download-cvmfs-rpms.sh

@@ -2,7 +2,7 @@ cvmfsversion=$1
 arch=$(uname -m)
 
 yum install -y wget
-if [ "$arch" = "ppc64le" ]
+if [ "$arch" = "riscv64" ]
 then
     yum install -y epel-release
     yum install -y rpm-build checkpolicy cmake fuse-devel fuse3-devel gcc gcc-c++ golang libattr-devel libcap-devel libuuid-devel openssl-devel python-devel python-setuptools python3-devel python3-setuptools selinux-policy-devel valgrind-devel hardlink selinux-policy-targeted