diff --git a/.github/workflows/apptainer_exachem.yml b/.github/workflows/apptainer_exachem.yml index 6a70b98..b842f86 100644 --- a/.github/workflows/apptainer_exachem.yml +++ b/.github/workflows/apptainer_exachem.yml @@ -22,7 +22,8 @@ jobs: - MPI-PR gpu: - cpu - - nvidia + - nvidia_70 + - nvidia_80 - amd_gfx908_rocm5.5.1 - amd_gfx90a_rocm5.5.1 - amd_gfx908_rocm6.0 @@ -44,7 +45,7 @@ jobs: else myversion="" fi - if [[ ${{ matrix.gpu }} != nvidia ]]; then + if [[ ${{ matrix.gpu }} != nvidia_70 ]]; then myversion+=.${{ matrix.gpu }} fi echo "exachemversion=$myversion" >> $GITHUB_ENV diff --git a/apptainer.ompi41x/Singularity b/apptainer.ompi41x/Singularity index 75b2af4..5946415 100644 --- a/apptainer.ompi41x/Singularity +++ b/apptainer.ompi41x/Singularity @@ -35,7 +35,7 @@ echo "$(readlink /proc/$$/exe) is the shell in use" export DEBIAN_FRONTEND=noninteractive export ompi=4.1.6 export ompi_prefix=ompi-$ompi -export ucx=1.14.0 +export ucx=1.15.0 export xpmem=2.7.3 export knem=1.1.4 export OMPI_DIR=/opt/ompi @@ -48,6 +48,9 @@ bash -o pipefail -c "if [[ $(echo $GPU | cut -c 1-3) == 'amd' ]]; then echo gp echo 'ROCM_VERSION='`echo $GPU | cut -c 16-` > rocm_version.txt ; echo 'GPU_ARCH='`echo $GPU | cut -c 5-10` > gpu_arch.txt ; echo 'GPU=amd ' > gpu.txt ; fi" +bash -o pipefail -c "if [[ $(echo $GPU | cut -c 1-6) == 'nvidia' ]]; then echo gpus is nvidia; +echo 'GPU_ARCH='`echo $GPU | cut -c 8-9` > gpu_arch.txt ; +echo 'GPU=nvidia ' > gpu.txt ; fi" export `cat ./gpu.txt` export `cat ./rocm_version.txt` export `cat ./gpu_arch.txt` @@ -97,6 +100,7 @@ echo TAMM_BRANCH is $TAMM_BRANCH python3 python3-dev cmake unzip zip perl automake libtool autoconf flex \ libnuma-dev libslurm-dev libpmi2-0-dev libpmi0-dev libnl-3-dev libnl-route-3-dev udev \ librdmacm-dev librdmacm1 rdma-core libibverbs-dev ibverbs-providers \ + libhwloc-dev \ pkg-config gawk software-properties-common linux-headers-generic \ && apt-get install -q=2 -y gcc-"$GCC_VERSION" g++-"$GCC_VERSION" gfortran-"$GCC_VERSION" libgfortran-"$GCC_VERSION"-dev make ssh patch curl rsync wget git file bzip2 tar gnupg2 cmake \ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-"$GCC_VERSION" 100 \ @@ -208,7 +212,7 @@ cd .. ; rm -rf *tar*gz $ompi-prefix ucx-$ucx ;fi " \ && mkdir build && cd build \ && CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=/opt/install/CC -DMODULES='CC;DFT' " \ && CMAKE_OPTIONS="$CMAKE_OPTIONS -DBLIS_CONFIG=zen3 -DMARCH_FLAGS=-march=znver3" \ -&& if [ ${GPU} = "nvidia" ]; then echo "nvidia gpu "; CMAKE_OPTIONS="$CMAKE_OPTIONS -DUSE_CUDA=ON -DGPU_ARCH=70" ; fi \ +&& if [ ${GPU} = "nvidia" ]; then echo "nvidia gpu "; CMAKE_OPTIONS="$CMAKE_OPTIONS -DUSE_CUDA=ON -DGPU_ARCH=${GPU_ARCH}" ; fi \ && if [ ${GPU} = "amd" ]; then echo "amd gpu "; CMAKE_OPTIONS="$CMAKE_OPTIONS -DUSE_HIP=ON -DROCM_ROOT=$ROCM_PATH -DGPU_ARCH=${GPU_ARCH} -DGCCROOT=/usr " ; ls -lrt $ROCM_PATH/include/rocblas/rocblas.h $ROCM_PATH/include/ ; ln -sf $ROCM_PATH/include/rocblas/rocblas.h $ROCM_PATH/include/rocblas.h; fi \ && export CMAKE_OPTIONS \ && echo CMAKE_OPTIONS $CMAKE_OPTIONS \