diff --git a/scripts/setonix-1node.submit b/scripts/setonix-1node.submit index 99a763aed..220e1b357 100644 --- a/scripts/setonix-1node.submit +++ b/scripts/setonix-1node.submit @@ -8,13 +8,12 @@ #SBATCH --exclusive #SBATCH --ntasks-per-node=8 #SBATCH --gpus-per-node=8 -#SBATCH --cpus-per-task=8 ##SBATCH --core-spec=8 #SBATCH -N 1 # load modules module load craype-accel-amd-gfx90a -module load rocm/5.4.3 +module load rocm/5.2.3 # workaround no longer needed for AMReX 23.07+ #export FI_MR_CACHE_MAX_COUNT=0 # libfabric disable caching diff --git a/scripts/setonix-64nodes.submit b/scripts/setonix-64nodes.submit index aad12d01c..886a41e22 100644 --- a/scripts/setonix-64nodes.submit +++ b/scripts/setonix-64nodes.submit @@ -8,13 +8,12 @@ #SBATCH --exclusive #SBATCH --ntasks-per-node=8 #SBATCH --gpus-per-node=8 -#SBATCH --cpus-per-task=8 ##SBATCH --core-spec=8 #SBATCH -N 64 # load modules module load craype-accel-amd-gfx90a -module load rocm/5.4.3 +module load rocm/5.2.3 # workaround no longer needed for AMReX 23.07+ #export FI_MR_CACHE_MAX_COUNT=0 # libfabric disable caching diff --git a/scripts/setonix-8nodes.submit b/scripts/setonix-8nodes.submit index a2fdbbc80..5b3a3b322 100644 --- a/scripts/setonix-8nodes.submit +++ b/scripts/setonix-8nodes.submit @@ -8,13 +8,12 @@ #SBATCH --exclusive #SBATCH --ntasks-per-node=8 #SBATCH --gpus-per-node=8 -#SBATCH --cpus-per-task=8 ##SBATCH --core-spec=8 #SBATCH -N 8 # load modules module load craype-accel-amd-gfx90a -module load rocm/5.4.3 +module load rocm/5.2.3 # workaround no longer needed for AMReX 23.07+ #export FI_MR_CACHE_MAX_COUNT=0 # libfabric disable caching diff --git a/scripts/setonix-rocm5.3.0.profile b/scripts/setonix-rocm5.3.0.profile deleted file mode 100644 index ba34627c7..000000000 --- a/scripts/setonix-rocm5.3.0.profile +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -source /opt/cray/pe/cpe/22.09/restore_lmod_system_defaults.sh - -module load cmake/3.21.4 -module load craype-accel-amd-gfx90a - -. /software/projects/pawsey0807/bwibking/spack/share/spack/setup-env.sh -spack load hip@5.3.0 -spack load rocrand@5.3.0 -spack load rocprim@5.3.0 - -module load cray-mpich -module load cce/14.0.3 -module load cray-hdf5 -module load cray-python/3.9.13.1 - -# GPU-aware MPI -export MPICH_GPU_SUPPORT_ENABLED=1 - -# optimize ROCm/HIP compilation for MI250X -export AMREX_AMD_ARCH=gfx90a - -# compiler environment hints -export CC=$(which hipcc) -export CXX=$(which hipcc) -export FC=$(which ftn) -export CFLAGS="-I${ROCM_PATH}/include -I${MPICH_DIR}/include -I${HDF5_DIR}/include" -export CXXFLAGS="-I${ROCM_PATH}/include -I${MPICH_DIR}/include -I${HDF5_DIR}/include" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 -L${MPICH_DIR}/lib -lmpi -L${HDF5_DIR}/lib -lhdf5 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa" diff --git a/scripts/setonix.profile b/scripts/setonix.profile index 7138354ec..29c6ded4d 100644 --- a/scripts/setonix.profile +++ b/scripts/setonix.profile @@ -1,12 +1,12 @@ #!/bin/bash -source /opt/cray/pe/cpe/22.09/restore_lmod_system_defaults.sh +source /opt/cray/pe/cpe/23.03/restore_lmod_system_defaults.sh -module load cmake/3.21.4 +#module load cmake/3.21.4 module load craype-accel-amd-gfx90a -module load rocm/5.0.2 +module load rocm/5.2.3 module load cray-mpich -module load cce/14.0.3 +module load cce/15.0.1 module load cray-hdf5 module load cray-python/3.9.13.1