diff --git a/scripts/summit-1node.bsub b/scripts/summit-1node.bsub deleted file mode 100644 index f92ed46f2..000000000 --- a/scripts/summit-1node.bsub +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Copyright 2019-2020 Maxence Thevenet, Axel Huebl -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# Refs.: -# https://jsrunvisualizer.olcf.ornl.gov/?s4f0o11n6c7g1r11d1b1l0= -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#cuda-aware-mpi - -#BSUB -P ast183 -#BSUB -W 00:15 -#BSUB -nnodes 1 -#BSUB -alloc_flags smt4 -#BSUB -J quokka -#BSUB -o benchmark-1node-o.%J -#BSUB -e benchmark-1node-e.%J - -# make output group-readable by default -umask 0027 - -# set tuning vars for maximizing network bandwidth -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#spectrum-mpi-tunings-needed-for-maximum-bandwidth -export PAMI_ENABLE_STRIPING=1 -export PAMI_IBV_ADAPTER_AFFINITY=1 -export PAMI_IBV_DEVICE_NAME="mlx5_0:1,mlx5_3:1" -export PAMI_IBV_DEVICE_NAME_1="mlx5_3:1,mlx5_0:1" - -# fix problems with collectives since RHEL8 update: OLCFHELP-3545 -# disable all the IBM optimized barriers and drop back to HCOLL or OMPI's barrier implementations -export OMPI_MCA_coll_ibm_skip_barrier=true - -# ROMIO has a hint for GPFS named IBM_largeblock_io which optimizes I/O with operations on large blocks -export IBM_largeblock_io=true - -# GPU-aware MPI does NOT work on Summit!! You MUST disable it by adding: amrex.use_gpu_aware_mpi=0 -# run Quokka -jsrun -r 6 -a 1 -g 1 -c 7 -l GPU-CPU -d packed -b rs build_summit/src/HydroBlast3D/test_hydro3d_blast tests/benchmark_unigrid_256.in amrex.use_gpu_aware_mpi=0 - diff --git a/scripts/summit-4096node.bsub b/scripts/summit-4096node.bsub deleted file mode 100644 index 049afc9dd..000000000 --- a/scripts/summit-4096node.bsub +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Copyright 2019-2020 Maxence Thevenet, Axel Huebl -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# Refs.: -# https://jsrunvisualizer.olcf.ornl.gov/?s4f0o11n6c7g1r11d1b1l0= -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#cuda-aware-mpi - -#BSUB -P ast183 -#BSUB -W 00:15 -#BSUB -nnodes 4096 -#BSUB -alloc_flags smt4 -#BSUB -J quokka -#BSUB -o benchmark-4096node-o.%J -#BSUB -e benchmark-4096node-e.%J - -# make output group-readable by default -umask 0027 - -# set tuning vars for maximizing network bandwidth -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#spectrum-mpi-tunings-needed-for-maximum-bandwidth -export PAMI_ENABLE_STRIPING=1 -export PAMI_IBV_ADAPTER_AFFINITY=1 -export PAMI_IBV_DEVICE_NAME="mlx5_0:1,mlx5_3:1" -export PAMI_IBV_DEVICE_NAME_1="mlx5_3:1,mlx5_0:1" - -# fix problems with collectives since RHEL8 update: OLCFHELP-3545 -# disable all the IBM optimized barriers and drop back to HCOLL or OMPI's barrier implementations -export OMPI_MCA_coll_ibm_skip_barrier=true - -# ROMIO has a hint for GPFS named IBM_largeblock_io which optimizes I/O with operations on large blocks -export IBM_largeblock_io=true - -# GPU-aware MPI does NOT work on Summit!! You MUST disable it by adding: amrex.use_gpu_aware_mpi=0 -# run Quokka -jsrun -r 6 -a 1 -g 1 -c 7 -l GPU-CPU -d packed -b rs build_summit/src/HydroBlast3D/test_hydro3d_blast tests/benchmark_unigrid_4096.in amrex.use_gpu_aware_mpi=0 diff --git a/scripts/summit-512node.bsub b/scripts/summit-512node.bsub deleted file mode 100644 index 0da741330..000000000 --- a/scripts/summit-512node.bsub +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Copyright 2019-2020 Maxence Thevenet, Axel Huebl -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# Refs.: -# https://jsrunvisualizer.olcf.ornl.gov/?s4f0o11n6c7g1r11d1b1l0= -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#cuda-aware-mpi - -#BSUB -P ast183 -#BSUB -W 00:15 -#BSUB -nnodes 512 -#BSUB -alloc_flags smt4 -#BSUB -J quokka -#BSUB -o benchmark-512node-o.%J -#BSUB -e benchmark-512node-e.%J - -# make output group-readable by default -umask 0027 - -# set tuning vars for maximizing network bandwidth -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#spectrum-mpi-tunings-needed-for-maximum-bandwidth -export PAMI_ENABLE_STRIPING=1 -export PAMI_IBV_ADAPTER_AFFINITY=1 -export PAMI_IBV_DEVICE_NAME="mlx5_0:1,mlx5_3:1" -export PAMI_IBV_DEVICE_NAME_1="mlx5_3:1,mlx5_0:1" - -# fix problems with collectives since RHEL8 update: OLCFHELP-3545 -# disable all the IBM optimized barriers and drop back to HCOLL or OMPI's barrier implementations -export OMPI_MCA_coll_ibm_skip_barrier=true - -# ROMIO has a hint for GPFS named IBM_largeblock_io which optimizes I/O with operations on large blocks -export IBM_largeblock_io=true - -# GPU-aware MPI does NOT work on Summit!! You MUST disable it by adding: amrex.use_gpu_aware_mpi=0 -# run Quokka -jsrun -r 6 -a 1 -g 1 -c 7 -l GPU-CPU -d packed -b rs build_summit/src/HydroBlast3D/test_hydro3d_blast tests/benchmark_unigrid_2048.in amrex.use_gpu_aware_mpi=0 diff --git a/scripts/summit-64node.bsub b/scripts/summit-64node.bsub deleted file mode 100644 index 1740a223b..000000000 --- a/scripts/summit-64node.bsub +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Copyright 2019-2020 Maxence Thevenet, Axel Huebl -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# Refs.: -# https://jsrunvisualizer.olcf.ornl.gov/?s4f0o11n6c7g1r11d1b1l0= -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#cuda-aware-mpi - -#BSUB -P ast183 -#BSUB -W 00:15 -#BSUB -nnodes 64 -#BSUB -alloc_flags smt4 -#BSUB -J quokka -#BSUB -o benchmark-64node-o.%J -#BSUB -e benchmark-64node-e.%J - -# make output group-readable by default -umask 0027 - -# set tuning vars for maximizing network bandwidth -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#spectrum-mpi-tunings-needed-for-maximum-bandwidth -export PAMI_ENABLE_STRIPING=1 -export PAMI_IBV_ADAPTER_AFFINITY=1 -export PAMI_IBV_DEVICE_NAME="mlx5_0:1,mlx5_3:1" -export PAMI_IBV_DEVICE_NAME_1="mlx5_3:1,mlx5_0:1" - -# fix problems with collectives since RHEL8 update: OLCFHELP-3545 -# disable all the IBM optimized barriers and drop back to HCOLL or OMPI's barrier implementations -export OMPI_MCA_coll_ibm_skip_barrier=true - -# ROMIO has a hint for GPFS named IBM_largeblock_io which optimizes I/O with operations on large blocks -export IBM_largeblock_io=true - -# GPU-aware MPI does NOT work on Summit!! You MUST disable it by adding: amrex.use_gpu_aware_mpi=0 -# run Quokka -jsrun -r 6 -a 1 -g 1 -c 7 -l GPU-CPU -d packed -b rs build_summit/src/HydroBlast3D/test_hydro3d_blast tests/benchmark_unigrid_1024.in amrex.use_gpu_aware_mpi=0 diff --git a/scripts/summit-8node.bsub b/scripts/summit-8node.bsub deleted file mode 100644 index a3186af45..000000000 --- a/scripts/summit-8node.bsub +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Copyright 2019-2020 Maxence Thevenet, Axel Huebl -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# Refs.: -# https://jsrunvisualizer.olcf.ornl.gov/?s4f0o11n6c7g1r11d1b1l0= -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#cuda-aware-mpi - -#BSUB -P ast183 -#BSUB -W 00:15 -#BSUB -nnodes 8 -#BSUB -alloc_flags smt4 -#BSUB -J quokka -#BSUB -o benchmark-8node-o.%J -#BSUB -e benchmark-8node-e.%J - -# make output group-readable by default -umask 0027 - -# set tuning vars for maximizing network bandwidth -# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#spectrum-mpi-tunings-needed-for-maximum-bandwidth -export PAMI_ENABLE_STRIPING=1 -export PAMI_IBV_ADAPTER_AFFINITY=1 -export PAMI_IBV_DEVICE_NAME="mlx5_0:1,mlx5_3:1" -export PAMI_IBV_DEVICE_NAME_1="mlx5_3:1,mlx5_0:1" - -# fix problems with collectives since RHEL8 update: OLCFHELP-3545 -# disable all the IBM optimized barriers and drop back to HCOLL or OMPI's barrier implementations -export OMPI_MCA_coll_ibm_skip_barrier=true - -# ROMIO has a hint for GPFS named IBM_largeblock_io which optimizes I/O with operations on large blocks -export IBM_largeblock_io=true - -# GPU-aware MPI does NOT work on Summit!! You MUST disable it by adding: amrex.use_gpu_aware_mpi=0 -# run Quokka -jsrun -r 6 -a 1 -g 1 -c 7 -l GPU-CPU -d packed -b rs build_summit/src/HydroBlast3D/test_hydro3d_blast tests/benchmark_unigrid_512.in amrex.use_gpu_aware_mpi=0 diff --git a/scripts/summit.profile b/scripts/summit.profile deleted file mode 100644 index 2154d062b..000000000 --- a/scripts/summit.profile +++ /dev/null @@ -1,43 +0,0 @@ -# please set your project account -export proj=" " # FILL THIS IN - -# required dependencies -module load cmake/3.20.2 -module load gcc/9.3.0 # newer versions conflict with HDF5 modules -module load cuda/11.7.1 -module load hdf5/1.12.2 - -# optional: faster re-builds -module load ccache -module load ninja - -# often unstable at runtime with dependencies -module unload darshan-runtime - -# optional: Ascent in situ support -export Ascent_DIR=/sw/summit/ums/ums010/ascent/0.8.0_warpx/summit/cuda/gnu/ascent-install/ - -# optional: for Python bindings or libEnsemble -module load python/3.8.10 -module load freetype/2.10.4 # matplotlib - -# an alias to request an interactive batch node for two hours -# for paralle execution, start on the batch node: jsrun -alias getNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -Is /bin/bash" - -# an alias to run a command on a batch node -# usage: runNode -alias runNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -I" - -# make output group-readable by default -umask 0027 - -# optimize CUDA compilation for V100 -export AMREX_CUDA_ARCH=7.0 - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=$(which g++)