For NCEP regtests, add option for gnu compiler and new machine Hercul…

…es (#1145)
NOAA-EMC · Apr 2, 2024 · 399df78 · 399df78
1 parent f66b6d4
commit 399df78
Show file tree

Hide file tree

Showing 2 changed files with 88 additions and 26 deletions.
diff --git a/model/src/wminitmd.F90 b/model/src/wminitmd.F90
@@ -1273,6 +1273,12 @@ SUBROUTINE WMINIT ( IDSI, IDSO, IDSS, IDST, IDSE, IFNAME,       &
         IF (WORDS(6) .EQ. 'T') THEN
           CALL NEXTLN ( COMSTR , MDSI , MDSE2 )
           READ (MDSI,*,END=2001,ERR=2002)(ODAT(I,1),I=5*(8-1)+1,5*8)
+        ELSE 
+          ODAT(5*(8-1)+1,1)=0
+          ODAT(5*(8-1)+2,1)=0
+          ODAT(5*(8-1)+3,1)=0
+          ODAT(5*(8-1)+4,1)=0
+          ODAT(5*8,1)=0
         END IF
       ELSE
         READ (MDSI,*,END=2001,ERR=2002)(ODAT(I,1),I=5*(J-1)+1,5*J)

diff --git a/regtests/bin/matrix_cmake_ncep b/regtests/bin/matrix_cmake_ncep
@@ -22,59 +22,109 @@ usage ()
 {
   cat 2>&1 << EOF
 
-  Usage: $myname model_dir
+  Usage: $myname model_dir compiler 
   Required:
     model_dir : path to model dir of WW3 source
+  Optional: 
+    compiler : intel (default) or gnu 
 EOF
 }
 
-
 # Get required arguments
   if [ ! $# = 0 ]
   then
     main_dir="$1" ; shift
+    if [ ! $# = 0 ]
+    then
+      compiler="$1"; shift 
+    else 
+      compiler='intel'
+    fi 
   else
     usage
     exit 1
   fi
+
+
 
 # Convert main_dir to absolute path
   main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`"
 
 # Module Versions from spack-stack that are common for all platforms
   modnetcdfc='netcdf-c/4.9.2'
-  modnetcdff='netcdf-fortran/4.6.0'
+  modnetcdff='netcdf-fortran/4.6.1'
   modjasper='jasper/2.0.32'
   modzlib='zlib/1.2.13'
   modpng='libpng/1.6.37'
   modhdf5='hdf5/1.14.0'
   modbacio='bacio/2.4.1'
   modg2='g2/3.4.5'
   modw3emc='w3emc/2.10.0'
-  modesmf='esmf/8.4.2'
+  modesmf='esmf/8.5.0'
   modscotch='scotch/7.0.4'
 
 # Set batchq queue, choose modules and other custom variables to fit system and
 # to define headers etc (default to original version if empty)
   ishera=`hostname | grep hfe`
   isorion=`hostname | grep Orion`
+  ishercules=`hostname | grep hercules`
   if [ $ishera ]
   then
-    # If no other h, assuming Hera
     batchq='slurm'
-    spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core'
-    modcomp='stack-intel/2021.5.0'
-    modmpi='stack-intel-oneapi-mpi/2021.5.1'
-    metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install'
-    modcmake='cmake/3.23.1'
+    if [ $compiler = "intel" ]
+    then 
+      spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core'
+      modcomp='stack-intel/2021.5.0'
+      modmpi='stack-intel-oneapi-mpi/2021.5.1'
+      metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hera/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    elif [ $compiler = "gnu" ]
+    then 
+      spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core'
+      modcomp='stack-gcc/9.2.0'
+      modmpi='stack-openmpi/4.1.5'
+      metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hera/gnu/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    else 
+      echo "Compiler $compiler not supported on hera" 
+      exit 1 
+    fi 
   elif [ $isorion ]
   then
+    if [ $compiler = "intel" ]
+    then
+      batchq='slurm'
+      spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core'
+      modcomp='stack-intel/2022.0.2'
+      modmpi='stack-intel-oneapi-mpi/2021.5.1'
+      metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/orion/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    else
+      echo "Compiler $compiler not supported on orion" 
+      exit 1
+    fi  
+  elif [ $ishercules ] 
+  then 
     batchq='slurm'
-    spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core'
-    modcomp='stack-intel/2022.0.2'
-    modmpi='stack-intel-oneapi-mpi/2021.5.1'
-    metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install'
-    modcmake='cmake/3.23.1'
+    if [ $compiler = "intel" ]
+    then
+      spackstackpath='/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core'
+      modcomp='stack-intel/2021.9.0'
+      modmpi='stack-intel-oneapi-mpi/2021.9.0'
+      metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/hercules/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    elif [ $compiler = "gnu" ]
+    then 
+      spackstackpath='/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core'
+      spackstackpath2='/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles'
+      modcomp='stack-gcc/12.2.0'
+      modmpi='stack-mvapich2/2.3.7'
+      metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/hercules/gnu/spack-stack/1.6.0/parmetis-4.0.3/install'
+      modcmake='cmake/3.23.1'
+    else
+      echo "Compiler $compiler not supported on hercules" 
+      exit 1
+    fi
   else
     batchq=
   fi
@@ -105,10 +155,19 @@ EOF
     echo '#SBATCH --exclusive'                              >> matrix.head
     echo ' '                                                >> matrix.head
     echo 'ulimit -s unlimited'                              >> matrix.head
-    echo 'ulimit -c 0'                                      >> matrix.head
-    echo 'export KMP_STACKSIZE=2G'                          >> matrix.head
-    echo 'export FI_OFI_RXM_BUFFER_SIZE=128000'             >> matrix.head
-    echo 'export FI_OFI_RXM_RX_SIZE=64000'                  >> matrix.head
+  elif [ $batchq = "slurm" ] && [ $ishercules ]
+  then
+    echo "#SBATCH -n ${np}"                                 >> matrix.head
+    echo "##SBATCH --cpus-per-task=${nth}"                  >> matrix.head
+    echo '#SBATCH -q batch'                                 >> matrix.head
+    echo '#SBATCH -t 08:00:00'                              >> matrix.head
+    echo '#SBATCH -A marine-cpu'                            >> matrix.head
+    echo '#SBATCH -J ww3_regtest'                           >> matrix.head
+    echo '#SBATCH -o matrix.out'                            >> matrix.head
+    echo '#SBATCH -p hercules'                              >> matrix.head
+    echo '#SBATCH --exclusive'                              >> matrix.head
+    echo ' '                                                >> matrix.head
+    echo 'ulimit -s unlimited'                              >> matrix.head
   elif [ $batchq = "slurm" ]
   then
     echo "#SBATCH -n ${np}"                                 >> matrix.head
@@ -133,13 +192,10 @@ EOF
 
 # Netcdf, Parmetis and SCOTCH modules & variables
   echo "  module purge"                                     >> matrix.head
-  if [ ! -z $basemodcomp ]; then
-    echo "  module load $basemodcomp"                       >> matrix.head
-  fi
-  if [ ! -z $basemodmpi ]; then
-    echo "  module load $basemodmpi"                        >> matrix.head
-  fi
-  echo "  module use  $spackstackpath"                      >> matrix.head
+  echo "  module use $spackstackpath"                       >> matrix.head
+  if [ ! -z $spackstackpath2 ]; then
+    echo "  module use $spackstackpath2"                    >> matrix.head
+  fi 
   echo "  module load $modcomp"                             >> matrix.head
   echo "  module load $modmpi"                              >> matrix.head
   echo "  module load $modcmake"                            >> matrix.head