update configuration in Tianhe

include/Makeoptions.Tianhe-XY

@@ -0,0 +1,39 @@
+# =======================================================
+# mpif90 - ifort 
+# 
+
+ FF = mpif90 
+
+ NETCDF_LIB = /APP/u22/x86/netcdf/4.9.2-icc-oneapi2023.2/lib/
+ NETCDF_INC = /APP/u22/x86/netcdf/4.9.2-icc-oneapi2023.2/include/
+
+ LAPACK_LIB = /APP/u22/x86/LAPACK/lapack-3.12.0-icc-oneapi2023.2_noimpi/lib/
+ BLAS_LIB = /APP/u22/x86/BLAS/icc-2023/lib-icc/
+
+ MOD_CMD = -module 
+
+ FOPTS = -qopenmp -O2 -traceback -r8 -free -check uninit -check bounds
+
+ LDFLAGS = -L${NETCDF_LIB} -lnetcdff -L${LAPACK_LIB} -llapack -L${BLAS_LIB} -lblas
+
+#============================================================
+# CaMa-Flood Mkinclude (for Linux, Intel fortran)
+
+RM = /bin/rm -f
+CP = /bin/cp
+#----
+# Pre-Prosessing options
+# DMPI=-DUseMPI: activate when MPI parallelization is used
+# DCDF=-DUseCDF: activate when using netCDF, comment out when not needed
+# DATM=-DNoAtom: activate when OMP ATOMIC calculation should be avoided (bit identical simulation)
+#----
+#DMPI=-DUseMPI
+DCDF=-DUseCDF
+#DATM=-DNoAtom
+CFLAGS=$(DMPI) $(DCDF) $(DATM) 
+#----
+# FCMP: main program (src/), FC: pre/post process (map/ etc/)
+FCMP = ifort -qopenmp
+FC = ifort
+LFLAGS =
+FFLAGS = -O3 -warn all -fpp -free -assume byterecl -heap-arrays -nogen-interface -lpthread -static-intel

run/scripts/batch.config

@@ -0,0 +1 @@
+batch.github.config

run/scripts/batch.github.config

@@ -0,0 +1,54 @@
+#------------------------------earthlab--------------------------------------------
+##!/bin/bash
+#
+##SBATCH -J <CASENAME>
+##SBATCH -p <QUEUE>
+##SBATCH -N <NNODES>
+##SBATCH -n <NPROCESSES>
+##SBATCH --ntasks-per-node=<NTASKSPERNODE>
+##SBATCH --mem=<MEMORY>
+##SBATCH -o colm.o%j
+##SBATCH -e colm.e%j
+##SBATCH --exclusive
+##SBATCH -t <WALLTIME>
+##
+##module purge
+##module load compiler/intel/2021.3.1
+##module load mpi/intelmpi/2018.4.274
+##module load mathlib/netcdf/intel/4.4.1
+##module load mathlib/hdf5/intel/1.8.20
+##
+##export I_MPI_FABRICS=shm:dapl
+##export I_MPI_DAPL_UD=1
+##export I_MPI_DAPL_UD_RDMA_MIXED=1
+##export I_MPI_LARGE_SCALE_THRESHOLD=8192
+##export I_MPI_DAPL_UD_ACK_SEND_POOL_SIZE=8704
+##export I_MPI_DAPL_UD_ACK_RECV_POOL_SIZE=8704
+##export I_MPI_DAPL_UD_RNDV_EP_NUM=2
+##
+##export DAPL_UCM_REP_TIME=8000 #  REQUEST timer, waiting for REPLY in millisecs
+##export DAPL_UCM_RTU_TIME=8000 #  REPLY timer, waiting for RTU in millisecs
+##export DAPL_UCM_RETRY=10 #  REQUEST and REPLY retries
+##export DAPL_UCM_CQ_SIZE=2000
+##export DAPL_UCM_QP_SIZE=2000
+##
+##export DAPL_UCM_DREQ_RETRY=4 #default == 1
+##export DAPL_UCM_DREP_TIME=200 #default == 200ms
+##export DAPL_UCM_WAIT_TIME=10000 #default == 60000ms
+##
+##ulimit -s unlimited
+##scontrol show hostname > nd
+##NP=$SLURM_NPROCS
+
+
+#-------------------------------baiduboat------------------------------------------
+#!/bin/bash
+
+#BSUB -J <CASENAME>
+#BSUB -q <QUEUE>
+#BSUB -o colm.o%
+#BSUB -e colm.e%
+#BSUB -n <NPROCESSES>
+#BSUB -R rusage[mem=<MEMORY>]
+#BSUB -R span[ptile=<NTASKSPERNODE>]
+

run/scripts/batch.slurm-earthlab.config

@@ -0,0 +1,44 @@
+#------------------------------earthlab--------------------------------------------
+##!/bin/bash
+#
+##SBATCH -J <CASENAME>
+##SBATCH -p <QUEUE>
+##SBATCH -N <NNODES>
+##SBATCH -n <NPROCESSES>
+##SBATCH --ntasks-per-node=<NTASKSPERNODE>
+##SBATCH --mem=<MEMORY>
+##SBATCH -o colm.o%j
+##SBATCH -e colm.e%j
+##SBATCH --exclusive
+##SBATCH -t <WALLTIME>
+##
+##module purge
+##module load compiler/intel/2021.3.1
+##module load mpi/intelmpi/2018.4.274
+##module load mathlib/netcdf/intel/4.4.1
+##module load mathlib/hdf5/intel/1.8.20
+##
+##export I_MPI_FABRICS=shm:dapl
+##export I_MPI_DAPL_UD=1
+##export I_MPI_DAPL_UD_RDMA_MIXED=1
+##export I_MPI_LARGE_SCALE_THRESHOLD=8192
+##export I_MPI_DAPL_UD_ACK_SEND_POOL_SIZE=8704
+##export I_MPI_DAPL_UD_ACK_RECV_POOL_SIZE=8704
+##export I_MPI_DAPL_UD_RNDV_EP_NUM=2
+##
+##export DAPL_UCM_REP_TIME=8000 #  REQUEST timer, waiting for REPLY in millisecs
+##export DAPL_UCM_RTU_TIME=8000 #  REPLY timer, waiting for RTU in millisecs
+##export DAPL_UCM_RETRY=10 #  REQUEST and REPLY retries
+##export DAPL_UCM_CQ_SIZE=2000
+##export DAPL_UCM_QP_SIZE=2000
+##
+##export DAPL_UCM_DREQ_RETRY=4 #default == 1
+##export DAPL_UCM_DREP_TIME=200 #default == 200ms
+##export DAPL_UCM_WAIT_TIME=10000 #default == 60000ms
+##
+##ulimit -s unlimited
+##scontrol show hostname > nd
+##NP=$SLURM_NPROCS
+
+
+

run/scripts/batch.slurm-tianhexy.config

@@ -0,0 +1,16 @@
+#------------------------------tianhexy--------------------------------------------
+#!/bin/bash
+
+#SBATCH -J <CASENAME>
+#SBATCH -p <QUEUE>
+#SBATCH -N <NNODES>
+#SBATCH -n <NPROCESSES>
+#SBATCH --ntasks-per-node=<NTASKSPERNODE>
+#SBATCH --mem=<MEMORY>
+#SBATCH -o colm.o%j
+#SBATCH -e colm.e%j
+#SBATCH --exclusive
+#SBATCH -t <WALLTIME>
+
+
+

run/scripts/batch.slurm.config

@@ -0,0 +1,44 @@
+#------------------------------earthlab--------------------------------------------
+##!/bin/bash
+#
+##SBATCH -J <CASENAME>
+##SBATCH -p <QUEUE>
+##SBATCH -N <NNODES>
+##SBATCH -n <NPROCESSES>
+##SBATCH --ntasks-per-node=<NTASKSPERNODE>
+##SBATCH --mem=<MEMORY>
+##SBATCH -o colm.o%j
+##SBATCH -e colm.e%j
+##SBATCH --exclusive
+##SBATCH -t <WALLTIME>
+##
+##module purge
+##module load compiler/intel/2021.3.1
+##module load mpi/intelmpi/2018.4.274
+##module load mathlib/netcdf/intel/4.4.1
+##module load mathlib/hdf5/intel/1.8.20
+##
+##export I_MPI_FABRICS=shm:dapl
+##export I_MPI_DAPL_UD=1
+##export I_MPI_DAPL_UD_RDMA_MIXED=1
+##export I_MPI_LARGE_SCALE_THRESHOLD=8192
+##export I_MPI_DAPL_UD_ACK_SEND_POOL_SIZE=8704
+##export I_MPI_DAPL_UD_ACK_RECV_POOL_SIZE=8704
+##export I_MPI_DAPL_UD_RNDV_EP_NUM=2
+##
+##export DAPL_UCM_REP_TIME=8000 #  REQUEST timer, waiting for REPLY in millisecs
+##export DAPL_UCM_RTU_TIME=8000 #  REPLY timer, waiting for RTU in millisecs
+##export DAPL_UCM_RETRY=10 #  REQUEST and REPLY retries
+##export DAPL_UCM_CQ_SIZE=2000
+##export DAPL_UCM_QP_SIZE=2000
+##
+##export DAPL_UCM_DREQ_RETRY=4 #default == 1
+##export DAPL_UCM_DREP_TIME=200 #default == 200ms
+##export DAPL_UCM_WAIT_TIME=10000 #default == 60000ms
+##
+##ulimit -s unlimited
+##scontrol show hostname > nd
+##NP=$SLURM_NPROCS
+
+
+

run/scripts/create_scripts

@@ -71,6 +71,8 @@ CreateHeader()
    if [ -f $CONFIG ];then
       case ${SCRIPTNAME} in
       mksrf.submit)
+	 EXEC=`awk '/Exe_command/ {print $2}' $CONFIG`
+	 EXEO=`awk '/Exe_opt/ {print $2}' $CONFIG`
          NP=`awk '/NProcesses_mksrf/ {print $2}' $CONFIG`
          NN=`awk '/NNodes_mksrf/ {print $2}' $CONFIG`
          NTPN=`awk '/NTasksPerNode_mksrf/ {print $2}' $CONFIG`
@@ -80,6 +82,8 @@ CreateHeader()
          OUTPUT='mksrf'
          ;;
       init.submit)
+	 EXEC=`awk '/Exe_command/ {print $2 $3}' $CONFIG`
+	 EXEO=`awk '/Exe_opt/ {print $2}' $CONFIG`
          NP=`awk '/NProcesses_mkini/ {print $2}' $CONFIG`
          NN=`awk '/NNodes_mkini/ {print $2}' $CONFIG`
          NTPN=`awk '/NTasksPerNode_mkini/ {print $2}' $CONFIG`
@@ -89,6 +93,8 @@ CreateHeader()
          OUTPUT='init'
          ;;
       case.submit)
+	 EXEC=`awk '/Exe_command/ {print $2 $3}' $CONFIG`
+	 EXEO=`awk '/Exe_opt/ {print $2}' $CONFIG`
          NP=`awk '/NProcesses_case/ {print $2}' $CONFIG`
          NN=`awk '/NNodes_case/ {print $2}' $CONFIG`
          NTPN=`awk '/NTasksPerNode_case/ {print $2}' $CONFIG`
@@ -106,6 +112,8 @@ CreateHeader()
       echo SCRIPTNAME $SCRIPTNAME
       echo HEADER $HEADER
       echo CONFIG $CONFIG
+      echo EXEC $EXEC
+      echo EXEO $EXEO
       if [ $NP -gt 0 ] 2>/dev/null;then
          echo Number of processes is assigned from $CONFIG
       else
@@ -150,6 +158,8 @@ CreateHeader()
          WT=24:00:00
          QUEUE=normal
          OUTPUT='mksrf'
+	 EXEC='mpirun'
+	 EXEO='-np'
          ;;
       init.submit)
          NP=24
@@ -159,6 +169,8 @@ CreateHeader()
          WT=24:00:00
          QUEUE=normal
          OUTPUT='init'
+	 EXEC='mpirun'
+	 EXEO='-np'
          ;;
       case.submit)
          NP=120
@@ -168,6 +180,8 @@ CreateHeader()
          WT=24:00:00
          QUEUE=normal
          OUTPUT='case'
+	 EXEC='mpirun'
+	 EXEO='-np'
          ;;
       *)
          echo "Error: Unknown Unknown script name: $SCRIPTNAME in CreateScripts"
@@ -231,7 +245,7 @@ CreateScripts()
    fi
    cat>>mksrf.submit<<EOF
 cd ${CASEPATH}/${CASENAME}/bld/run/
-mpirun -np $NP ./mksrfdata.x ../../input_${CASENAME}.nml > ../../logmksrfdata
+${EXE} $NP ./mksrfdata.x ../../input_${CASENAME}.nml > ../../logmksrfdata
 
 EOF
 
@@ -257,7 +271,7 @@ EOF
    fi
    cat>>init.submit<<EOF
 cd ${CASEPATH}/${CASENAME}/bld/run/
-mpirun -np $NP ./mkinidata.x ../../input_${CASENAME}.nml > ../../logini
+${EXEC} ${EXEO} $NP ./mkinidata.x ../../input_${CASENAME}.nml > ../../logini
 
 EOF
 
@@ -285,7 +299,7 @@ EOF
       none)
          cat>>'case.submit'<<EOF
 cd ${CASEPATH}/${CASENAME}/bld/run/
-mpirun -np $NP ./colm.x ../../input_${CASENAME}.nml > ../../log
+${EXEC} ${EXEO} $NP ./colm.x ../../input_${CASENAME}.nml > ../../log
 EOF
          ;;
       nd)
@@ -294,7 +308,7 @@ iloop=1
 while [ ${ILOOP} -le 100 ]
 do
    cd ${CASEPATH}/${CASENAME}/bld/run/
-   mpirun -np $NP ./colm.x ../../input_${CASENAME}.nml > ../../log-${ILOOP}
+   ${EXEC} ${EXEO} $NP ./colm.x ../../input_${CASENAME}.nml > ../../log-${ILOOP}
    mkdir -p ${CASEPATH}/${CASENAME}/restart/loop-${ILOOP}
    mkdir -p ${CASEPATH}/${CASENAME}/history/loop-${ILOOP}
    cd ${CASEPATH}/${CASENAME}/restart
@@ -322,9 +336,9 @@ while [ ${ILOOP} -le 130 ]
 do
    cd ${CASEPATH}/${CASENAME}/bld/run/
    if [ ${ILOOP} -le 100 ];then
-      mpirun -np 240 ./colm.x ../../input_${CASENAME}-SASU.nml > ../../log-${ILOOP}
+      ${EXEC} ${EXEO} 240 ./colm.x ../../input_${CASENAME}-SASU.nml > ../../log-${ILOOP}
    else
-      mpirun -np 240 ./colm.x ../../input_${CASENAME}.nml > ../../log-${ILOOP}
+      ${EXEC} ${EXEO} 240 ./colm.x ../../input_${CASENAME}.nml > ../../log-${ILOOP}
    fi
    mkdir -p ${CASEPATH}/${CASENAME}/restart/loop-${ILOOP}
    mkdir -p ${CASEPATH}/${CASENAME}/history/loop-${ILOOP}