Update all wrf shell scripting for Derecho

NCAR · Feb 10, 2024 · e5bd5c2 · e5bd5c2
1 parent dbad6e9
commit e5bd5c2
Show file tree

Hide file tree

Showing 7 changed files with 37 additions and 38 deletions.
diff --git a/models/wrf/shell_scripts/assim_advance.csh b/models/wrf/shell_scripts/assim_advance.csh
@@ -60,15 +60,15 @@ $yyyy $mm $dd $hh $nn $ss
 mpirun.lsf ./wrf.exe
 EOF
 
-else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+else if ( $SUPER_PLATFORM == 'derecho' ) then
 
 # module load openmpi
 cat >! $RUN_DIR/advance_temp${emember}/wrf.info << EOF
 ${gdatef[2]}  ${gdatef[1]}
 ${gdate[2]}   ${gdate[1]}
 $yyyy $mm $dd $hh $nn $ss
            $domains
- mpiexec_mpt dplace -s 1  ./wrf.exe
+ mpiexec -n 128 -ppn 128  ./wrf.exe
 EOF
 
 endif

diff --git a/models/wrf/shell_scripts/assimilate.csh b/models/wrf/shell_scripts/assimilate.csh
@@ -29,14 +29,13 @@ if ( $SUPER_PLATFORM == 'yellowstone' ) then
    setenv FORT_BUFFERED true
    mpirun.lsf ./filter || exit 1
 
-else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+else if ( $SUPER_PLATFORM == 'derecho' ) then
 
-# TJH MPI_SHEPHERD TRUE may be a very bad thing
    setenv MPI_SHEPHERD FALSE
 
    setenv TMPDIR  /dev/shm
    limit stacksize unlimited
-   mpiexec_mpt dplace -s 1 ./filter || exit 1
+   mpiexec -n 256 -ppn 128 ./filter || exit 1
 
 endif
 

diff --git a/models/wrf/shell_scripts/driver.csh b/models/wrf/shell_scripts/driver.csh
@@ -82,9 +82,9 @@ while ( 1 == 1 )
       set ic_queue = caldera
       set logfile = "${RUN_DIR}/ic_gen.log"
       set sub_command = "bsub -q ${ic_queue} -W 00:05 -o ${logfile} -n 1 -P ${COMPUTER_CHARGE_ACCOUNT}"
-   else if ( $SUPER_PLATFORM == 'cheyenne' ) then
-      set ic_queue = "economy"
-      set sub_command = "qsub -l select=1:ncpus=2:mpiprocs=36:mem=5GB -l walltime=00:03:00 -q ${ic_queue} -A ${COMPUTER_CHARGE_ACCOUNT} -j oe -k eod -N icgen "
+   else if ( $SUPER_PLATFORM == 'derecho' ) then
+      set ic_queue = "main"
+      set sub_command = "qsub -l select=1:ncpus=128:mpiprocs=128:mem=5GB -l walltime=00:03:00 -q ${ic_queue} -A ${COMPUTER_CHARGE_ACCOUNT} -j oe -k eod -N icgen "
    endif
 
    echo "this platform is $SUPER_PLATFORM and the job submission command is $sub_command"
@@ -120,7 +120,7 @@ while ( 1 == 1 )
 
    set n = 1
    while ( $n <= $NUM_ENS )
-      if ( $SUPER_PLATFORM == 'cheyenne' ) then   # can't pass along arguments in the same way
+      if ( $SUPER_PLATFORM == 'derecho' ) then   # can't pass along arguments in the same way
          $sub_command -v mem_num=${n},date=${datep},domain=${domains},paramf=${paramfile} ${SHELL_SCRIPTS_DIR}/prep_ic.csh
       else
          $sub_command " ${SHELL_SCRIPTS_DIR}/prep_ic.csh ${n} ${datep} ${dn} ${paramfile} "
@@ -147,9 +147,8 @@ while ( 1 == 1 )
             @ loop++
             if ( $loop > 60 ) then    # wait 5 minutes for the ic file to be ready, else run manually
                echo "gave up on ic member $n - redo"
-               # TJH this is not the command for cheyenne, why not $sub_command from above
                ${SHELL_SCRIPTS_DIR}/prep_ic.csh ${n} ${datep} ${dn} ${paramfile}
-               # TJH the job queued above is still queued and should be killed ...
+               # If manual execution of script, shouldn't queued job be killed?
             endif
          endif
       end
@@ -241,7 +240,7 @@ while ( 1 == 1 )
       endif
       set this_filter_runtime = $FILTER_TIME
 
-   else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+   else if ( $SUPER_PLATFORM == 'derecho' ) then
 
       echo "2i\"                                                                          >! script.sed
       echo "#=================================================================\"          >> script.sed
@@ -250,6 +249,7 @@ while ( 1 == 1 )
       echo "#PBS -A ${COMPUTER_CHARGE_ACCOUNT}\"                                          >> script.sed
       echo "#PBS -l walltime=${FILTER_TIME}\"                                             >> script.sed
       echo "#PBS -q ${FILTER_QUEUE}\"                                                     >> script.sed
+      echo "#PBS -l job_priority=${FILTER_PRIORITY}\"                                     >> script.sed
       echo "#PBS -m ae\"                                                                  >> script.sed
       echo "#PBS -M ${EMAIL}\"                                                            >> script.sed
       echo "#PBS -k eod\"                                                                 >> script.sed
@@ -407,7 +407,7 @@ while ( 1 == 1 )
             bsub < assim_advance_mem${n}.csh
          endif
 
-      else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+      else if ( $SUPER_PLATFORM == 'derecho' ) then
 
          echo "2i\"                                                                             >! script.sed
          echo "#=================================================================\"             >> script.sed
@@ -416,6 +416,7 @@ while ( 1 == 1 )
          echo "#PBS -A ${COMPUTER_CHARGE_ACCOUNT}\"                                             >> script.sed
          echo "#PBS -l walltime=${ADVANCE_TIME}\"                                               >> script.sed
          echo "#PBS -q ${ADVANCE_QUEUE}\"                                                       >> script.sed
+         echo "#PBS -l job_priority=${ADVANCE_PRIORITY}\"                                       >> script.sed
          echo "#PBS -m a\"                                                                      >> script.sed
          echo "#PBS -M ${EMAIL}\"                                                               >> script.sed
          echo "#PBS -k eod\"                                                                    >> script.sed
@@ -470,7 +471,7 @@ while ( 1 == 1 )
 
                endif
 
-            else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+            else if ( $SUPER_PLATFORM == 'derecho' ) then
 
                if ( `qstat -wa | grep assim_advance_${n} | wc -l` == 0 ) then
 
@@ -511,7 +512,7 @@ while ( 1 == 1 )
                      bsub < assim_advance_mem${n}.csh
       	          endif
 
-               else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+               else if ( $SUPER_PLATFORM == 'derecho' ) then
 
                   qsub assim_advance_mem${n}.csh
 

diff --git a/models/wrf/shell_scripts/first_advance.csh b/models/wrf/shell_scripts/first_advance.csh
@@ -48,19 +48,16 @@ if ( $SUPER_PLATFORM == 'yellowstone' ) then
  mpirun.lsf ./wrf.exe
 EOF
 
-else if ( $SUPER_PLATFORM == 'cheyenne' ) then
+else if ( $SUPER_PLATFORM == 'derecho' ) then
 
-   # TJH MPI_IB_CONGESTED, MPI_LAUNCH_TIMEOUT used after cheyenne O/S change in July 2019
-   # TJH setenv MPI_IB_CONGESTED 1
-   # TJH setenv MPI_LAUNCH_TIMEOUT 40
    setenv MPI_SHEPHERD false
 
    cat >! $RUN_DIR/advance_temp${emember}/wrf.info << EOF
  ${gdatef[2]}  ${gdatef[1]}
  ${gdate[2]}   ${gdate[1]}
  $yyyy $mm $dd $hh $nn $ss
            $domains
- mpiexec_mpt dplace -s 1 ./wrf.exe
+ mpiexec -n 128 -ppn 128 ./wrf.exe
 EOF
 
 endif

diff --git a/models/wrf/shell_scripts/gen_retro_icbc.csh b/models/wrf/shell_scripts/gen_retro_icbc.csh
@@ -42,7 +42,7 @@ echo "gen_retro_icbc.csh is running in `pwd`"
 
 set datea     = 2017042700
 set datefnl   = 2017042712 # set this appropriately #%%%#
-set paramfile = /glade2/scratch2/USERNAME/WORK_DIR/scripts/param.csh   # set this appropriately #%%%#
+set paramfile = /glade/derecho/scratch/USERNAME/WORK_DIR/scripts/param.csh   # set this appropriately #%%%#
 
 source $paramfile
 
@@ -171,22 +171,23 @@ EOF
      #if ( -e rsl.out.0000 )  cat rsl.out.0000 >> out.real.exe
 
       rm script.sed real_done rsl.*
-      echo "2i\"                                      >! script.sed
-      echo "#======================================\" >> script.sed
-      echo "#PBS -N run_real\"                        >> script.sed
-      echo "#PBS -A ${COMPUTER_CHARGE_ACCOUNT}\"      >> script.sed
-      echo "#PBS -l walltime=00:05:00\"               >> script.sed
-      echo "#PBS -q ${ADVANCE_QUEUE}\"                >> script.sed
-      echo "#PBS -o run_real.out\"                    >> script.sed
-      echo "#PBS -j oe\"                              >> script.sed
-      echo "#PBS -k eod\"                             >> script.sed
-      echo "#PBS -l select=3:ncpus=36:mpiprocs=36\"   >> script.sed
-      echo "#PBS -V\"                                 >> script.sed
-      echo "#======================================\" >> script.sed
-      echo "\"                                        >> script.sed
-      echo ""                                         >> script.sed
-      echo 's%${1}%'"${paramfile}%g"                  >> script.sed
-      sed -f script.sed ${SHELL_SCRIPTS_DIR}/real.csh >! real.csh
+      echo "2i\"                                       >! script.sed
+      echo "#======================================\"  >> script.sed
+      echo "#PBS -N run_real\"                         >> script.sed
+      echo "#PBS -A ${COMPUTER_CHARGE_ACCOUNT}\"       >> script.sed
+      echo "#PBS -l walltime=00:05:00\"                >> script.sed
+      echo "#PBS -q ${ADVANCE_QUEUE}\"                 >> script.sed
+      echo "#PBS -l job_priority=${ADVANCE_PRIORITY}\" >> script.sed
+      echo "#PBS -o run_real.out\"                     >> script.sed
+      echo "#PBS -j oe\"                               >> script.sed
+      echo "#PBS -k eod\"                              >> script.sed
+      echo "#PBS -l select=1:ncpus=128:mpiprocs=128\"  >> script.sed
+      echo "#PBS -V\"                                  >> script.sed
+      echo "#======================================\"  >> script.sed
+      echo "\"                                         >> script.sed
+      echo ""                                          >> script.sed
+      echo 's%${1}%'"${paramfile}%g"                   >> script.sed
+      sed -f script.sed ${SHELL_SCRIPTS_DIR}/real.csh  >! real.csh
 
       qsub real.csh
 

diff --git a/models/wrf/shell_scripts/init_ensemble_var.csh b/models/wrf/shell_scripts/init_ensemble_var.csh
@@ -83,6 +83,7 @@ EOF
 #PBS -A ${COMPUTER_CHARGE_ACCOUNT}
 #PBS -l walltime=${ADVANCE_TIME}
 #PBS -q ${ADVANCE_QUEUE}
+#PBS -l job_priority=${ADVANCE_PRIORITY}
 #PBS -m ae
 #PBS -M ${EMAIL}
 #PBS -k eod

diff --git a/models/wrf/shell_scripts/real.csh b/models/wrf/shell_scripts/real.csh
@@ -5,7 +5,7 @@
  source $paramfile
 
  cd ${ICBC_DIR}
- mpiexec_mpt dplace -s 1 ${RUN_DIR}/WRF_RUN/real.exe
+ mpiexec -n 128 -ppn 128 ${RUN_DIR}/WRF_RUN/real.exe
 
 #if ( `grep "Successful completion of program real.exe" ./rsl.out.0000 | wc -l ` == 1  )  touch ${ICBC_DIR}/real_done