Merge pull request #12 from cacraigucar/cam_derecho_test

Add GPU regression testing

cime_config/testdefs/testlist_cam.xml

@@ -1480,6 +1480,24 @@
 <!--                    (unsupported)                         -->
 <!--              @@@@@@@@@@@@@@@@@@@@@@@@@@@                 -->
 
+  <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERS_Ln9_G4-a100-openacc" testmods="cam/outfrq9s_mg3_default">
+    <machines>
+      <machine name="derecho" compiler="nvhpc" category="derecho_gpu"/>
+      <machine name="derecho" compiler="nvhpc" category="aux_cam"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:30:00</option>
+    </options>
+  </test>   
+  <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERS_Ln9_G4-a100-openacc" testmods="cam/outfrq9s_mg3_pcols760">
+    <machines>
+      <machine name="derecho" compiler="nvhpc" category="derecho_gpu"/>
+      <machine name="derecho" compiler="nvhpc" category="prealpha"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:30:00</option>
+    </options>
+  </test>
   <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERP_D_Ln9" testmods="cam/outfrq9s" supported="false">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_pumas"/>
@@ -1542,51 +1560,6 @@
       <machine name="derecho" compiler="intel" category="camchem"/>
     </machines>
   </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9" testmods="cam/outfrq9s_mg3_default">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9" testmods="cam/outfrq9s_mg2_default">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9" testmods="cam/outfrq9s_mg3_nondefault">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9" testmods="cam/outfrq9s_mg3_pcols1536">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9_G4" testmods="cam/outfrq9s_mg3_default">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
   <test compset="QPC4" grid="f19_f19_mg17" name="ERP_Ln9" testmods="cam/outfrq9s">
     <machines>
       <machine name="derecho" compiler="intel" category="test_release"/>

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands

@@ -1,4 +1,4 @@
-./xmlchange NTASKS=36
+./xmlchange NTASKS=128
 ./xmlchange NTHRDS=1
 ./xmlchange ROOTPE='0'
 ./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/shell_commands → cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/shell_commands

@@ -1,8 +1,8 @@
-./xmlchange NTASKS=36
+./xmlchange NTASKS=64
 ./xmlchange NTHRDS=1
 ./xmlchange ROOTPE='0'
 ./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
 ./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
-./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -pcols 1536' --append
+./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -pcols 760 ' --append
 ./xmlchange TIMER_DETAIL='6'
 ./xmlchange TIMER_LEVEL='999'

test/system/archive_baseline.sh

@@ -12,7 +12,7 @@ cat << EOF1
 NAME
 
 	archive_baseline.sh - archive pretag baselines to set locations on
-                              hobart, izumi and derecho.
+                              izumi and derecho.
 
 
 SYNOPSIS
@@ -24,41 +24,28 @@ SYNOPSIS
 ENVIROMENT VARIABLES
 
 	CESM_TESTDIR - Directory that contains the CESM finished results you wish to archive.
-	CAM_FC      - Compiler used, only used on hobart and izumi (PGI,NAG), where the compiler
+	CAM_FC      - Compiler used,  used on derecho (INTEL, NVHPC) and izumi (GNU,NAG), where the compiler
                       name is appended to the archive directory.
 
 
 BASELINE ARCHIVED LOCATION
 
-	hobart, izumi:     /fs/cgd/csm/models/atm/cam/pretag_bl/TAGNAME_pgi
-	                   /fs/cgd/csm/models/atm/cam/pretag_bl/TAGNAME_nag
-        derecho:  /glade/campaign/cesm/community/amwg/cam_baselines/TAGNAME
+	izumi:    /fs/cgd/csm/models/atm/cam/pretag_bl/TAGNAME_gnu
+	          /fs/cgd/csm/models/atm/cam/pretag_bl/TAGNAME_nag
+        derecho:  /glade/campaign/cesm/community/amwg/cam_baselines/TAGNAME_intel
+                  /glade/campaign/cesm/community/amwg/cam_baselines/TAGNAME_nvhpc
 
 
 
 HOW TO USE ARCHIVE BASELINES
 
-	Set BL_TESTDIR to the archived baseline you wish to load.
+	on izumi:
+          env CESM_TESTDIR=/scratch/cluster/YourName/aux_cam_gnu_yyyymmddsssss CAM_FC=GNU ./archive_baseline.sh cam6_4_XXX
+          env CESM_TESTDIR=/scratch/cluster/YourName/aux_cam_nag_yyyymmddsssss CAM_FC=NAG ./archive_baseline.sh cam6_3_XXX
 
-
-WORK FLOW
-
-	This is an example for hobart or izumi.
-
-	Modify your sandbox with the changes you want.
-        setenv CAM_FC PGI
-        setenv CAM_TESTDIR /scratch/cluster/fischer/cam5_2_06
-        Run the cam test suite.
-        Make your trunk tag
-	archive_baseline.sh cam5_2_06
-
-	Create a new sandbox.
-        setenv CAM_FC PGI
-	setenv CAM_TESTDIR /scratch/cluster/fischer/cam5_2_07
-        setenv BL_TESTDIR /fs/cgd/csm/models/atm/cam/pretag_bl/cam5_2_06_pgi
-        Run the cam test suite.
-        Make your trunk tag
-        archive_baseline.sh cam5_2_07
+        on derecho:
+          env CESM_TESTDIR=/glade/derecho/scratch/YourName/aux_cam_intel_yyyymmddsssss CAM_FC=INTEL ./archive_baseline.sh cam6_4_XXX
+          env CESM_TESTDIR=/glade/derecho/scratch/YourName/aux_cam_nvhpc_yyyymmddsssss CAM_FC=NVHPC ./archive_baseline.sh cam6_4_XXX
 
 
 WARNING
@@ -73,20 +60,10 @@ fi
 hostname=`hostname`
 case $hostname in
 
-  ho*)
-    echo "server: hobart"
-    if [ -z "$CAM_FC" ]; then
-      CAM_FC="PGI"
-    fi
-    test_file_list="tests_pretag_hobart_${CAM_FC,,}"
-    cam_tag=$1_${CAM_FC,,}
-    baselinedir="/fs/cgd/csm/models/atm/cam/pretag_bl/$cam_tag"
-  ;;
-
   iz*)
     echo "server: izumi"
     if [ -z "$CAM_FC" ]; then
-      CAM_FC="PGI"
+       echo "Must specify CAM_FC"
     fi
     test_file_list="tests_pretag_izumi_${CAM_FC,,}"
     cam_tag=$1_${CAM_FC,,}
@@ -96,9 +73,9 @@ case $hostname in
   de*)
     echo "server: derecho"
     if [ -z "$CAM_FC" ]; then
-      CAM_FC="INTEL"
+      echo "Must specify CAM_FC"
     fi
-    test_file_list="tests_pretag_derecho"
+    test_file_list="tests_pretag_derecho_${CAM_FC,,}"
     cam_tag=$1
     baselinedir="/glade/campaign/cesm/community/amwg/cam_baselines/$cam_tag"
   ;;
@@ -130,7 +107,7 @@ fi
 
 case $hostname in
 
-    ch* | hobart | izumi)
+    de* | izumi)
 	if [ -z "$CESM_TESTDIR" ]; then
 	    echo '***********************************************************************************'
 	    echo 'INFO: The aux_cam and test_cam tests were NOT archived'

test/system/test_driver.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+!/bin/sh
 #
 # test_driver.sh:  driver for the testing of CAM with standalone scripts
 #
@@ -466,6 +466,9 @@ if [ "${hostname:0:6}" == "casper" ] || [ "${hostname:0:5}" == "crhtc" ]; then
 fi
 if [ -n "${CAM_FC}" ]; then
   comp="_${CAM_FC,,}"
+else
+  echo "ERROR: Must specify CAM_FC"
+  exit 1
 fi
 
 if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then
@@ -547,8 +550,6 @@ if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then
 
     if [ -n "${CAM_FC}" ]; then
       testargs="${testargs} --xml-compiler ${CAM_FC,,}"
-    else
-      testargs="${testargs} --xml-compiler intel"
     fi
     case $hostname in
         # derecho
@@ -586,8 +587,6 @@ if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then
         cmd="query_testlists --xml-category $cesm_test --xml-machine  ${cesm_test_mach}"
         if [ -n "${CAM_FC}" ]; then
             cmd="${cmd} --xml-compiler ${CAM_FC,,}"
-        else
-            cmd="${cmd} --xml-compiler intel"
         fi
         cmd="${CIME_ROOT}/scripts/"$cmd
         cime_testlist=`$cmd`