Centralize test data and reduce unnecessary copying of test data (NOA…

…A-EMC#147)

This PR centralizes test data and reduce unnecessary copying.
It addresses issue NOAA-EMC#93 and NOAA-EMC#127
All these will speed up the build process and save disk space.

.gitignore

@@ -146,8 +146,14 @@ build*/
 install*/
 
 # Ignore the following bundle repositories
-!/bundle/CMakeLists.txt
 /bundle/*
+!/bundle/CMakeLists.txt
+!/bundle/rrfs-test-data
+!/bundle/test-data-release
+!/bundle/ufo-data
+!/bundle/ioda-data
+!/bundle/fv3-jedi-data
+!/bundle/mpas-jedi-data
 
 # 
 expr

CMakeLists.txt

@@ -35,9 +35,7 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 # Handle user options.
 option(BUILD_RDASBUNDLE "Build RDAS Bundle" ON)
-option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF)
-option(CLONE_RRFSDATA "Clone RRFS test data" OFF)
-option(WORKFLOW_TESTS "Include regional-workflow dependent tests" OFF)
+option(ADD_RRFS_TESTS "Add RRFS tests" OFF)
 option(MPAS_DYCORE "Use MPAS" OFF)
 option(FV3_DYCORE "Use FV3" ON)
 

build.sh

@@ -25,7 +25,6 @@ usage() {
   echo "  -f  force a clean build             DEFAULT: NO"
   echo "  -s  only build a subset of the bundle  DEFAULT: NO"
   echo "  -m  select dycore                      DEFAULT: FV3andMPAS"
-  echo "  --notestdata  do not include JCSDA and rrfs ctest data  DEFAULT: NO"
   echo "  -h  display this message and quit"
   echo
   exit 1
@@ -38,8 +37,7 @@ INSTALL_PREFIX=""
 CMAKE_OPTS=""
 BUILD_TARGET="${MACHINE_ID:-'localhost'}"
 BUILD_VERBOSE="NO"
-CLONE_JCSDADATA="YES"
-CLONE_RRFSDATA="YES"
+ADD_RRFS_TESTS="YES"
 CLEAN_BUILD="NO"
 BUILD_JCSDA="YES"
 DYCORE="FV3andMPAS"
@@ -68,12 +66,6 @@ while getopts "p:t:c:m:hvfs-:" opt; do
     s)
       BUILD_JCSDA=NO
       ;;
-    -)
-      if [[ "${OPTARG}" == "notestdata" ]]; then
-        CLONE_JCSDADATA=NO
-        CLONE_RRFSDATA=NO
-      fi 
-      ;;
     h|\?|:)
       usage
       ;;
@@ -96,7 +88,7 @@ case ${BUILD_TARGET} in
     ;;
 esac
 
-CMAKE_OPTS+=" -DCLONE_JCSDADATA=$CLONE_JCSDADATA -DCLONE_RRFSDATA=$CLONE_RRFSDATA"
+CMAKE_OPTS+=" -DADD_RRFS_TESTS=$ADD_RRFS_TESTS"
 
 BUILD_DIR=${BUILD_DIR:-$dir_root/build}
 if [[ $CLEAN_BUILD == 'YES' ]]; then
@@ -145,17 +137,7 @@ if [[ $DYCORE == 'MPAS' || $DYCORE == 'FV3andMPAS' ]]; then
   $dir_root/rrfs-test/scripts/link_mpasjedi_expr.sh
 fi
 
-# JCSDA changed test data things, need to make a dummy CRTM directory
-if [[ $BUILD_TARGET == 'hera' ]]; then
-  if [ -d "$dir_root/bundle/fix/test-data-release/" ]; then rm -rf $dir_root/bundle/fix/test-data-release/; fi
-  if [ -d "$dir_root/bundle/test-data-release/" ]; then rm -rf $dir_root/bundle/test-data-release/; fi
-  mkdir -p $dir_root/bundle/fix/test-data-release/
-  mkdir -p $dir_root/bundle/test-data-release/
-  ln -sf $RDASAPP_TESTDATA/crtm $dir_root/bundle/fix/test-data-release/crtm
-  ln -sf $RDASAPP_TESTDATA/crtm $dir_root/bundle/test-data-release/crtm
-fi
-
-  CMAKE_OPTS+=" -DMPIEXEC_MAX_NUMPROCS:STRING=120"
+CMAKE_OPTS+=" -DMPIEXEC_MAX_NUMPROCS:STRING=120"
 # Configure
 echo "Configuring ..."
 set -x

bundle/CMakeLists.txt

@@ -33,8 +33,6 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 # Handle user options.
 option(BUILD_RDASBUNDLE "Build RDAS Bundle" ON)
-option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF)
-option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF)
 
 # Initialize bundle
 # -----------------
@@ -110,38 +108,9 @@ endif()
 # rdas mega executable
   ecbuild_bundle( PROJECT rdas SOURCE "../mains" )
 
-# ioda, ufo, fv3-jedi, and saber test data
-#---------------------------------
-  if(CLONE_JCSDADATA)
-
-   set(JCSDA_DATA_ROOT "$ENV{RDASAPP_TESTDATA}/jcsda")
-   ecbuild_bundle( PROJECT ioda-data SOURCE "${JCSDA_DATA_ROOT}/ioda-data" )
-   ecbuild_bundle( PROJECT ufo-data SOURCE "${JCSDA_DATA_ROOT}/ufo-data" )
-   if(FV3_DYCORE)
-     ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "${JCSDA_DATA_ROOT}/fv3-jedi-data" )
-   endif()
-   if(MPAS_DYCORE)
-     set(DESTINATION_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
-     file( COPY "${JCSDA_DATA_ROOT}/mpas-jedi-data" DESTINATION ${DESTINATION_DIR} )
-   endif()
-
-  endif(CLONE_JCSDADATA)
-# rrfs-test data (fv3 and/or mpas)
-#---------------------------------
-  if(CLONE_RRFSDATA)
-     set(RDAS_RRFS_DATA_ROOT "$ENV{RDAS_RRFS_DATA_ROOT}/rrfs-test-data")
-     set(DESTINATION_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
-     if(NOT EXISTS "${DESTINATION_DIR}/rrfs-test-data")
-        file(MAKE_DIRECTORY "${DESTINATION_DIR}/rrfs-test-data")
-     endif()
-     if(FV3_DYCORE)
-     file( COPY "${RDAS_RRFS_DATA_ROOT}/rrfs-data_fv3jedi_2022052619" DESTINATION "${DESTINATION_DIR}/rrfs-test-data" )
-     endif()
-     if(MPAS_DYCORE)
-     file( COPY "${CMAKE_CURRENT_SOURCE_DIR}/../expr/mpas_2024052700/" DESTINATION "${DESTINATION_DIR}/rrfs-test-data/rrfs-data_mpasjedi_2024052700" )
-     endif()
-  ecbuild_bundle( PROJECT rrfs-test SOURCE "../rrfs-test" )
-  endif(CLONE_RRFSDATA)
+  if(ADD_RRFS_TESTS)
+    ecbuild_bundle( PROJECT rrfs-test SOURCE "../rrfs-test" )
+  endif(ADD_RRFS_TESTS)
 
 endif(BUILD_RDASBUNDLE)
 

bundle/fv3-jedi-data

@@ -0,0 +1 @@
+../fix/jcsda/fv3-jedi-data

bundle/ioda-data

@@ -0,0 +1 @@
+../fix/jcsda/ioda-data

bundle/mpas-jedi-data/testinput_tier_1/384km/init/x1.4002.init.2018-04-15_00.00.00.nc

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/384km/init/x1.4002.init.2018-04-15_00.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km/bg/ensemble

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km/bg/ensemble

bundle/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-14_21.00.00.nc

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-14_21.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-15_00.00.00.nc

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-15_00.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-15_03.00.00.nc

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-15_03.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km_2stream/mpasout.2018-04-15_00.00.00.nc

@@ -0,0 +1 @@
+../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km_2stream/mpasout.2018-04-15_00.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km_2stream/static.nc

@@ -0,0 +1 @@
+../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km_2stream/static.nc

bundle/mpas-jedi-data/testinput_tier_1/480km_2stream/x1.2562.grid.nc

@@ -0,0 +1 @@
+../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km_2stream/x1.2562.grid.nc

bundle/mpas-jedi-data/testinput_tier_1/480km_2stream/x1.2562.init.2018-04-14_18.00.00.nc

@@ -0,0 +1 @@
+../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km_2stream/x1.2562.init.2018-04-14_18.00.00.nc

bundle/rrfs-test-data/rrfs-data_fv3jedi_2022052619

@@ -0,0 +1 @@
+../../fix/rrfs-test-data/rrfs-data_fv3jedi_2022052619

bundle/rrfs-test-data/rrfs-data_mpasjedi_2024052700

@@ -0,0 +1 @@
+../../expr/mpas_2024052700

bundle/test-data-release/crtm

@@ -0,0 +1 @@
+../../fix/crtm

bundle/ufo-data

@@ -0,0 +1 @@
+../fix/jcsda/ufo-data

fix/crtm

@@ -0,0 +1 @@
+.agent/crtm

fix/jcsda/fv3-jedi-data

@@ -0,0 +1 @@
+../.agent/jcsda/fv3-jedi-data

fix/jcsda/ioda-data

@@ -0,0 +1 @@
+../.agent/jcsda/ioda-data

fix/jcsda/mpas-jedi-data

@@ -0,0 +1 @@
+../.agent/jcsda/mpas-jedi-data

fix/jcsda/ufo-data

@@ -0,0 +1 @@
+../.agent/jcsda/ufo-data_20240909

fix/rrfs-test-data

@@ -0,0 +1 @@
+.agent/rrfs-test-data

modulefiles/RDAS/hera.intel.lua

@@ -84,14 +84,6 @@ local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
-setenv("CRTM_FIX","/scratch1/NCEPDEV/da/Cory.R.Martin/RDASApp/fix/crtm/2.4.0")
-setenv("RDASAPP_TESTDATA","/scratch1/NCEPDEV/da/Cory.R.Martin/CI/RDASApp/data")
---setenv("RDAS_RRFS_DATA_ROOT","/scratch2/NCEPDEV/fv3-cam/Ting.Lei/dr-emc-rdas/")
---setenv("RDAS_RRFS_DATA_ROOT", "/scratch1/NCEPDEV/da/Xiaoyan.Zhang/RRFS")
-setenv("RDAS_RRFS_DATA_ROOT", "/scratch1/NCEPDEV/da/Samuel.Degelia/RRFS")
-
---prepend_path("PATH","/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/prod_util/1.2.2/bin")
-
 whatis("Name: ".. pkgName)
 whatis("Version: ".. pkgVersion)
 whatis("Category: RDASApp")

modulefiles/RDAS/hercules.intel.lua

@@ -80,10 +80,6 @@ local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
-setenv("CRTM_FIX","/work2/noaa/da/cmartin/RDASApp/fix/crtm/2.4.0")
-setenv("RDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/RDASApp/data")
-prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin")
-
 execute{cmd="ulimit -s unlimited",modeA={"load"}}
 
 whatis("Name: ".. pkgName)

modulefiles/RDAS/jet.intel.lua

@@ -85,10 +85,6 @@ local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
-setenv("CRTM_FIX","/lfs5/BMC/nrtrr/RDAS_DATA/crtm/2.4.0")
-setenv("RDASAPP_TESTDATA","/lfs5/BMC/nrtrr/RDAS_DATA")
-setenv("RDAS_RRFS_DATA_ROOT", "/lfs5/BMC/nrtrr/RDAS_DATA")
-
 whatis("Name: ".. pkgName)
 whatis("Version: ".. pkgVersion)
 whatis("Category: RDASApp")

modulefiles/RDAS/orion.intel.lua

@@ -79,12 +79,6 @@ local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
-setenv("CRTM_FIX","/work2/noaa/da/cmartin/RDASApp/fix/crtm/2.4.0")
-setenv("RDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/RDASApp/data")
---setenv("RDAS_RRFS_DATA_ROOT","/work/noaa/fv3-cam/tlei/dr-emc-rdas/")
-setenv("RDAS_RRFS_DATA_ROOT", "/work2/noaa/fv3-cam/sdegelia/RRFS")
-prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin")
-
 execute{cmd="ulimit -s unlimited",modeA={"load"}}
 
 whatis("Name: ".. pkgName)

rrfs-test/CMakeLists.txt

@@ -18,8 +18,7 @@ add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_letkf"    "mpasj
 add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_getkf"    "mpasjedi_enkf.x")
 add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_bumploc"  "mpasjedi_error_covariance_toolbox.x")
 
-# rrfs data - from rrfs-test-data repo if found, from local directory, or from tarball
-if (CLONE_RRFSDATA)
+if (ADD_RRFS_TESTS)
    message(STATUS "Use test data from rrfs-test-data repository")
    set (rrfs-test_data_local "${CMAKE_SOURCE_DIR}/rrfs-test-data/")
    set (src_yaml "${CMAKE_SOURCE_DIR}/rrfs-test/testinput")