diff --git a/test/gw-ci/CMakeLists.txt b/test/gw-ci/CMakeLists.txt index 6c9faedf1..dc21fc84a 100644 --- a/test/gw-ci/CMakeLists.txt +++ b/test/gw-ci/CMakeLists.txt @@ -1,11 +1,37 @@ +# Function that adds a test for a given GW task +function(add_task task_name test_prefix cycle pslot RUNTESTS PROJECT_SOURCE_DIR) + # Get subtask names if task is a metatask + if("${task_name}" STREQUAL "gdas_fcst") + set(subtask_names "gdas_fcst_seg0") + elseif("${task_name}" STREQUAL "enkfgdas_fcst") + set(subtask_names "enkfgdas_fcst_mem001 enkfgdas_fcst_mem002") + elseif("${task_name}" STREQUAL "gdas_atmos_prod") + set(subtask_names "gdas_atmos_prod_f000 gdas_atmos_prod_f001 gdas_atmos_prod_f002 gdas_atmos_prod_f003 gdas_atmos_prod_f004 gdas_atmos_prod_f005 gdas_atmos_prod_f006 gdas_atmos_prod_f007 gdas_atmos_prod_f008 gdas_atmos_prod_f009") + elseif("${task_name}" STREQUAL "enkfgdas_epmn") + set(subtask_names "enkfgdas_epos001 enkfgdas_epos002 enkfgdas_epos003 enkfgdas_epos004 enkfgdas_epos005 enkfgdas_epos006") + elseif("${task_name}" STREQUAL "enkfgdas_ecmn") + set(subtask_names "enkfgdas_ecen000") + else() + set(subtask_names ${task_name}) + endif() + + # Add Task + set(test_name ${test_prefix}_${task_name}_${cycle}) + message(STATUS "preparing ${subtask_names} for ${test_prefix} ctest") + add_test(NAME ${test_name} + COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} ${cycle} ${subtask_names}" + WORKING_DIRECTORY ${RUNTESTS}) + set_tests_properties(${test_name} PROPERTIES LABELS "manual") +endfunction() + # Function that generates the 1/2 cycle forecast and DA tasks function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR TASK_LIST) - set(test_name test_gdasapp_${pslot}) + set(test_prefix test_gdasapp_${pslot}) # Prepare the COMROOT and EXPDIR for the cycling ctests - add_test(NAME ${test_name} + add_test(NAME ${test_prefix} COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/create_exp.sh ${YAML_PATH} ${pslot} ${HOMEgfs} ${RUNTESTS}" WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name} PROPERTIES LABELS "manual") + set_tests_properties(${test_prefix} PROPERTIES LABELS "manual") # Get the 1/2 cycle and full cycle's dates execute_process( @@ -18,29 +44,23 @@ function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR T list(GET DATES_LIST 0 HALF_CYCLE) list(GET DATES_LIST 1 FULL_CYCLE) - # stage IC's - message(STATUS "staging the 1/2 cycle IC's for ${test_name} ctest") - add_test(NAME ${test_name}_gdas_stage_ic_${HALF_CYCLE} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_stage_ic ${HALF_CYCLE}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_gdas_stage_ic_${HALF_CYCLE} PROPERTIES LABELS "manual") + # Add tests used by all CI tests + add_task("gdas_stage_ic" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR}) + add_task("gdas_fcst" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR}) - # 1/2 cycle gdas_fcst - message(STATUS "preparing 1/2 cycle gdas_fcst for ${pslot} ctest") - add_test(NAME ${test_name}_gdas_fcst_${HALF_CYCLE} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_fcst_seg0 ${HALF_CYCLE}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_gdas_fcst_${HALF_CYCLE} PROPERTIES LABELS "manual") + # Atm-hyb-C96C48 tests + if(pslot STREQUAL "Atm-hyb-C96C48") + add_task("enkfgdas_stage_ic" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR}) + add_task("enkfgdas_fcst" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR}) + add_task("gdas_atmos_prod" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR}) + add_task("enkfgdas_echgres" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR}) + add_task("enkfgdas_epmn" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR}) + endif() # Select the list of tasks to run for the full cycle message(STATUS "Tasks ${TASK_LIST}") - - foreach(task ${TASK_LIST}) - message(STATUS "preparing the full cycle ${task} for ${pslot} ctest") - add_test(NAME ${test_name}_${task}_${FULL_CYCLE} - COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} ${task} ${FULL_CYCLE}" - WORKING_DIRECTORY ${RUNTESTS}) - set_tests_properties(${test_name}_${task}_${FULL_CYCLE} PROPERTIES LABELS "manual") + foreach(task_name ${TASK_LIST}) + add_task(${task_name} ${test_prefix} ${FULL_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR}) endforeach() endfunction() @@ -77,7 +97,25 @@ if (RUN_GW_CI) # --------------- set(pslot "Atm-hyb-C96C48") set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C96C48_ufs_hybatmDA.yaml) - set(TASK_LIST) # empty list for now + set(TASK_LIST + "gdas_prep" + "gdas_prepatmiodaobs" + "gdas_atmanlinit" + "gdas_atmanlvar" + "gdas_atmanlfv3inc" + "gdas_atmanlfinal" + "gdas_sfcanl" + "gdas_analcalc" + "gdas_fcst" + "enkfgdas_atmensanlinit" + "enkfgdas_atmensanlobs" + "enkfgdas_atmensanlsol" + "enkfgdas_atmensanlfv3inc" + "enkfgdas_atmensanlfinal" + "enkfgdas_ecmn" + "enkfgdas_esfc" + "enkfgdas_fcst" + ) add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}") # GFSv17, 3DVAR prototype diff --git a/test/gw-ci/run_exp.sh b/test/gw-ci/run_exp.sh index 8040860a0..4555cdb62 100755 --- a/test/gw-ci/run_exp.sh +++ b/test/gw-ci/run_exp.sh @@ -1,39 +1,50 @@ #!/bin/bash pslot=$1 -TASK_NAME=$2 -CYCLE=$3 +CYCLE=$2 +shift +shift +TASK_NAMES=("$@") + +task_args=$(printf " -t %s" "${TASK_NAMES[@]}") +num_tasks=${#TASK_NAMES[@]} # Define the workflow XML and database files WORKFLOW_XML=${pslot}/EXPDIR/${pslot}/${pslot}.xml WORKFLOW_DB=${pslot}/EXPDIR/${pslot}/${pslot}.db # Boot the task -echo "booting $TASK_NAME for cycle $CYCLE" +echo "booting ${TASK_NAMES[@]} for cycle $CYCLE" if [[ ! -e "$WORKFLOW_DB" ]]; then - rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE" + rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" "$task_args" -c "$CYCLE" fi -rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE" +rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" "$task_args" -c "$CYCLE" while true; do # Update the status of the task - rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE" + rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" "$task_args" -c "$CYCLE" - # Check the task status - OUTPUT=$(rocotostat -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE") - STATUS=$(echo "$OUTPUT" | awk '$2 == task {print $4}' task="$TASK_NAME") + num_succeeded=0 + for task in "${TASK_NAMES[@]}"; do + # Check the task status + OUTPUT=$(rocotostat -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task" -c "$CYCLE") + STATUS=$(echo "$OUTPUT" | awk '$2 == task {print $4}' task="$task") - if [[ "$STATUS" == "SUCCEEDED" ]]; then - echo "The task succeeded." + if [[ "$STATUS" == "SUCCEEDED" ]]; then + echo "$task succeeded." + num_succeeded=$((num_succeeded + 1)) + elif [[ "$STATUS" == "FAILED" ]]; then + echo "$task failed." + exit 1 + elif [[ "$STATUS" == "DEAD" ]]; then + echo "$task is dead." + exit 1 + else + echo "$task is in state: $STATUS" + fi + done + if [[ "$num_succeeded" == "$num_tasks" ]]; then exit 0 - elif [[ "$STATUS" == "FAILED" ]]; then - echo "The task failed." - exit 1 - elif [[ "$STATUS" == "DEAD" ]]; then - echo "The task is dead." - exit 1 - else - echo "The task is in state: $STATUS" fi sleep 10 done