Skip to content

Commit

Permalink
Initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
DavidNew-NOAA committed Nov 4, 2024
1 parent 764f58c commit 88af2ab
Show file tree
Hide file tree
Showing 2 changed files with 91 additions and 42 deletions.
84 changes: 61 additions & 23 deletions test/gw-ci/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,11 +1,37 @@
# Function that adds a test for a given GW task
function(add_task task_name test_prefix cycle pslot RUNTESTS PROJECT_SOURCE_DIR)
# Get subtask names if task is a metatask
if("${task_name}" STREQUAL "gdas_fcst")
set(subtask_names "gdas_fcst_seg0")
elseif("${task_name}" STREQUAL "enkfgdas_fcst")
set(subtask_names "enkfgdas_fcst_mem001 enkfgdas_fcst_mem002")
elseif("${task_name}" STREQUAL "gdas_atmos_prod")
set(subtask_names "gdas_atmos_prod_f000 gdas_atmos_prod_f001 gdas_atmos_prod_f002 gdas_atmos_prod_f003 gdas_atmos_prod_f004 gdas_atmos_prod_f005 gdas_atmos_prod_f006 gdas_atmos_prod_f007 gdas_atmos_prod_f008 gdas_atmos_prod_f009")
elseif("${task_name}" STREQUAL "enkfgdas_epmn")
set(subtask_names "enkfgdas_epos001 enkfgdas_epos002 enkfgdas_epos003 enkfgdas_epos004 enkfgdas_epos005 enkfgdas_epos006")
elseif("${task_name}" STREQUAL "enkfgdas_ecmn")
set(subtask_names "enkfgdas_ecen000")
else()
set(subtask_names ${task_name})
endif()

# Add Task
set(test_name ${test_prefix}_${task_name}_${cycle})
message(STATUS "preparing ${subtask_names} for ${test_prefix} ctest")
add_test(NAME ${test_name}
COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} ${cycle} ${subtask_names}"
WORKING_DIRECTORY ${RUNTESTS})
set_tests_properties(${test_name} PROPERTIES LABELS "manual")
endfunction()

# Function that generates the 1/2 cycle forecast and DA tasks
function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR TASK_LIST)
set(test_name test_gdasapp_${pslot})
set(test_prefix test_gdasapp_${pslot})
# Prepare the COMROOT and EXPDIR for the cycling ctests
add_test(NAME ${test_name}
add_test(NAME ${test_prefix}
COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/create_exp.sh ${YAML_PATH} ${pslot} ${HOMEgfs} ${RUNTESTS}"
WORKING_DIRECTORY ${RUNTESTS})
set_tests_properties(${test_name} PROPERTIES LABELS "manual")
set_tests_properties(${test_prefix} PROPERTIES LABELS "manual")

# Get the 1/2 cycle and full cycle's dates
execute_process(
Expand All @@ -18,29 +44,23 @@ function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR T
list(GET DATES_LIST 0 HALF_CYCLE)
list(GET DATES_LIST 1 FULL_CYCLE)

# stage IC's
message(STATUS "staging the 1/2 cycle IC's for ${test_name} ctest")
add_test(NAME ${test_name}_gdas_stage_ic_${HALF_CYCLE}
COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_stage_ic ${HALF_CYCLE}"
WORKING_DIRECTORY ${RUNTESTS})
set_tests_properties(${test_name}_gdas_stage_ic_${HALF_CYCLE} PROPERTIES LABELS "manual")
# Add tests used by all CI tests
add_task("gdas_stage_ic" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR})
add_task("gdas_fcst" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR})

# 1/2 cycle gdas_fcst
message(STATUS "preparing 1/2 cycle gdas_fcst for ${pslot} ctest")
add_test(NAME ${test_name}_gdas_fcst_${HALF_CYCLE}
COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdas_fcst_seg0 ${HALF_CYCLE}"
WORKING_DIRECTORY ${RUNTESTS})
set_tests_properties(${test_name}_gdas_fcst_${HALF_CYCLE} PROPERTIES LABELS "manual")
# Atm-hyb-C96C48 tests
if(pslot STREQUAL "Atm-hyb-C96C48")
add_task("enkfgdas_stage_ic" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR})
add_task("enkfgdas_fcst" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR})
add_task("gdas_atmos_prod" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR})
add_task("enkfgdas_echgres" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR})
add_task("enkfgdas_epmn" ${test_prefix} ${HALF_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR})
endif()

# Select the list of tasks to run for the full cycle
message(STATUS "Tasks ${TASK_LIST}")

foreach(task ${TASK_LIST})
message(STATUS "preparing the full cycle ${task} for ${pslot} ctest")
add_test(NAME ${test_name}_${task}_${FULL_CYCLE}
COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} ${task} ${FULL_CYCLE}"
WORKING_DIRECTORY ${RUNTESTS})
set_tests_properties(${test_name}_${task}_${FULL_CYCLE} PROPERTIES LABELS "manual")
foreach(task_name ${TASK_LIST})
add_task(${task_name} ${test_prefix} ${FULL_CYCLE} ${pslot} ${RUNTESTS} ${PROJECT_SOURCE_DIR})
endforeach()
endfunction()

Expand Down Expand Up @@ -77,7 +97,25 @@ if (RUN_GW_CI)
# ---------------
set(pslot "Atm-hyb-C96C48")
set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C96C48_ufs_hybatmDA.yaml)
set(TASK_LIST) # empty list for now
set(TASK_LIST
"gdas_prep"
"gdas_prepatmiodaobs"
"gdas_atmanlinit"
"gdas_atmanlvar"
"gdas_atmanlfv3inc"
"gdas_atmanlfinal"
"gdas_sfcanl"
"gdas_analcalc"
"gdas_fcst"
"enkfgdas_atmensanlinit"
"enkfgdas_atmensanlobs"
"enkfgdas_atmensanlsol"
"enkfgdas_atmensanlfv3inc"
"enkfgdas_atmensanlfinal"
"enkfgdas_ecmn"
"enkfgdas_esfc"
"enkfgdas_fcst"
)
add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}")

# GFSv17, 3DVAR prototype
Expand Down
49 changes: 30 additions & 19 deletions test/gw-ci/run_exp.sh
Original file line number Diff line number Diff line change
@@ -1,39 +1,50 @@
#!/bin/bash

pslot=$1
TASK_NAME=$2
CYCLE=$3
CYCLE=$2
shift
shift
TASK_NAMES=("$@")

task_args=$(printf " -t %s" "${TASK_NAMES[@]}")
num_tasks=${#TASK_NAMES[@]}

# Define the workflow XML and database files
WORKFLOW_XML=${pslot}/EXPDIR/${pslot}/${pslot}.xml
WORKFLOW_DB=${pslot}/EXPDIR/${pslot}/${pslot}.db

# Boot the task
echo "booting $TASK_NAME for cycle $CYCLE"
echo "booting ${TASK_NAMES[@]} for cycle $CYCLE"
if [[ ! -e "$WORKFLOW_DB" ]]; then
rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE"
rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" "$task_args" -c "$CYCLE"
fi
rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE"
rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" "$task_args" -c "$CYCLE"

while true; do
# Update the status of the task
rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE"
rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" "$task_args" -c "$CYCLE"

# Check the task status
OUTPUT=$(rocotostat -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$TASK_NAME" -c "$CYCLE")
STATUS=$(echo "$OUTPUT" | awk '$2 == task {print $4}' task="$TASK_NAME")
num_succeeded=0
for task in "${TASK_NAMES[@]}"; do
# Check the task status
OUTPUT=$(rocotostat -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task" -c "$CYCLE")
STATUS=$(echo "$OUTPUT" | awk '$2 == task {print $4}' task="$task")

if [[ "$STATUS" == "SUCCEEDED" ]]; then
echo "The task succeeded."
if [[ "$STATUS" == "SUCCEEDED" ]]; then
echo "$task succeeded."
num_succeeded=$((num_succeeded + 1))
elif [[ "$STATUS" == "FAILED" ]]; then
echo "$task failed."
exit 1
elif [[ "$STATUS" == "DEAD" ]]; then
echo "$task is dead."
exit 1
else
echo "$task is in state: $STATUS"
fi
done
if [[ "$num_succeeded" == "$num_tasks" ]]; then
exit 0
elif [[ "$STATUS" == "FAILED" ]]; then
echo "The task failed."
exit 1
elif [[ "$STATUS" == "DEAD" ]]; then
echo "The task is dead."
exit 1
else
echo "The task is in state: $STATUS"
fi
sleep 10
done

0 comments on commit 88af2ab

Please sign in to comment.