diff --git a/tests/rt.sh b/tests/rt.sh index c55eeaaddb..51592bfd43 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -28,7 +28,6 @@ usage() { echo " -v verbose output" echo " -w for weekly_test, skip comparing baseline results" echo - set -x } [[ $# -eq 0 ]] && usage @@ -155,7 +154,6 @@ update_rtconf() { generate_log() { echo "rt.sh: Generating Regression Testing Log..." - set -x COMPILE_COUNTER=0 FAILED_COMPILES=() TEST_COUNTER=0 @@ -674,14 +672,10 @@ done #B&N not run together [[ ${NEW_BASELINES_FILE} != '' && ${RUN_SINGLE_TEST} == true ]] && die "-b and -n options cannot be used at the same time" -[[ -o xtrace ]] && set_x='set -x' || set_x='set +x' - if [[ ${RTVERBOSE} == true ]]; then set -x fi -[[ -o xtrace ]] && set_x='set -x' || set_x='set +x' - if [[ -z "${ACCNR}" ]]; then echo "Please use -a to set group account to use on HPC" exit 1 @@ -694,7 +688,6 @@ echo "Account: ${ACCNR}" case ${MACHINE_ID} in wcoss2|acorn) echo "rt.sh: Setting up WCOSS2/Acorn" - set -x if [[ "${ECFLOW:-false}" == true ]] ; then module load ecflow/5.6.0.13 fi @@ -714,7 +707,6 @@ case ${MACHINE_ID} in ;; gaea) echo "rt.sh: Setting up gaea..." - set -x if [[ "${ROCOTO:-false}" == true ]] ; then module use /ncrc/proj/epic/rocoto/modulefiles module load rocoto @@ -747,7 +739,6 @@ case ${MACHINE_ID} in ;; hera) echo "rt.sh: Setting up hera..." - set -x if [[ "${ROCOTO:-false}" == true ]] ; then module load rocoto ROCOTO_SCHEDULER=slurm @@ -770,7 +761,6 @@ case ${MACHINE_ID} in ;; orion) echo "rt.sh: Setting up orion..." - set -x module load git/2.28.0 module load gcc/10.2.0 module load python/3.9.2 @@ -800,7 +790,6 @@ case ${MACHINE_ID} in ;; hercules) echo "rt.sh: Setting up hercules..." - set -x if [[ "${ROCOTO:-false}" == true ]] ; then module load contrib rocoto ROCOTO_SCHEDULER="slurm" @@ -828,7 +817,6 @@ case ${MACHINE_ID} in ;; jet) echo "rt.sh: Setting up jet..." - set -x CurJetOS=$(lsb_release -is) echo "=======Running on ${CurJetOS}=======" if [[ ${CurJetOS} == "CentOS" ]]; then @@ -861,7 +849,6 @@ case ${MACHINE_ID} in ;; s4) echo "rt.sh: Setting up s4..." - set -x if [[ "${ROCOTO:-false}" == true ]] ; then module load rocoto/1.3.2 ROCOTO_SCHEDULER=slurm @@ -892,7 +879,6 @@ case ${MACHINE_ID} in ;; derecho) echo "rt.sh: Setting up derecho..." - set -x if [[ "${ROCOTO:-false}" == true ]] ; then module use /glade/work/epicufsrt/contrib/derecho/rocoto/modulefiles module load rocoto @@ -930,7 +916,6 @@ case ${MACHINE_ID} in ;; stampede) echo "rt.sh: Setting up stampede..." - set -x export PYTHONPATH= if [[ "${ECFLOW:-false}" == true ]] ; then ECFLOW_START= @@ -948,7 +933,6 @@ case ${MACHINE_ID} in ;; expanse) echo "rt.sh: Setting up expanse..." - set -x export PYTHONPATH= if [[ "${ECFLOW:-false}" == true ]] ; then @@ -965,7 +949,6 @@ case ${MACHINE_ID} in ;; noaacloud) echo "rt.sh: Setting up noaacloud..." - set -x export PATH="/contrib/EPIC/bin:${PATH}" module use /apps/modules/modulefiles @@ -987,7 +970,6 @@ case ${MACHINE_ID} in die "Unknown machine ID, please edit detect_machine.sh file" ;; esac -eval "${set_x}" mkdir -p "${STMP}/${USER}" @@ -1374,5 +1356,4 @@ fi ## Lets verify all tests were run and that they passed generate_log -eval "${set_x}" echo "******Regression Testing Script Completed******" diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh index 239419e271..cf047e3473 100755 --- a/tests/rt_utils.sh +++ b/tests/rt_utils.sh @@ -16,8 +16,7 @@ jobid=0 function compute_petbounds_and_tasks() { echo "rt_utils.sh: ${TEST_ID}: Computing PET bounds and tasks." - [[ -o xtrace ]] && set_x='set -x' || set_x='set +x' - set +x + # each test MUST define ${COMPONENT}_tasks variable for all components it is using # and MUST NOT define those that it's not using or set the value to 0. @@ -96,12 +95,10 @@ function compute_petbounds_and_tasks() { # TASKS is now set to UFS_TASKS export TASKS=${UFS_tasks} - eval "${set_x}" } interrupt_job() { echo "rt_utils.sh: Job ${jobid} interupted" - set -x #echo "run_util.sh: interrupt_job called | Job#: ${jobid}" case ${SCHEDULER} in pbs) @@ -120,9 +117,6 @@ submit_and_wait() { echo "rt_utils.sh: Submitting job on scheduler: ${SCHEDULER}" [[ -z $1 ]] && exit 1 - [[ -o xtrace ]] && set_x='set -x' || set_x='set +x' - set +x - local -r job_card=$1 ROCOTO=${ROCOTO:-false} @@ -156,9 +150,9 @@ submit_and_wait() { do case ${SCHEDULER} in pbs) - set +e + set +e job_info=$( qstat "${jobid}" ) - set -e + set -e ;; slurm) job_info=$( squeue -u "${USER}" -j "${jobid}" ) @@ -185,9 +179,9 @@ submit_and_wait() { do case ${SCHEDULER} in pbs) - set +e + set +e job_info=$( qstat "${jobid}" ) - set -e + set -e ;; slurm) job_info=$( squeue -u "${USER}" -j "${jobid}" ) @@ -253,14 +247,10 @@ submit_and_wait() { (( n=n+1 )) sleep 60 & wait $! done - - eval "${set_x}" } check_results() { echo "rt_utils.sh: Checking results of the regression test: ${TEST_ID}" - [[ -o xtrace ]] && set_x='set -x' || set_x='set +x' - set +x ROCOTO=${ROCOTO:-false} ECFLOW=${ECFLOW:-false} @@ -271,7 +261,7 @@ check_results() { #sleep 60 { - echo + echo echo "baseline dir = ${RTPWD}/${CNTL_DIR}_${RT_COMPILER}" echo "working dir = ${RUNDIR}" echo "Checking test ${TEST_ID} results ...." @@ -391,8 +381,6 @@ check_results() { exit 1 fi fi - - eval "${set_x}" } @@ -528,7 +516,7 @@ rocoto_kill() { rocoto_step() { echo "rt_utils.sh: Runnung one iteration of rocotorun and rocotostat..." - set -e + echo "Unknown" > rocoto_workflow.state # Run one iteration of rocotorun and rocotostat. ${ROCOTORUN} -v 10 -w "${ROCOTO_XML}" -d "${ROCOTO_DB}" @@ -566,19 +554,15 @@ rocoto_run() { set -e if [[ "${state:-Unknown}" == Done ]] ; then - set +x echo "Rocoto workflow has completed." - set -x return 0 elif [[ ${result} == 0 ]] ; then break # rocoto_step succeeded elif (( now_time-start_time > max_time || step_attempts >= max_step_attempts )) ; then - set +x hostnamein=$(hostname) echo "Rocoto commands have failed ${step_attempts} times, for $(( (now_time-start_time+30)/60 )) minutes." echo "There may be something wrong with the ${hostnamein} node or the batch system." echo "I'm giving up. Sorry." - set -x return 2 fi sleep $(( naptime * 2**((step_attempts-1)%4) * RANDOM/32767 )) @@ -625,7 +609,7 @@ EOF else echo " trigger compile_${COMPILE_ID} == complete" >> "${ECFLOW_RUN}/${ECFLOW_SUITE}.def" fi - + } ecflow_run() { @@ -633,7 +617,7 @@ ecflow_run() { # NOTE: ECFLOW IS NOT SAFE TO RUN WITH set -e, PLEASE AVOID #ECF_HOST="${ECF_HOST:-${HOSTNAME}}" - + # Make sure ECF_HOST and ECF_PORT are set/ready on systems that have an # explicit ecflow node if [[ ${MACHINE_ID} == wcoss2 || ${MACHINE_ID} == acorn ]]; then @@ -665,11 +649,11 @@ ecflow_run() { ecflow_client --ping --host="${ECF_HOST}" --port="${ECF_PORT}" not_running=$? set -e - + if [[ ${not_running} -eq 1 ]]; then echo "rt_utils.sh: ecflow_server is not running on ${ECF_HOST}:${ECF_PORT}" echo "rt_utils.sh: attempting to start ecflow_server..." - + save_traps=$(trap) trap "" SIGINT # Ignore INT signal during ecflow startup case ${MACHINE_ID} in @@ -679,7 +663,7 @@ ecflow_run() { ;; *) ${ECFLOW_START} -p "${ECF_PORT}" -d "${RUNDIR_ROOT}/ecflow_server" - ;; + ;; esac ECFLOW_RUNNING=true @@ -689,7 +673,7 @@ ecflow_run() { ecflow_client --ping --host="${ECF_HOST}" --port="${ECF_PORT}" not_running=$? set -e - + if [[ ${not_running} -eq 1 ]]; then echo "rt_utils.sh: ERROR -- Failure to start ecflow. Exiting..." exit 1 @@ -698,7 +682,7 @@ ecflow_run() { echo "rt_utils.sh: Confirmed: ecflow_server is running on ${ECF_HOST}:${ECF_PORT}" ECFLOW_RUNNING=true fi - + echo "rt_utils.sh: Starting ECFLOW tasks..." set +e ecflow_client --load="${ECFLOW_RUN}/${ECFLOW_SUITE}.def" --host="${ECF_HOST}" --port="${ECF_PORT}"