Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert IC staging job into a proper j-job #1806

Merged
merged 38 commits into from
Sep 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
29f571b
task actions for #911
AnilKumar-NOAA Aug 18, 2023
6974137
updated JGLOBAL_STAGE_IC and stage_ic.sh
AnilKumar-NOAA Aug 18, 2023
c809033
staging ic files in exglobal_stage_ic.sh
AnilKumar-NOAA Aug 18, 2023
66d3533
few changes in JGLOBAL_STAGE components and exglobal stage ic
AnilKumar-NOAA Aug 18, 2023
8904144
update with some initial arguments and error messages in exglobal_sta…
AnilKumar-NOAA Aug 18, 2023
ebc5b6a
update in exglobal_stage_ic.sh
AnilKumar-NOAA Aug 22, 2023
42bd7d8
coupled_ic is renamed to stage_ic in exglobal_stage_ic.sh and gfs_tas…
AnilKumar-NOAA Aug 22, 2023
f4ad3ae
made changes in error messages in the top section of exglobal_stage_i…
AnilKumar-NOAA Aug 22, 2023
b23d7a3
remove warning and errors in exglobal_stage_is.sh
AnilKumar-NOAA Aug 28, 2023
7abab5c
shellcheck remove
AnilKumar-NOAA Aug 28, 2023
d05f73b
shell check warning remove from stage_ic
AnilKumar-NOAA Aug 28, 2023
0c7cf07
second shell check warning remove from stage_ic
AnilKumar-NOAA Aug 28, 2023
99e2f12
small spell error corrected on first line
AnilKumar-NOAA Aug 28, 2023
cc4e8ee
under config/gfs we changed config.control_ic to config.stage_ic
AnilKumar-NOAA Aug 28, 2023
30bc7d9
revert changes (( err = err + rc )) to err=0 in script/exglobal_stage…
AnilKumar-NOAA Aug 29, 2023
0289b84
revert changes (( remove shellcheck errors in exglobal_stage_ic
AnilKumar-NOAA Aug 29, 2023
146566b
PDY issue in exglobal_stage_ic
AnilKumar-NOAA Aug 29, 2023
62383e5
PDY issue in exglobal_stage_ic
AnilKumar-NOAA Aug 29, 2023
ef67859
PDY issue in exglobal_stage_ic
AnilKumar-NOAA Aug 29, 2023
9280bf2
added shellcheck disable=SC2153 in exglobal_stage_ic.sh
AnilKumar-NOAA Aug 29, 2023
8356273
replaced coupled_ic with stage_ic in config.resources
AnilKumar-NOAA Aug 29, 2023
64d65eb
Merge branch 'NOAA-EMC:develop' into features_911
AnilKumar-NOAA Aug 29, 2023
94599f2
fixes some basic errors and issues with conda init
AnilKumar-NOAA Sep 7, 2023
81a05c6
Merge branch 'features_911' of https://github.com/AnilKumar-NOAA/glob…
AnilKumar-NOAA Sep 7, 2023
9aad75d
fixes declaration error in the exglobal_stage_ic
AnilKumar-NOAA Sep 11, 2023
91b98ff
add changes to resolve conflict issues
AnilKumar-NOAA Sep 11, 2023
9e159c2
merge develop
AnilKumar-NOAA Sep 11, 2023
fe77ad6
merge and resolve conflict files
AnilKumar-NOAA Sep 11, 2023
53bc4c6
changes to remove shellcheck
AnilKumar-NOAA Sep 11, 2023
8752b55
changes to remove shellcheck
AnilKumar-NOAA Sep 11, 2023
e2adc2c
remove shell checks from config.resources
AnilKumar-NOAA Sep 11, 2023
a5cc631
remove coupled_ic statments from config.resources
AnilKumar-NOAA Sep 11, 2023
811fa45
Update ush/load_fv3gfs_modules.sh
aerorahul Sep 11, 2023
7606404
commit sugestion in stage_ic.sh
AnilKumar-NOAA Sep 11, 2023
09ed53f
Update jobs/rocoto/stage_ic.sh
aerorahul Sep 11, 2023
ef6143e
ush/load_fv3gfs_modules.sh changes [[ NO == NO ]]
AnilKumar-NOAA Sep 11, 2023
03955d2
deleted submodule global-workflow
AnilKumar-NOAA Sep 11, 2023
09e9df6
Merge branch 'features_911' of https://github.com/AnilKumar-NOAA/glob…
AnilKumar-NOAA Sep 11, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions jobs/JGLOBAL_STAGE_IC
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#! /usr/bin/env bash

source "${HOMEgfs}/ush/preamble.sh"
source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic"


# Execute the Script
"${HOMEgfs}/scripts/exglobal_stage_ic.sh"

##########################################
# Remove the Temporary working directory
##########################################
cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"

exit 0
18 changes: 18 additions & 0 deletions jobs/rocoto/stage_ic.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#! /usr/bin/env bash

source "${HOMEgfs}/ush/preamble.sh"

# Source FV3GFS workflow modules
. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
status=$?
[[ "${status}" -ne 0 ]] && exit "${status}"

export job="stage_ic"
export jobid="${job}.$$"

# Execute the JJOB
"${HOMEgfs}/jobs/JGLOBAL_STAGE_IC"
status=$?


exit "${status}"
12 changes: 6 additions & 6 deletions parm/config/gfs/config.resources
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ if [[ $# -ne 1 ]]; then

echo "Must specify an input task argument to set resource variables!"
echo "argument can be any one of the following:"
echo "coupled_ic aerosol_init"
echo "stage_ic aerosol_init"
echo "prep preplandobs prepatmiodaobs"
echo "atmanlinit atmanlrun atmanlfinal"
echo "atmensanlinit atmensanlrun atmensanlfinal"
Expand Down Expand Up @@ -762,12 +762,12 @@ elif [[ ${step} = "arch" || ${step} = "earc" || ${step} = "getic" ]]; then
eval "export memory_${step}=50GB"
fi

elif [[ ${step} = "coupled_ic" ]]; then
elif [[ ${step} = "stage_ic" ]]; then

export wtime_coupled_ic="00:15:00"
export npe_coupled_ic=1
export npe_node_coupled_ic=1
export nth_coupled_ic=1
export wtime_stage_ic="00:15:00"
export npe_stage_ic=1
export npe_node_stage_ic=1
export nth_stage_ic=1
export is_exclusive=True

elif [[ "${step}" = "atmensanlinit" ]]; then
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
#! /usr/bin/env bash

########## config.coupled_ic ##########
########## config.stage_ic ##########

echo "BEGIN: config.coupled_ic"
echo "BEGIN: config.stage_ic"

# Get task specific resources
source "${EXPDIR}/config.resources" coupled_ic
source "${EXPDIR}/config.resources" stage_ic

case "${CASE}" in
"C48" | "C96")
Expand All @@ -32,4 +32,4 @@ case "${CASE}" in
;;
esac

echo "END: config.coupled_ic"
echo "END: config.stage_ic"
25 changes: 7 additions & 18 deletions jobs/rocoto/coupled_ic.sh → scripts/exglobal_stage_ic.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,8 @@

source "${HOMEgfs}/ush/preamble.sh"

# Source FV3GFS workflow modules
. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
status=$?
[[ ${status} -ne 0 ]] && exit ${status}

export job="coupled_ic"
export jobid="${job}.$$"

# Execute the JJOB

source "${HOMEgfs}/ush/jjob_header.sh" -e "coupled_ic" -c "base coupled_ic"

# Locally scoped variables and functions
# shellcheck disable=SC2153
GDATE=$(date -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H)
github-advanced-security[bot] marked this conversation as resolved.
Fixed
Show resolved Hide resolved
gPDY="${GDATE:0:8}"
gcyc="${GDATE:8:2}"
Expand All @@ -36,15 +25,15 @@ source="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${CDUMP}/${CASE}/INPUT/gfs_ctrl.
target="${COM_ATMOS_INPUT}/gfs_ctrl.nc"
${NCP} "${source}" "${target}"
rc=$?
[[ ${rc} -ne 0 ]] && error_message "${source}" "${target}" "${rc}"
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
for ftype in gfs_data sfc_data; do
for tt in $(seq 1 6); do
source="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${CDUMP}/${CASE}/INPUT/${ftype}.tile${tt}.nc"
target="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
${NCP} "${source}" "${target}"
rc=$?
[[ ${rc} -ne 0 ]] && error_message "${source}" "${target}" "${rc}"
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
done
done
Expand All @@ -57,7 +46,7 @@ if [[ "${DO_OCN:-}" = "YES" ]]; then
target="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res.nc"
${NCP} "${source}" "${target}"
rc=$?
[[ ${rc} -ne 0 ]] && error_message "${source}" "${target}" "${rc}"
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
case "${OCNRES}" in
"500" | "100") # Only 5 degree or 1 degree ocean does not have MOM.res_[1-4].nc files
Expand All @@ -69,7 +58,7 @@ if [[ "${DO_OCN:-}" = "YES" ]]; then
target="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res_${nn}.nc"
${NCP} "${source}" "${target}"
rc=$?
[[ ${rc} -ne 0 ]] && error_message "${source}" "${target}" "${rc}"
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
fi
done
Expand All @@ -91,7 +80,7 @@ if [[ "${DO_ICE:-}" = "YES" ]]; then
target="${COM_ICE_RESTART}/${PDY}.${cyc}0000.cice_model.res.nc"
${NCP} "${source}" "${target}"
rc=$?
[[ ${rc} -ne 0 ]] && error_message "${source}" "${target}" "${rc}"
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
fi

Expand All @@ -104,7 +93,7 @@ if [[ "${DO_WAVE:-}" = "YES" ]]; then
target="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}"
${NCP} "${source}" "${target}"
rc=$?
[[ ${rc} -ne 0 ]] && error_message "${source}" "${target}" "${rc}"
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
done
fi
Expand Down
4 changes: 2 additions & 2 deletions workflow/applications/gfs_forecast_only.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def _get_app_configs(self):
Returns the config_files that are involved in the forecast-only app
"""

configs = ['coupled_ic', 'fcst', 'arch']
configs = ['stage_ic', 'fcst', 'arch']

if self.do_atm:
configs += ['post', 'vrfy']
Expand Down Expand Up @@ -65,7 +65,7 @@ def get_task_names(self):
This is the place where that order is set.
"""

tasks = ['coupled_ic']
tasks = ['stage_ic']

if self.do_aero:
tasks += ['aerosol_init']
Expand Down
10 changes: 5 additions & 5 deletions workflow/rocoto/gfs_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ def _is_this_a_gdas_task(cdump, task_name):
raise TypeError(f'{task_name} must be part of the "enkfgdas" cycle and not {cdump}')

# Specific Tasks begin here
def coupled_ic(self):
def stage_ic(self):

cpl_ic = self._configs['coupled_ic']
cpl_ic = self._configs['stage_ic']

deps = []

Expand Down Expand Up @@ -72,8 +72,8 @@ def coupled_ic(self):

dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)

resources = self.get_resource('coupled_ic')
task = create_wf_task('coupled_ic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
resources = self.get_resource('stage_ic')
task = create_wf_task('stage_ic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)

return task

Expand Down Expand Up @@ -484,7 +484,7 @@ def fcst(self):
def _fcst_forecast_only(self):
dependencies = []

dep_dict = {'type': 'task', 'name': f'{self.cdump}coupled_ic'}
dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'}
dependencies.append(rocoto.add_dependency(dep_dict))

if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps:
Expand Down
2 changes: 1 addition & 1 deletion workflow/rocoto/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

class Tasks:
SERVICE_TASKS = ['arch', 'earc']
VALID_TASKS = ['aerosol_init', 'coupled_ic',
VALID_TASKS = ['aerosol_init', 'stage_ic',
'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch',
'prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlfinal',
'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy',
Expand Down