From cd6635c364cc9935164fb4847e7d752d8f7e059c Mon Sep 17 00:00:00 2001 From: Dan Rosen Date: Tue, 28 Mar 2023 11:30:46 -0600 Subject: [PATCH 01/87] Checkout ufs-weather-model from ESMF for fire comp --- Externals.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index b4a5840568..8adbeede1e 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -9,10 +9,10 @@ required = True [ufs-weather-model] protocol = git -repo_url = https://github.com/ufs-community/ufs-weather-model +repo_url = https://github.com/esmf-org/ufs-weather-model # Specify either a branch name or a hash but not both. -#branch = develop -hash = 788897d +branch = feature/ufs_fire +#hash = 85dc0fd local_path = sorc/ufs-weather-model required = True From 389a865e2abf70d79e7cdbb43df733e4bcd6b1d3 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 5 Sep 2023 22:17:55 -0600 Subject: [PATCH 02/87] First hooks for UFS FIRE added to workflow. Generates experiment successfully, but won't run correctly. --- devbuild.sh | 2 +- ush/config.fire.yaml | 71 ++++++++++++++++++++++++ ush/config_defaults.yaml | 107 ++++++++++++++++++++++++++++++++++++ ush/predef_grid_params.yaml | 76 +++++++++++++++++++++++-- ush/setup.py | 25 +++++++++ ush/valid_param_vals.yaml | 4 +- 6 files changed, 277 insertions(+), 8 deletions(-) create mode 100644 ush/config.fire.yaml diff --git a/devbuild.sh b/devbuild.sh index b21998fc8e..96e8245d59 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -16,7 +16,7 @@ OPTIONS (e.g. intel | gnu | cray | gccgfortran) -a, --app=APPLICATION weather model application to build; for example, ATMAQ for Online-CMAQ - (e.g. ATM | ATMAQ | ATMW | S2S | S2SW) + (e.g. ATM | ATMAQ | ATMF | ATMW | S2S | S2SW) --ccpp="CCPP_SUITE1,CCPP_SUITE2..." CCPP suites (CCPP_SUITES) to include in build; delimited with ',' --enable-options="OPTION1,OPTION2,..." diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml new file mode 100644 index 0000000000..c36093e05d --- /dev/null +++ b/ush/config.fire.yaml @@ -0,0 +1,71 @@ +metadata: + description: >- + Sample fire config +user: + RUN_ENVIR: community + MACHINE: cheyenne + ACCOUNT: NWSA0002 +platform: + DOMAIN_PREGEN_BASEDIR: "/glade/work/frediani/Projects/UFS-fire/UFS/pregen_dir" +workflow: + USE_CRON_TO_RELAUNCH: false + EXPT_SUBDIR: 2020081318_CO-HRRR_3km + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: SUBCONUS_CO_3km + DATE_FIRST_CYCL: '2020081318' + DATE_LAST_CYCL: '2020081318' + FCST_LEN_HRS: 27 + PREEXISTING_DIR_METHOD: rename + VERBOSE: true + COMPILER: intel +task_get_extrn_ics: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_ICS: "/glade/scratch/frediani/WRF-Fire/input/HRRR" + EXTRN_MDL_NAME_ICS: HRRR + FV3GFS_FILE_FMT_ICS: grib2 + EXTRN_MDL_FILES_ICS: + - '{yy}{mm}{hh}.hrrr.t{fcst_hr:02d}z.wrfprsf00.grib2' +task_get_extrn_lbcs: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_LBCS: "/glade/scratch/frediani/WRF-Fire/input/HRRR" + EXTRN_MDL_FILES_LBCS: + - '{yy}{mm}{hh}.hrrr.t{fcst_hr:02d}z.wrfprsf00.grib2' + EXTRN_MDL_NAME_LBCS: HRRR + EXTRN_MDL_LBCS_OFFSET_HRS: 0 + LBC_SPEC_INTVL_HRS: 3 + FV3GFS_FILE_FMT_LBCS: grib2 +task_run_fcst: + QUILTING: true +task_plot_allvars: + COMOUT_REF: "" +fire: + UFS_FIRE: True + FIRE_INPUT_DIR: /glade/scratch/kavulich/FIRE/esmf_fork/ufs-srweather-app/fire_data + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS1: 0.05 + FIRE_IGNITION_START_LAT1: 40.609 + FIRE_IGNITION_START_LON1: -105.879 + FIRE_IGNITION_END_LAT1: 40.609 + FIRE_IGNITION_END_LON1: -105.879 + FIRE_IGNITION_RADIUS1: 250 + FIRE_IGNITION_START_TIME1: 6480 + FIRE_IGNITION_END_TIME1: 7000 + FIRE_WIND_HEIGHT: 5.0 + FIRE_PRINT_MSG: 1 + FIRE_FUEL_LEFT_IRL: 2 + FIRE_FUEL_LEFT_JRL: 2 + FIRE_ATM_FEEDBACK: 0.0 + FIRE_GROWS_ONLY: 1 + FIRE_VISCOSITY: 0.4 + FIRE_UPWINDING: 9 + FIRE_FUEL_LEFT_METHOD: 1 + FIRE_LFN_EXT_UP: 1.0 + FIRE_LSM_ZCOUPLING: False + FIRE_LSM_ZCOUPLING_REF: 60.0 +rocoto: + tasks: + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 02:00:00 + diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 09571845a9..3c4a2d6e72 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2698,6 +2698,113 @@ cpl_aqm_parm: NEXUS_GFS_SFC_DIR: "" NEXUS_GFS_SFC_ARCHV_DIR: "/NCEPPROD/hpssprod/runhistory" +#---------------------------- +# UFS FIRE config parameters +#----------------------------- +fire: + # + #----------------------------------------------------------------------- + # + # UFS_FIRE: + # Switch for turning on fire simulation + # + # FIRE_INPUT_DIR + # Directory where fire input file (geo_em.d01.nc) can be found + # + # FIRE_NUM_IGNITIONS + # Number of fire ignitions + # + # FIRE_IGNITION_ROS1 + # + # + # FIRE_IGNITION_START_LAT1 + # + # + # FIRE_IGNITION_START_LON1 + # + # + # FIRE_IGNITION_END_LAT1 + # + # + # FIRE_IGNITION_END_LON1 + # + # + # FIRE_IGNITION_RADIUS1 + # + # + # FIRE_IGNITION_START_TIME1 + # + # + # FIRE_IGNITION_END_TIME1 + # + # + # FIRE_WIND_HEIGHT + # Height to interpolate winds to for calculating fire spread rate + # + # FIRE_PRINT_MSG + # Debug print level for the weather model fire core. Levels greater than 1 will print extra + # messages to the log file at run time. + # 0: no extra prints + # 1: Extra prints + # 2: More extra prints + # 3: Even more extra prints + # + # FIRE_FUEL_LEFT_IRL + # Refinements to integrate fuel_left, must be even + # + # FIRE_FUEL_LEFT_JRL + # Refinements to integrate fuel_left, must be even + # + # FIRE_ATM_FEEDBACK + # Multiplier for heat fluxes. Use 1.0 for normal two-way coupling. Use 0.0 for one-way coupling. + # Intermediate values will vary the amount of forcing provided from the fire to the dynamical core + # NOTE: ONLY ONE-WAY COUPLING IS CURRENTLY SUPPORTED + # + # FIRE_GROWS_ONLY + # If >0, level set function cannot increase = fire can only grow + # + # FIRE_VISCOSITY + # Artificial viscosity in level set method. Maximum 1, required for FIRE_UPWINDING=0 + # + # FIRE_UPWINDING = 9 + # 0=NONE, 1=STANDARD, 2=GODUNOV, 3=ENO, 4=SETHIAN + # + # FIRE_FUEL_LEFT_METHOD + # For now, use 1 only + # + # FIRE_LFN_EXT_UP + # 0.=extend level set function at boundary by reflection, 1.=always up + # + # FIRE_LSM_ZCOUPLING + # + # + # FIRE_LSM_ZCOUPLING_REF + # + + UFS_FIRE: True + FIRE_INPUT_DIR: "" + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS1: 0.05 + FIRE_IGNITION_START_LAT1: 40.609 + FIRE_IGNITION_START_LON1: -105.879 + FIRE_IGNITION_END_LAT1: 40.609 + FIRE_IGNITION_END_LON1: -105.879 + FIRE_IGNITION_RADIUS1: 250 + FIRE_IGNITION_START_TIME1: 6480 + FIRE_IGNITION_END_TIME1: 7000 + FIRE_WIND_HEIGHT: 5.0 + FIRE_PRINT_MSG: 0 + FIRE_FUEL_LEFT_IRL: 2 + FIRE_FUEL_LEFT_JRL: 2 + FIRE_ATM_FEEDBACK: 0.0 + FIRE_GROWS_ONLY: 1 + FIRE_VISCOSITY: 0.4 + FIRE_UPWINDING: 9 + FIRE_FUEL_LEFT_METHOD: 1 + FIRE_LFN_EXT_UP: 1.0 + FIRE_LSM_ZCOUPLING: False + FIRE_LSM_ZCOUPLING_REF: 60.0 + rocoto: attrs: "" cycledefs: "" diff --git a/ush/predef_grid_params.yaml b/ush/predef_grid_params.yaml index afcbdddf7c..3dbbcb87a5 100644 --- a/ush/predef_grid_params.yaml +++ b/ush/predef_grid_params.yaml @@ -294,10 +294,77 @@ # #----------------------------------------------------------------------- # -# The RRFS Alaska domain with ~13km cells. +# A 3km grid over Colorado, created for UFS FIRE (but can be used by any experiments) # -# Note: -# This grid has not been thoroughly tested (as of 20201027). +#----------------------------------------------------------------------- +# +"SUBCONUS_CO_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -105.6041 + ESGgrid_LAT_CTR: 39.01737 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 210 + ESGgrid_NY: 200 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 36 + LAYOUT_X: 5 + LAYOUT_Y: 5 + BLOCKSIZE: 40 + QUILTING: + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 5 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -105.6041 + WRTCMP_cen_lat: 39.01736 + WRTCMP_stdlat1: 39.01736 + WRTCMP_stdlat2: 39.01736 + WRTCMP_nx: 207 + WRTCMP_ny: 197 + WRTCMP_lon_lwr_left: -109.0989 + WRTCMP_lat_lwr_left: 36.2794 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# A 1km grid over Colorado, created for UFS FIRE (but can be used by any experiments) +# +#----------------------------------------------------------------------- +# +"SUBCONUS_CO_1km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -105.6041 + ESGgrid_LAT_CTR: 39.01737 + ESGgrid_DELX: 1000.0 + ESGgrid_DELY: 1000.0 + ESGgrid_NX: 630 + ESGgrid_NY: 600 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 6 + LAYOUT_X: 9 + LAYOUT_Y: 6 + BLOCKSIZE: 32 + QUILTING: + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 6 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -105.6041 + WRTCMP_cen_lat: 39.01736 + WRTCMP_stdlat1: 39.01736 + WRTCMP_stdlat2: 39.01736 + WRTCMP_nx: 599 + WRTCMP_ny: 570 + WRTCMP_lon_lwr_left: -109.1096 + WRTCMP_lat_lwr_left: 36.2700 + WRTCMP_dx: 1000.0 + WRTCMP_dy: 1000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS Alaska domain with ~13km cells. # #----------------------------------------------------------------------- # @@ -334,9 +401,6 @@ # # The RRFS Alaska domain with ~3km cells. # -# Note: -# This grid has not been thoroughly tested (as of 20201027). -# #----------------------------------------------------------------------- # "RRFS_AK_3km": diff --git a/ush/setup.py b/ush/setup.py index bcd04101a2..65606bfc14 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1532,6 +1532,31 @@ def dict_find(user_dict, substring): logging.debug(f'New fix file mapping:\n{fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"]=}') + # ----------------------------------------------------------------------- + # + # Check that UFS FIRE settings are correct and consistent + # + # ----------------------------------------------------------------------- + fire_conf = expt_config["fire"] + if fire_conf["UFS_FIRE"]: + fire_input_file=os.path.join(fire_conf["FIRE_INPUT_DIR"],"geo_em.d01.nc") + if not os.path.isfile(fire_input_file): + raise FileNotFoundError( + dedent( + f""" + The fire input file (geo_em.d01.nc) does not exist in the specified directory: + {fire_conf["FIRE_INPUT_DIR"]} + Check that the specified path is correct, and that the file exists and is readable + """ + ) + ) + # Only one-way coupling supported for now + if fire_conf["FIRE_ATM_FEEDBACK"] > 0.0: + raise ValueError("Two-way coupling not supported; FIRE_ATM_FEEDBACK cannot be > 0.0") + + if fire_conf["FIRE_UPWINDING"] == 0 and fire_conf["FIRE_VISCOSITY"] == 0.0: + raise ValueError("FIRE_VISCOSITY must be > 0.0 if FIRE_UPWINDING == 0") + # # ----------------------------------------------------------------------- # diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index e73f57c2f7..cb6ed345df 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -24,7 +24,9 @@ valid_vals_PREDEF_GRID_NAME: [ "RRFS_NA_13km", "RRFS_NA_3km", "SUBCONUS_Ind_3km", -"WoFS_3km" +"WoFS_3km", +"SUBCONUS_CO_3km", +"SUBCONUS_CO_1km" ] valid_vals_CCPP_PHYS_SUITE: [ "FV3_GFS_2017_gfdlmp", From 42996cb808cfc0250c73a1105fda7df80e91501a Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 18 Sep 2023 01:42:16 -0600 Subject: [PATCH 03/87] Add capability to specify YAML config file name in generate function --- .../BuildingRunningTesting/DefaultVarsTable.rst | 2 +- .../CustomizingTheWorkflow/ConfigWorkflow.rst | 4 +--- tests/WE2E/run_WE2E_tests.py | 2 +- tests/test_python/test_generate_FV3LAM_wflow.py | 2 +- ush/config_defaults.yaml | 13 +++++-------- ush/generate_FV3LAM_wflow.py | 13 ++++++++----- 6 files changed, 17 insertions(+), 19 deletions(-) diff --git a/docs/UsersGuide/source/BuildingRunningTesting/DefaultVarsTable.rst b/docs/UsersGuide/source/BuildingRunningTesting/DefaultVarsTable.rst index faaf9129c2..c22751656c 100644 --- a/docs/UsersGuide/source/BuildingRunningTesting/DefaultVarsTable.rst +++ b/docs/UsersGuide/source/BuildingRunningTesting/DefaultVarsTable.rst @@ -20,7 +20,7 @@ Table of Variables in ``config_defaults.yaml`` TEST_VX_FCST_INPUT_BASEDIR, FIXgsm, FIXaer, FIXlut, FIXorg, FIXsfc, FIXshp, FIXcrtm, FIXcrtmupp, EXTRN_MDL_DATA_STORES * - Workflow - WORKFLOW_ID, RELATIVE_LINK_FLAG, USE_CRON_TO_RELAUNCH, CRON_RELAUNCH_INTVL_MNTS, CRONTAB_LINE, LOAD_MODULES_RUN_TASK_FP, EXPT_BASEDIR, EXPT_SUBDIR, EXEC_SUBDIR, - EXPTDIR, DOT_OR_USCORE, EXPT_CONFIG_FN, CONSTANTS_FN, RGNL_GRID_NML_FN, FV3_NML_FN, FV3_NML_BASE_SUITE_FN, FV3_NML_YAML_CONFIG_FN, FV3_NML_BASE_ENS_FN, + EXPTDIR, DOT_OR_USCORE, CONSTANTS_FN, RGNL_GRID_NML_FN, FV3_NML_FN, FV3_NML_BASE_SUITE_FN, FV3_NML_YAML_CONFIG_FN, FV3_NML_BASE_ENS_FN, FV3_EXEC_FN, DATA_TABLE_FN, DIAG_TABLE_FN, FIELD_TABLE_FN, DIAG_TABLE_TMPL_FN, FIELD_TABLE_TMPL_FN, MODEL_CONFIG_FN, NEMS_CONFIG_FN, AQM_RC_FN, AQM_RC_TMPL_FN, FV3_NML_BASE_SUITE_FP, FV3_NML_YAML_CONFIG_FP, FV3_NML_BASE_ENS_FP, DATA_TABLE_TMPL_FP, DIAG_TABLE_TMPL_FP, FIELD_TABLE_TMPL_FP, MODEL_CONFIG_TMPL_FP, NEMS_CONFIG_TMPL_FP, AQM_RC_TMPL_FP, DATA_TABLE_FP, FIELD_TABLE_FP, NEMS_CONFIG_FP, FV3_NML_FP, diff --git a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst index 11d882c4f9..33b876e766 100644 --- a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -107,6 +107,7 @@ If non-default parameters are selected for the variables in this section, they s ``SCHED``: (Default: "") The job scheduler to use (e.g., Slurm) on the specified ``MACHINE``. Leaving this an empty string allows the experiment generation script to set it automatically depending on the machine the workflow is running on. Valid values: ``"slurm"`` | ``"pbspro"`` | ``"lsf"`` | ``"lsfcray"`` | ``"none"`` + Machine-Dependent Parameters ------------------------------- These parameters vary depending on machine. On `Level 1 and 2 `__ systems, the appropriate values for each machine can be viewed in the ``ush/machine/.sh`` scripts. To specify a value other than the default, add these variables and the desired value in the ``config.yaml`` file so that they override the ``config_defaults.yaml`` and machine default values. @@ -339,9 +340,6 @@ Pre-Processing File Separator Parameters Set File Name Parameters ---------------------------- -``EXPT_CONFIG_FN``: (Default: "config.yaml") - Name of the user-specified configuration file for the forecast experiment. - ``CONSTANTS_FN``: (Default: "constants.yaml") Name of the file containing definitions of various mathematical, physical, and SRW App constants. diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index 3ce6ce39a8..e987499183 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -238,7 +238,7 @@ def run_we2e_tests(homedir, args) -> None: if args.quiet: console_handler = logging.getLogger().handlers[1] console_handler.setLevel(logging.WARNING) - expt_dir = generate_FV3LAM_wflow(ushdir,logfile=f"{ushdir}/log.generate_FV3LAM_wflow", + expt_dir = generate_FV3LAM_wflow(ushdir,"config.yaml",logfile=f"{ushdir}/log.generate_FV3LAM_wflow", debug=args.debug) if args.quiet: if args.debug: diff --git a/tests/test_python/test_generate_FV3LAM_wflow.py b/tests/test_python/test_generate_FV3LAM_wflow.py index 9e9e9f5274..9ef6ee9f6c 100644 --- a/tests/test_python/test_generate_FV3LAM_wflow.py +++ b/tests/test_python/test_generate_FV3LAM_wflow.py @@ -30,7 +30,7 @@ def test_generate_FV3LAM_wflow(self): # run workflows in separate process to avoid conflict between community and nco settings def run_workflow(USHdir, logfile): - p = Process(target=generate_FV3LAM_wflow, args=(USHdir, logfile)) + p = Process(target=generate_FV3LAM_wflow, args=(USHdir,"config.yaml",logfile)) p.start() p.join() exit_code = p.exitcode diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 3c4a2d6e72..7a3e751b3a 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -339,7 +339,7 @@ platform: # that will point to a subdirectory (having the name of the grid being # used) under this directory. This variable should be set to a null # string in this file, but it can be specified in the user-specified - # workflow configuration file (EXPT_CONFIG_FN). + # workflow configuration file # #----------------------------------------------------------------------- # @@ -546,9 +546,6 @@ workflow: # # Set file names. Definitions: # - # EXPT_CONFIG_FN: - # Name of the user-specified configuration file for the forecast experiment. - # # CONSTANTS_FN: # Name of the file containing definitions of various mathematical, physical, # and SRW App constants. @@ -929,11 +926,11 @@ workflow: # they are also set to predefined values for the specified grid. # # * If PREDEF_GRID_NAME is set to an empty string, it implies the user - # will provide the native grid parameters in the user-specified - # experiment configuration file (EXPT_CONFIG_FN). In this case, the + # is providing the native grid parameters in the user-specified + # experiment configuration file. In this case, the # grid generation method GRID_GEN_METHOD, the native grid parameters, # the write-component grid parameters, the forecast model's - # main time step DT_ATMOS, and the computational + # main time step DT_ATMOS, and the computational # parameters (LAYOUT_X, LAYOUT_Y, and BLOCKSIZE) must be set in that # configuration file; otherwise, the values of all of these parameters # in this default experiment configuration file will be used. @@ -1810,7 +1807,7 @@ task_run_fcst: # # 1) If the experiment is using a predefined grid, then if the user # sets the parameter in the user-specified experiment configuration - # file (EXPT_CONFIG_FN), that value will be used in the forecast(s). + # file, that value will be used in the forecast(s). # Otherwise, the default value of the parameter for that predefined # grid will be used. # diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 516cbf94cb..07e044cde3 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -2,7 +2,7 @@ """ User interface to create an experiment directory consistent with the -user-defined config.yaml file. +user-defined YAML configuration file. """ # pylint: disable=invalid-name @@ -40,6 +40,7 @@ # pylint: disable=too-many-locals,too-many-branches, too-many-statements def generate_FV3LAM_wflow( ushdir, + config: str = "config.yaml", logfile: str = "log.generate_FV3LAM_wflow", debug: bool = False) -> str: """Function to setup a forecast experiment and create a workflow @@ -69,7 +70,7 @@ def generate_FV3LAM_wflow( # The setup function reads the user configuration file and fills in # non-user-specified values from config_defaults.yaml - expt_config = setup(ushdir,debug=debug) + expt_config = setup(ushdir,user_config_fn=config,debug=debug) # # ----------------------------------------------------------------------- @@ -91,7 +92,7 @@ def generate_FV3LAM_wflow( # Create a multiline variable that consists of a yaml-compliant string # specifying the values that the jinja variables in the template rocoto # XML should be set to. These values are set either in the user-specified - # workflow configuration file (EXPT_CONFIG_FN) or in the setup() function + # workflow configuration file (config) or in the setup() function # called above. Then call the python script that generates the XML. # # ----------------------------------------------------------------------- @@ -659,7 +660,7 @@ def generate_FV3LAM_wflow( # # ----------------------------------------------------------------------- # - cp_vrfy(os.path.join(ushdir, EXPT_CONFIG_FN), EXPTDIR) + cp_vrfy(os.path.join(ushdir, config), EXPTDIR) # # ----------------------------------------------------------------------- @@ -755,6 +756,8 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals description="Script for setting up a forecast and creating a workflow"\ "according to the parameters specified in the config file\n") + parser.add_argument('-c', '--config', default='config.yaml', + help='Name of experiment config file in YAML format') parser.add_argument('-d', '--debug', action='store_true', help='Script will be run in debug mode with more verbose output') pargs = parser.parse_args() @@ -765,7 +768,7 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals # Call the generate_FV3LAM_wflow function defined above to generate the # experiment/workflow. try: - expt_dir = generate_FV3LAM_wflow(USHdir, wflow_logfile, pargs.debug) + expt_dir = generate_FV3LAM_wflow(USHdir, pargs.config, wflow_logfile, pargs.debug) except: # pylint: disable=bare-except logging.exception( dedent( From 035282db01e1f1b6a046aca1962fee582067ae87 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 18 Sep 2023 12:24:26 -0600 Subject: [PATCH 04/87] Implement better/more concise jinja templating for nems.configure file, cleanup create_nems_configure_file.py to use arguments rather than environment variables, add fire capabilities --- parm/nems.configure | 54 +++++++------------ ush/create_nems_configure_file.py | 86 +++++++++++++++++++++---------- 2 files changed, 79 insertions(+), 61 deletions(-) diff --git a/parm/nems.configure b/parm/nems.configure index 14d9503c47..5668927e1a 100644 --- a/parm/nems.configure +++ b/parm/nems.configure @@ -3,27 +3,25 @@ ############################################# # ESMF # -{%- if print_esmf %} -logKindFlag: ESMF_LOGKIND_MULTI -{%- else %} -logKindFlag: ESMF_LOGKIND_MULTI_ON_ERROR -{%- endif %} +logKindFlag: {{ logKindFlag }} globalResourceControl: true -{% if cpl_aqm %} # EARTH # -EARTH_component_list: ATM AQM +EARTH_component_list: {{ EARTH_cl }} EARTH_attributes:: Verbosity = 0 :: # ATM # ATM_model: fv3 -ATM_petlist_bounds: -1 -1 +ATM_petlist_bounds: {{ ATM_pb }}{{ +ATM_omp_num_threads_line }} ATM_attributes:: - Verbosity = 0 + Verbosity = 0{{ +ATM_diag_line }} :: +{% if cpl_aqm %} # AQM # AQM_model: aqm AQM_petlist_bounds: -1 -1 @@ -31,35 +29,21 @@ AQM_attributes:: Verbosity = 0 Diagnostic = 0 :: - -# Run Sequence # -runSeq:: - @{{ dt_atmos }} - ATM phase1 - ATM -> AQM - AQM - AQM -> ATM - ATM phase2 - @ -:: -{% else %} -# EARTH # -EARTH_component_list: ATM -EARTH_attributes:: - Verbosity = 0 -:: - -# ATM # -ATM_model: fv3 -ATM_petlist_bounds: 0 {{ pe_member01_m1 }} -ATM_omp_num_threads: {{ atm_omp_num_threads }} -ATM_attributes:: - Verbosity = 0 +{% endif %} +{% if ufs_fire %} +# FIRE # +FIRE_model: fire_behavior +FIRE_petlist_bounds: 0 0 +FIRE_attributes:: + Verbosity = high Diagnostic = 0 :: +{% endif %} # Run Sequence # runSeq:: - ATM +{% for line in runseq -%} +{{ line }} +{%- endfor %} :: -{% endif %} + diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index 7a2ef0723f..0b60e82163 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -23,23 +23,57 @@ # These come from ush/python_utils/workflow-tools from scripts.templater import set_template -def create_nems_configure_file(run_dir): +def create_nems_configure_file(run_dir,cfg): """ Creates a nems configuration file in the specified run directory Args: run_dir: run directory + cfg: dictionary of config settings Returns: Boolean """ - print_input_args(locals()) - - #import all environment variables - import_vars() - # pylint: disable=undefined-variable + # Set necessary variables for each coupled configuration + + pe_member01_m1 = str(int(cfg["PE_MEMBER01"])-1) + + if cfg["CPL_AQM"]: + EARTH_component_list = 'ATM AQM' + ATM_petlist_bounds = '-1 -1' + ATM_omp_num_threads_line = '' + ATM_diag_line = '' + runseq = [ f" cfg['DT_ATMOS']\n", + " ATM phase1\n", + " ATM -> AQM\n", + " AQM\n", + " AQM -> ATM\n", + " ATM phase2\n", + " @" ] + elif cfg["UFS_FIRE"]: + EARTH_component_list = 'ATM FIRE' + ATM_petlist_bounds = '-1 -1' + ATM_omp_num_threads_line = '' + ATM_diag_line = '' + runseq = [ f" cfg['DT_ATMOS']\n", + " ATM -> FIRE\n", + " ATM\n", + " FIRE\n", + " @" ] + else: + EARTH_component_list = 'ATM' + ATM_petlist_bounds = f'0 {pe_member01_m1}' + ATM_omp_num_threads_line = \ + f'ATM_omp_num_threads: {cfg["OMP_NUM_THREADS_RUN_FCST"]}' + ATM_diag_line = ' Diagnostic = 0' + runseq = [ " ATM" ] + + if cfg["PRINT_ESMF"]: + logkindflag = 'ESMF_LOGKIND_MULTI' + else: + logkindflag = 'ESMF_LOGKIND_MULTI_ON_ERROR' # #----------------------------------------------------------------------- # @@ -48,14 +82,13 @@ def create_nems_configure_file(run_dir): #----------------------------------------------------------------------- # print_info_msg(f''' - Creating a nems.configure file (\"{NEMS_CONFIG_FN}\") in the specified + Creating a nems.configure file (\"{cfg["NEMS_CONFIG_FN"]}\") in the specified run directory (run_dir): - run_dir = \"{run_dir}\"''', verbose=VERBOSE) + {run_dir=}''', verbose=cfg["VERBOSE"]) # # Set output file path # - nems_config_fp = os.path.join(run_dir, NEMS_CONFIG_FN) - pe_member01_m1 = str(int(PE_MEMBER01)-1) + nems_config_fp = os.path.join(run_dir, cfg["NEMS_CONFIG_FN"]) # #----------------------------------------------------------------------- # @@ -66,33 +99,37 @@ def create_nems_configure_file(run_dir): #----------------------------------------------------------------------- # settings = { - "dt_atmos": DT_ATMOS, - "print_esmf": PRINT_ESMF, - "cpl_aqm": CPL_AQM, - "pe_member01_m1": pe_member01_m1, - "atm_omp_num_threads": OMP_NUM_THREADS_RUN_FCST, + "cpl_aqm": cfg["CPL_AQM"], + "ufs_fire": cfg["UFS_FIRE"], + "logKindFlag": logkindflag, + "EARTH_cl": EARTH_component_list, + "ATM_pb": ATM_petlist_bounds, + "ATM_omp_num_threads_line": ATM_omp_num_threads_line, + "ATM_diag_line": ATM_diag_line, + "runseq": runseq } settings_str = cfg_to_yaml_str(settings) print_info_msg( dedent( f""" - The variable \"settings\" specifying values to be used in the \"{NEMS_CONFIG_FN}\" + The variable \"settings\" specifying values to be used in the \"{cfg["NEMS_CONFIG_FN"]}\" file has been set as follows:\n settings =\n\n""" ) + settings_str, - verbose=VERBOSE, + verbose=cfg["VERBOSE"], ) # #----------------------------------------------------------------------- # - # Call a python script to generate the experiment's actual NEMS_CONFIG_FN - # file from the template file. + # Call set_template function from workflow_tools to fill in jinja template + # to create nems.configure file for this experiment # #----------------------------------------------------------------------- - # - # Store the settings in a temporary file + + # Store the settings in a temporary file; hopefully soon workflow tools will be able + # to accept a dictionary as an argument directly with tempfile.NamedTemporaryFile(dir="./", mode="w+t", prefix="nems_config_settings", @@ -100,7 +137,7 @@ def create_nems_configure_file(run_dir): tmpfile.write(settings_str) tmpfile.seek(0) - set_template(["-c", tmpfile.name, "-i", NEMS_CONFIG_TMPL_FP, "-o", nems_config_fp]) + set_template(["-c", tmpfile.name, "-i", cfg["NEMS_CONFIG_TMPL_FP"], "-o", nems_config_fp]) return True def parse_args(argv): @@ -125,7 +162,4 @@ def parse_args(argv): args = parse_args(sys.argv[1:]) cfg = load_shell_config(args.path_to_defns) cfg = flatten_dict(cfg) - import_vars(dictionary=cfg) - create_nems_configure_file( - run_dir=args.run_dir, - ) + create_nems_configure_file(run_dir=args.run_dir,cfg=cfg) From 1a5785d6b2e1b7cbb811c9ac5ef9774a896a6dbb Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 18 Sep 2023 22:37:07 -0600 Subject: [PATCH 05/87] Almost there! - Add remainder of configurable fire namelist options to config_defaults - Add fire namelist filenames and paths to config_defaults - Add LEVP (number of vertical levels) as a config variable, and explicitly set levp and npz as namelist variables. This will require some updates to documentation - Generate the majority of the UFS_FIRE namelist in the workflow generation script. The rest (start and end times) will be set in the run_fcst step to account for cycling --- ush/config.fire.yaml | 2 ++ ush/config_defaults.yaml | 27 +++++++++++++++++++--- ush/generate_FV3LAM_wflow.py | 45 ++++++++++++++++++++++++++++++++++++ ush/setup.py | 17 ++++++++++++-- 4 files changed, 86 insertions(+), 5 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index c36093e05d..5bda685281 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -41,6 +41,8 @@ task_plot_allvars: fire: UFS_FIRE: True FIRE_INPUT_DIR: /glade/scratch/kavulich/FIRE/esmf_fork/ufs-srweather-app/fire_data + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 FIRE_NUM_IGNITIONS: 1 FIRE_IGNITION_ROS1: 0.05 FIRE_IGNITION_START_LAT1: 40.609 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 7a3e751b3a..67ff81770a 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -619,9 +619,10 @@ workflow: # AQM_RC_TMPL_FN: # Template file name of resource file for NOAA Air Quality Model (AQM) # + # FIRE_NML_FN: + # Name of namelist file for UFS_FIRE capability. #----------------------------------------------------------------------- # - EXPT_CONFIG_FN: "config.yaml" CONSTANTS_FN: "constants.yaml" RGNL_GRID_NML_FN: "regional_grid.nml" @@ -641,6 +642,7 @@ workflow: NEMS_CONFIG_FN: "nems.configure" AQM_RC_FN: "aqm.rc" AQM_RC_TMPL_FN: "aqm.rc" + FIRE_NML_FN: "namelist.fire" # #----------------------------------------------------------------------- @@ -683,6 +685,7 @@ workflow: MODEL_CONFIG_TMPL_FP: '{{ [user.PARMdir, MODEL_CONFIG_FN]|path_join }}' NEMS_CONFIG_TMPL_FP: '{{ [user.PARMdir, NEMS_CONFIG_FN]|path_join }}' AQM_RC_TMPL_FP: '{{ [user.PARMdir, AQM_RC_TMPL_FN]|path_join }}' + FIRE_NML_BASE_FP: '{{ [user.PARMdir, FIRE_NML_FN]|path_join }}' # #----------------------------------------------------------------------- @@ -759,6 +762,7 @@ workflow: NEMS_CONFIG_FP: '{{ [EXPTDIR, NEMS_CONFIG_FN]|path_join }}' FV3_NML_FP: '{{ [EXPTDIR, FV3_NML_FN]|path_join }}' FV3_NML_STOCH_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_stoch"]|join ]|path_join }}' + FIRE_NML_FP: '{{ [EXPTDIR, FIRE_NML_FN]|path_join }}' FCST_MODEL: "ufs-weather-model" WFLOW_XML_FN: "FV3LAM_wflow.xml" @@ -1661,9 +1665,16 @@ task_make_ics: # FV3-LAM grid. This file will be copied later to a new location and name # changed to fvcom.nc # + # LEVP: + # Number of vertical levels in the atmosphere. In order to change this + # number, the user will additionally need to create a vertical coordinate + # distribution file; this process is described in the Users Guide section + # "Changing the Number of Vertical Levels". This value should be the same + # in both make_ics and make_lbcs. + # # VCOORD_FILE: - # Full path to the file used to set the vertical coordinate in FV3. - # This file should be the same in both make_ics and make_lbcs. + # Full path to the vertical coordinate distribution file. This file should + # be the same in both make_ics and make_lbcs. # #------------------------------------------------------------------------ # @@ -1671,6 +1682,7 @@ task_make_ics: FVCOM_WCSTART: "cold" FVCOM_DIR: "" FVCOM_FILE: "fvcom.nc" + LEVP: 65 VCOORD_FILE: "{{ workflow.FIXam }}/global_hyblev.l65.txt" #---------------------------- @@ -1696,6 +1708,7 @@ task_make_lbcs: KMP_AFFINITY_MAKE_LBCS: "scatter" OMP_NUM_THREADS_MAKE_LBCS: 1 OMP_STACKSIZE_MAKE_LBCS: "1024m" + LEVP: 65 VCOORD_FILE: "{{ workflow.FIXam }}/global_hyblev.l65.txt" #---------------------------- @@ -2708,6 +2721,12 @@ fire: # FIRE_INPUT_DIR # Directory where fire input file (geo_em.d01.nc) can be found # + # DT_FIRE + # The fire behavior component’s integration timestep + # + # OUTPUT_DT_FIRE + # The fire behavior component’s output timestep + # # FIRE_NUM_IGNITIONS # Number of fire ignitions # @@ -2780,6 +2799,8 @@ fire: UFS_FIRE: True FIRE_INPUT_DIR: "" + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 FIRE_NUM_IGNITIONS: 1 FIRE_IGNITION_ROS1: 0.05 FIRE_IGNITION_START_LAT1: 40.609 diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 07e044cde3..f9e810c2e5 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -298,6 +298,9 @@ def generate_FV3LAM_wflow( npx = NX + 1 npy = NY + 1 # + # Set npz, which is just LEVP minus 1. + npz = LEVP - 1 + # # For the physics suites that use RUC LSM, set the parameter kice to 9, # Otherwise, leave it unspecified (which means it gets set to the default # value in the forecast model). @@ -366,6 +369,8 @@ def generate_FV3LAM_wflow( "npy": npy, "layout": [LAYOUT_X, LAYOUT_Y], "bc_update_interval": LBC_SPEC_INTVL_HRS, + "levp": LEVP, + "npz": npz, }) if CCPP_PHYS_SUITE in ("FV3_GFS_2017_gfdl_mp", "FV3_GFS_2017_gfdlmp_regional", @@ -650,6 +655,46 @@ def generate_FV3LAM_wflow( if not debug: args.append("-q") set_namelist(args) + # + #----------------------------------------------------------------------- + # + # Generate UFS_FIRE namelist if needed. Most variables in the &time section + # will be updated at the run_fcst step + # + #----------------------------------------------------------------------- + # + if expt_config['fire'].get('UFS_FIRE'): + fire_nml_dict = {} + fire_nml_dict['atm'] = {} + fire_nml_dict['time'] = {} + fire_nml_dict['fire'] = {} + # Fill in &atm variables + fire_nml_dict['atm']['interval_atm'] = expt_config['task_run_fcst']['DT_ATMOS'] + fire_nml_dict['atm']['kde'] = expt_config['task_make_ics']['LEVP'] + # Fill in &fire and static &time variables + for setting in expt_config['fire']: + # Would like to use pattern matching here but don't want to force Python 3.10 + print(fire_nml_dict) + if setting == "UFS_FIRE": + pass + elif setting == "DT_FIRE": + fire_nml_dict['time']['dt'] = expt_config['fire'][setting] + elif setting == "OUTPUT_DT_FIRE": + fire_nml_dict['time']['interval_output'] = expt_config['fire'][setting] + else: + # For all other settings in config.yaml, convert to lowercase + # and enter into namelist.fire's &fire section + fire_nml_dict['fire'][setting.lower()] = expt_config['fire'][setting] + + settings_str = cfg_to_yaml_str(fire_nml_dict) + + args=[ "-n", FIRE_NML_BASE_FP, + "-u", settings_str, + "-o", FIRE_NML_FP, + ] +# if not debug: +# args.append("-q") + set_namelist(args) # # ----------------------------------------------------------------------- diff --git a/ush/setup.py b/ush/setup.py index 65606bfc14..5182562153 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -665,8 +665,7 @@ def get_location(xcs, fmt, expt_cfg): ) - # Make sure the vertical coordinate file for both make_lbcs and - # make_ics is the same. + # Make sure the vertical coordinate file and LEVP for both make_lbcs and make_ics is the same. if ics_vcoord := expt_config.get("task_make_ics", {}).get("VCOORD_FILE") != \ (lbcs_vcoord := expt_config.get("task_make_lbcs", {}).get("VCOORD_FILE")): raise ValueError( @@ -682,6 +681,20 @@ def get_location(xcs, fmt, expt_cfg): VCOORD_FILE: {lbcs_vcoord} """ ) + if ics_levp := expt_config.get("task_make_ics", {}).get("LEVP") != \ + (lbcs_levp := expt_config.get("task_make_lbcs", {}).get("LEVP")): + raise ValueError( + f""" + The number of vertical levels LEVP must be set to the same value for both the + make_ics task and the make_lbcs tasks. They are currently set to: + + make_ics: + LEVP: {ics_levp} + + make_lbcs: + LEVP: {lbcs_levp} + """ + ) # # ----------------------------------------------------------------------- From a19c6327d9d7ff8277d808a4ea694393b11fcf09 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 19 Sep 2023 01:24:22 -0600 Subject: [PATCH 06/87] Some fixes up through the run_fcst step: - Needed correct naming template for input ICS and LBCS in config.fire.yaml - Incorrectly formatted lines in create_nems_configure_file.py - Fix incorrect namelist location for levp - Forgot to ignore FIRE_INPUT_DIR when creating &fire namelist --- ush/config.fire.yaml | 7 ++++--- ush/create_nems_configure_file.py | 4 ++-- ush/generate_FV3LAM_wflow.py | 8 +++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 5bda685281..f63a003517 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -4,7 +4,7 @@ metadata: user: RUN_ENVIR: community MACHINE: cheyenne - ACCOUNT: NWSA0002 + ACCOUNT: P48503002 platform: DOMAIN_PREGEN_BASEDIR: "/glade/work/frediani/Projects/UFS-fire/UFS/pregen_dir" workflow: @@ -24,18 +24,19 @@ task_get_extrn_ics: EXTRN_MDL_NAME_ICS: HRRR FV3GFS_FILE_FMT_ICS: grib2 EXTRN_MDL_FILES_ICS: - - '{yy}{mm}{hh}.hrrr.t{fcst_hr:02d}z.wrfprsf00.grib2' + - '{yyyy}{mm}{dd}.hrrr.t{hh}z.wrfprsf00.grib2' task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true EXTRN_MDL_SOURCE_BASEDIR_LBCS: "/glade/scratch/frediani/WRF-Fire/input/HRRR" EXTRN_MDL_FILES_LBCS: - - '{yy}{mm}{hh}.hrrr.t{fcst_hr:02d}z.wrfprsf00.grib2' + - '{yyyy}{mm}{dd}.hrrr.t{hh}z.wrfprsf00.grib2' EXTRN_MDL_NAME_LBCS: HRRR EXTRN_MDL_LBCS_OFFSET_HRS: 0 LBC_SPEC_INTVL_HRS: 3 FV3GFS_FILE_FMT_LBCS: grib2 task_run_fcst: QUILTING: true + PRINT_ESMF: True task_plot_allvars: COMOUT_REF: "" fire: diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index 0b60e82163..5a8a89ea48 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -45,7 +45,7 @@ def create_nems_configure_file(run_dir,cfg): ATM_petlist_bounds = '-1 -1' ATM_omp_num_threads_line = '' ATM_diag_line = '' - runseq = [ f" cfg['DT_ATMOS']\n", + runseq = [ f" @{cfg['DT_ATMOS']}\n", " ATM phase1\n", " ATM -> AQM\n", " AQM\n", @@ -57,7 +57,7 @@ def create_nems_configure_file(run_dir,cfg): ATM_petlist_bounds = '-1 -1' ATM_omp_num_threads_line = '' ATM_diag_line = '' - runseq = [ f" cfg['DT_ATMOS']\n", + runseq = [ f" @{cfg['DT_ATMOS']}\n", " ATM -> FIRE\n", " ATM\n", " FIRE\n", diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index f9e810c2e5..028a026a0d 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -369,7 +369,6 @@ def generate_FV3LAM_wflow( "npy": npy, "layout": [LAYOUT_X, LAYOUT_Y], "bc_update_interval": LBC_SPEC_INTVL_HRS, - "levp": LEVP, "npz": npz, }) if CCPP_PHYS_SUITE in ("FV3_GFS_2017_gfdl_mp", @@ -439,6 +438,10 @@ def generate_FV3LAM_wflow( }) settings["gfs_physics_nml"] = gfs_physics_nml_dict + # Update levp in external_ic_nml; this should be the only variable that needs changing + + settings["external_ic_nml"] = {"levp": LEVP} + # # Add to "settings" the values of those namelist variables that specify # the paths to fixed files in the FIXam directory. As above, these namelist @@ -674,8 +677,7 @@ def generate_FV3LAM_wflow( # Fill in &fire and static &time variables for setting in expt_config['fire']: # Would like to use pattern matching here but don't want to force Python 3.10 - print(fire_nml_dict) - if setting == "UFS_FIRE": + if setting in ["UFS_FIRE", "FIRE_INPUT_DIR"]: pass elif setting == "DT_FIRE": fire_nml_dict['time']['dt'] = expt_config['fire'][setting] From 865c9977719b05004809ea9f8c9e169c331dabfb Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 19 Sep 2023 01:31:26 -0600 Subject: [PATCH 07/87] In the final stretch I think! Edits to run_fcst script: - Fill in &time namelist section - Link namelist.fire to namelist.input for now (until we can get the name changed) - Link in geo_em file Stuck on namelist errors for now due to old weather model hash. --- scripts/exregional_run_fcst.sh | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 6dfad9673b..c26d3c222f 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -642,6 +642,48 @@ fi # #----------------------------------------------------------------------- # +# Call the function for updating the &time section of namelist.fire +# +#----------------------------------------------------------------------- +# +FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) +settings=" +'time': { + 'start_year': '${CDATE:0:4}', + 'start_month': '${CDATE:4:2}', + 'start_day': '${CDATE:6:2}', + 'start_hour': '${CDATE:8:2}', + 'start_minute': '00', + 'start_second': '00', + 'end_year': '${FCST_END_DATE:0:4}', + 'end_month': '${FCST_END_DATE:4:2}', + 'end_day': '${FCST_END_DATE:6:2}', + 'end_hour': '${FCST_END_DATE:8:2}', + 'end_minute': '${FCST_END_DATE:10:2}', + 'end_second': '${FCST_END_DATE:12:2}', + } +" +${USHdir}/set_namelist.py -n "${FIRE_NML_FP}" -u "$settings" -o "${FIRE_NML_FN}" || \ + print_err_msg_exit "\ +Call to python script set_namelist.py to update ${FIRE_NML_FN} failed. +Parameters passed to this script are: + FIRE_NML_FN = \"${FIRE_NML_FN}\" + FIRE_NML_FP = \"${FIRE_NML_FP}\" + Namelist settings specified on command line: + settings = +$settings" + +## TEMPORARY LINK namelist.fire to namelist.input +ln -sf namelist.fire namelist.input + +# Link fire input file +ln -sf FIRE_INPUT_DIR +create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ + symlink="geo_em.d01.nc" \ + relative="FALSE" +# +#----------------------------------------------------------------------- +# # Call the function that creates the diag_table file within each cycle # directory. # From 5156f493d205540553fafc1737812680257226cf Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 21 Sep 2023 09:36:50 -0600 Subject: [PATCH 08/87] Fire capability should be False by default --- ush/config_defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 67ff81770a..bbec5639e2 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2797,7 +2797,7 @@ fire: # FIRE_LSM_ZCOUPLING_REF # - UFS_FIRE: True + UFS_FIRE: False FIRE_INPUT_DIR: "" DT_FIRE: 0.5 OUTPUT_DT_FIRE: 300 From 87040c8a0e7d9703881d531012800dc62ebdb121 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 22 Sep 2023 10:54:20 -0600 Subject: [PATCH 09/87] Forgot to commit namelist.fire parm file --- parm/namelist.fire | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 parm/namelist.fire diff --git a/parm/namelist.fire b/parm/namelist.fire new file mode 100644 index 0000000000..9f3488cae0 --- /dev/null +++ b/parm/namelist.fire @@ -0,0 +1,45 @@ + &time + start_year = 2020 + start_month = 08 + start_day = 13 + start_hour = 18 + start_minute = 00 + start_second = 00 + end_year = 2020 + end_month = 08 + end_day = 14 + end_hour = 21 + end_minute = 00 + end_second = 00 + dt = 0.5 + interval_output = 300 +/ + + &atm + interval_atm = 1 + kde = 65 + / + + &fire + fire_num_ignitions = 1, + fire_ignition_ros1 = 0.05, + fire_ignition_start_lat1 = 40.609, + fire_ignition_start_lon1 = -105.879, + fire_ignition_end_lat1 = 40.609, + fire_ignition_end_lon1 = -105.879, + fire_ignition_radius1 = 250, + fire_ignition_start_time1 = 6480, + fire_ignition_end_time1 = 7000, + fire_wind_height = 5.0, ! height to interpolate winds to for calculating fire spread rate + fire_print_msg = 1, ! 1 print fire debugging messages + fire_fuel_left_irl = 2, ! refinement to integrate fuel_left, must be even + fire_fuel_left_jrl = 2, ! refinement to integrate fuel_left, must be even + fire_atm_feedback = 1.0, ! real, multiplier for heat fluxes, 1.=normal, 0.=turn off two-way coupling + fire_grows_only = 1, ! if >0 level set function cannot increase = fire can only grow + fire_viscosity = 0.4, ! artificial viscosity in level set method (max 1, needed with fire_upwinding=0) + fire_upwinding = 9, ! 0=none, 1=standard, 2=godunov, 3=eno, 4=sethian + fire_fuel_left_method = 1, ! for now, use 1 only + fire_lfn_ext_up = 1.0, ! 0.=extend level set function at boundary by reflection, 1.=always up + fire_lsm_zcoupling = .false., + fire_lsm_zcoupling_ref = 60.0, +/ From 8b80ff599103c05e3283ff6b4b46cb618fd26b3e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 22 Sep 2023 10:55:55 -0600 Subject: [PATCH 10/87] Better error (FileNotFoundError) for set_namelist.py if the provided file does not exist. This avoids weird error handling from calling an argparse error class. --- ush/set_namelist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/set_namelist.py b/ush/set_namelist.py index e578d3201f..21da3f377c 100755 --- a/ush/set_namelist.py +++ b/ush/set_namelist.py @@ -109,7 +109,7 @@ def file_exists(arg): if not os.path.exists(arg): msg = f"{arg} does not exist!" - raise argparse.ArgumentTypeError(msg) + raise FileNotFoundError(msg) return arg From 7d58fdd1874dca10724839310162923e9da48880 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 3 Oct 2023 23:13:06 -0600 Subject: [PATCH 11/87] - Weather model has been updated, need to remove deprecated options - Force bash to output decimal numbers when creating fire namelist - Remove rogue ln command --- parm/namelist.fire | 3 --- scripts/exregional_run_fcst.sh | 27 ++++++++++++++------------- ush/config.fire.yaml | 3 --- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/parm/namelist.fire b/parm/namelist.fire index 9f3488cae0..5f3f22d3a2 100644 --- a/parm/namelist.fire +++ b/parm/namelist.fire @@ -32,13 +32,10 @@ fire_ignition_end_time1 = 7000, fire_wind_height = 5.0, ! height to interpolate winds to for calculating fire spread rate fire_print_msg = 1, ! 1 print fire debugging messages - fire_fuel_left_irl = 2, ! refinement to integrate fuel_left, must be even - fire_fuel_left_jrl = 2, ! refinement to integrate fuel_left, must be even fire_atm_feedback = 1.0, ! real, multiplier for heat fluxes, 1.=normal, 0.=turn off two-way coupling fire_grows_only = 1, ! if >0 level set function cannot increase = fire can only grow fire_viscosity = 0.4, ! artificial viscosity in level set method (max 1, needed with fire_upwinding=0) fire_upwinding = 9, ! 0=none, 1=standard, 2=godunov, 3=eno, 4=sethian - fire_fuel_left_method = 1, ! for now, use 1 only fire_lfn_ext_up = 1.0, ! 0.=extend level set function at boundary by reflection, 1.=always up fire_lsm_zcoupling = .false., fire_lsm_zcoupling_ref = 60.0, diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index c26d3c222f..ae3a2c7f8c 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -647,20 +647,22 @@ fi #----------------------------------------------------------------------- # FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) +# This horrible syntax $((10#$VARNAME)) is to force bash to treat numbers as decimal instead of +# trying to octal all up in our business settings=" 'time': { - 'start_year': '${CDATE:0:4}', - 'start_month': '${CDATE:4:2}', - 'start_day': '${CDATE:6:2}', - 'start_hour': '${CDATE:8:2}', - 'start_minute': '00', - 'start_second': '00', - 'end_year': '${FCST_END_DATE:0:4}', - 'end_month': '${FCST_END_DATE:4:2}', - 'end_day': '${FCST_END_DATE:6:2}', - 'end_hour': '${FCST_END_DATE:8:2}', - 'end_minute': '${FCST_END_DATE:10:2}', - 'end_second': '${FCST_END_DATE:12:2}', + 'start_year': $((10#${CDATE:0:4})), + 'start_month': $((10#${CDATE:4:2})), + 'start_day': $((10#${CDATE:6:2})), + 'start_hour': $((10#${CDATE:8:2})), + 'start_minute': 00, + 'start_second': 00, + 'end_year': $((10#${FCST_END_DATE:0:4})), + 'end_month': $((10#${FCST_END_DATE:4:2})), + 'end_day': $((10#${FCST_END_DATE:6:2})), + 'end_hour': $((10#${FCST_END_DATE:8:2})), + 'end_minute': $((10#${FCST_END_DATE:10:2})), + 'end_second': $((10#${FCST_END_DATE:12:2})), } " ${USHdir}/set_namelist.py -n "${FIRE_NML_FP}" -u "$settings" -o "${FIRE_NML_FN}" || \ @@ -677,7 +679,6 @@ $settings" ln -sf namelist.fire namelist.input # Link fire input file -ln -sf FIRE_INPUT_DIR create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ symlink="geo_em.d01.nc" \ relative="FALSE" diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index f63a003517..e033fbedfd 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -55,13 +55,10 @@ fire: FIRE_IGNITION_END_TIME1: 7000 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 1 - FIRE_FUEL_LEFT_IRL: 2 - FIRE_FUEL_LEFT_JRL: 2 FIRE_ATM_FEEDBACK: 0.0 FIRE_GROWS_ONLY: 1 FIRE_VISCOSITY: 0.4 FIRE_UPWINDING: 9 - FIRE_FUEL_LEFT_METHOD: 1 FIRE_LFN_EXT_UP: 1.0 FIRE_LSM_ZCOUPLING: False FIRE_LSM_ZCOUPLING_REF: 60.0 From efbf66e0607271a1eff0f93766c4f59383f91215 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 4 Oct 2023 13:12:47 -0600 Subject: [PATCH 12/87] Almost there... - Create a new variable FIRE_NUM_TASKS to choose the number of tasks to assign to the fire behavior model. - Modify PE_MEMBER01 so it accounts for FIRE_NUM_TASKS if set - Check that FIRE_NUM_TASKS is positive if running the fire model - Remove last remnants of old namelist variables - Update nems.configure to assign weather model to first petlist, then the fire tasks last --- parm/nems.configure | 2 +- ush/config.fire.yaml | 1 + ush/config_defaults.yaml | 17 ++++------------- ush/create_nems_configure_file.py | 12 +++++++++--- ush/generate_FV3LAM_wflow.py | 2 +- ush/setup.py | 3 +++ 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/parm/nems.configure b/parm/nems.configure index 5668927e1a..4dba888063 100644 --- a/parm/nems.configure +++ b/parm/nems.configure @@ -33,7 +33,7 @@ AQM_attributes:: {% if ufs_fire %} # FIRE # FIRE_model: fire_behavior -FIRE_petlist_bounds: 0 0 +FIRE_petlist_bounds: {{ FIRE_pb }} FIRE_attributes:: Verbosity = high Diagnostic = 0 diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index e033fbedfd..49b457a475 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -41,6 +41,7 @@ task_plot_allvars: COMOUT_REF: "" fire: UFS_FIRE: True + FIRE_NUM_TASKS: 1 FIRE_INPUT_DIR: /glade/scratch/kavulich/FIRE/esmf_fork/ufs-srweather-app/fire_data DT_FIRE: 0.5 OUTPUT_DT_FIRE: 300 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index bbec5639e2..d444e28fd8 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1954,7 +1954,7 @@ task_run_fcst: QUILTING: true PRINT_ESMF: false - PE_MEMBER01: '{{ LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group if QUILTING else LAYOUT_Y * LAYOUT_X}}' + PE_MEMBER01: '{{ LAYOUT_Y * LAYOUT_X + fire.FIRE_NUM_TASKS + WRTCMP_write_groups * WRTCMP_write_tasks_per_group if QUILTING else LAYOUT_Y * LAYOUT_X + fire.FIRE_NUM_TASKS }}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" @@ -2765,12 +2765,6 @@ fire: # 2: More extra prints # 3: Even more extra prints # - # FIRE_FUEL_LEFT_IRL - # Refinements to integrate fuel_left, must be even - # - # FIRE_FUEL_LEFT_JRL - # Refinements to integrate fuel_left, must be even - # # FIRE_ATM_FEEDBACK # Multiplier for heat fluxes. Use 1.0 for normal two-way coupling. Use 0.0 for one-way coupling. # Intermediate values will vary the amount of forcing provided from the fire to the dynamical core @@ -2785,9 +2779,6 @@ fire: # FIRE_UPWINDING = 9 # 0=NONE, 1=STANDARD, 2=GODUNOV, 3=ENO, 4=SETHIAN # - # FIRE_FUEL_LEFT_METHOD - # For now, use 1 only - # # FIRE_LFN_EXT_UP # 0.=extend level set function at boundary by reflection, 1.=always up # @@ -2796,6 +2787,8 @@ fire: # # FIRE_LSM_ZCOUPLING_REF # + # FIRE_NUM_TASKS + # Number of MPI tasks assigned to the FIRE_BEHAVIOR component. UFS_FIRE: False FIRE_INPUT_DIR: "" @@ -2812,16 +2805,14 @@ fire: FIRE_IGNITION_END_TIME1: 7000 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 0 - FIRE_FUEL_LEFT_IRL: 2 - FIRE_FUEL_LEFT_JRL: 2 FIRE_ATM_FEEDBACK: 0.0 FIRE_GROWS_ONLY: 1 FIRE_VISCOSITY: 0.4 FIRE_UPWINDING: 9 - FIRE_FUEL_LEFT_METHOD: 1 FIRE_LFN_EXT_UP: 1.0 FIRE_LSM_ZCOUPLING: False FIRE_LSM_ZCOUPLING_REF: 60.0 + FIRE_NUM_TASKS: 0 rocoto: attrs: "" diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index 5a8a89ea48..79acf12d76 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -38,7 +38,9 @@ def create_nems_configure_file(run_dir,cfg): # Set necessary variables for each coupled configuration - pe_member01_m1 = str(int(cfg["PE_MEMBER01"])-1) + atm_end = str(int(cfg["PE_MEMBER01"]) - int(cfg["FIRE_NUM_TASKS"]) -1) + fire_start = str(int(cfg["PE_MEMBER01"]) - int(cfg["FIRE_NUM_TASKS"])) + fire_end = str(int(cfg["PE_MEMBER01"]) - 1) if cfg["CPL_AQM"]: EARTH_component_list = 'ATM AQM' @@ -54,9 +56,10 @@ def create_nems_configure_file(run_dir,cfg): " @" ] elif cfg["UFS_FIRE"]: EARTH_component_list = 'ATM FIRE' - ATM_petlist_bounds = '-1 -1' + ATM_petlist_bounds = f'0 {atm_end}' ATM_omp_num_threads_line = '' ATM_diag_line = '' + FIRE_petlist_bounds = f'{fire_start} {fire_end}' runseq = [ f" @{cfg['DT_ATMOS']}\n", " ATM -> FIRE\n", " ATM\n", @@ -64,7 +67,7 @@ def create_nems_configure_file(run_dir,cfg): " @" ] else: EARTH_component_list = 'ATM' - ATM_petlist_bounds = f'0 {pe_member01_m1}' + ATM_petlist_bounds = f'0 {atm_end}' ATM_omp_num_threads_line = \ f'ATM_omp_num_threads: {cfg["OMP_NUM_THREADS_RUN_FCST"]}' ATM_diag_line = ' Diagnostic = 0' @@ -108,6 +111,9 @@ def create_nems_configure_file(run_dir,cfg): "ATM_diag_line": ATM_diag_line, "runseq": runseq } + if cfg["UFS_FIRE"]: + settings["FIRE_pb"] = FIRE_petlist_bounds + settings_str = cfg_to_yaml_str(settings) print_info_msg( diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 028a026a0d..890b83c7ae 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -677,7 +677,7 @@ def generate_FV3LAM_wflow( # Fill in &fire and static &time variables for setting in expt_config['fire']: # Would like to use pattern matching here but don't want to force Python 3.10 - if setting in ["UFS_FIRE", "FIRE_INPUT_DIR"]: + if setting in ["UFS_FIRE", "FIRE_INPUT_DIR", "FIRE_NUM_TASKS"]: pass elif setting == "DT_FIRE": fire_nml_dict['time']['dt'] = expt_config['fire'][setting] diff --git a/ush/setup.py b/ush/setup.py index 5182562153..a4881d0a28 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1563,6 +1563,9 @@ def dict_find(user_dict, substring): """ ) ) + if fire_conf["FIRE_NUM_TASKS"] < 1: + raise ValueError("FIRE_NUM_TASKS must be > 0") + # Only one-way coupling supported for now if fire_conf["FIRE_ATM_FEEDBACK"] > 0.0: raise ValueError("Two-way coupling not supported; FIRE_ATM_FEEDBACK cannot be > 0.0") From d4e3af011481e423194492542b65e235c027e1cc Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 6 Oct 2023 11:02:43 -0600 Subject: [PATCH 13/87] Fire run is working! Add some fixes for non-fire runs --- scripts/exregional_run_fcst.sh | 34 ++++++++++++++++--------------- ush/create_nems_configure_file.py | 5 +++-- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index ae3a2c7f8c..5e09d5c939 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -646,11 +646,12 @@ fi # #----------------------------------------------------------------------- # -FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) -# This horrible syntax $((10#$VARNAME)) is to force bash to treat numbers as decimal instead of -# trying to octal all up in our business -settings=" -'time': { +if [ "${UFS_FIRE}" = "TRUE" ]; then + FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) + # This horrible syntax $((10#$VARNAME)) is to force bash to treat numbers as decimal instead of + # trying to octal all up in our business + settings=" + 'time': { 'start_year': $((10#${CDATE:0:4})), 'start_month': $((10#${CDATE:4:2})), 'start_day': $((10#${CDATE:6:2})), @@ -663,25 +664,26 @@ settings=" 'end_hour': $((10#${FCST_END_DATE:8:2})), 'end_minute': $((10#${FCST_END_DATE:10:2})), 'end_second': $((10#${FCST_END_DATE:12:2})), - } + } " -${USHdir}/set_namelist.py -n "${FIRE_NML_FP}" -u "$settings" -o "${FIRE_NML_FN}" || \ - print_err_msg_exit "\ -Call to python script set_namelist.py to update ${FIRE_NML_FN} failed. -Parameters passed to this script are: + ${USHdir}/set_namelist.py -n "${FIRE_NML_FP}" -u "$settings" -o "${FIRE_NML_FN}" || \ + print_err_msg_exit "\ + Call to python script set_namelist.py to update ${FIRE_NML_FN} failed. + Parameters passed to this script are: FIRE_NML_FN = \"${FIRE_NML_FN}\" FIRE_NML_FP = \"${FIRE_NML_FP}\" Namelist settings specified on command line: settings = $settings" -## TEMPORARY LINK namelist.fire to namelist.input -ln -sf namelist.fire namelist.input + ## TEMPORARY LINK namelist.fire to namelist.input + ln -sf namelist.fire namelist.input -# Link fire input file -create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ - symlink="geo_em.d01.nc" \ - relative="FALSE" + # Link fire input file + create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ + symlink="geo_em.d01.nc" \ + relative="FALSE" +fi # #----------------------------------------------------------------------- # diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index 79acf12d76..4ec4b6e6f6 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -69,7 +69,7 @@ def create_nems_configure_file(run_dir,cfg): EARTH_component_list = 'ATM' ATM_petlist_bounds = f'0 {atm_end}' ATM_omp_num_threads_line = \ - f'ATM_omp_num_threads: {cfg["OMP_NUM_THREADS_RUN_FCST"]}' + f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" ATM_diag_line = ' Diagnostic = 0' runseq = [ " ATM" ] @@ -109,7 +109,8 @@ def create_nems_configure_file(run_dir,cfg): "ATM_pb": ATM_petlist_bounds, "ATM_omp_num_threads_line": ATM_omp_num_threads_line, "ATM_diag_line": ATM_diag_line, - "runseq": runseq + "runseq": runseq, + "FIRE_pb": "" } if cfg["UFS_FIRE"]: settings["FIRE_pb"] = FIRE_petlist_bounds From 901479fbebeb2d8543f5377c2e8696b2be522dfe Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 7 Oct 2023 00:49:09 -0600 Subject: [PATCH 14/87] QOL improvements for WE2E_summary file --- tests/WE2E/WE2E_summary.py | 22 ++++++++++++++++------ tests/WE2E/utils.py | 8 +++++++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/WE2E/WE2E_summary.py b/tests/WE2E/WE2E_summary.py index de478a0f38..0bd2f5bd4c 100755 --- a/tests/WE2E/WE2E_summary.py +++ b/tests/WE2E/WE2E_summary.py @@ -6,7 +6,13 @@ sys.path.append("../../ush") -from python_utils import load_config_file +try: + from python_utils import load_config_file +except ModuleNotFoundError: + print("\n\nERROR: Could not load python utilities.") + print('Note that this script can only be run in the SRW App from the directory:') + print("ufs-srweather-app/tests/WE2E\n\n") + raise from check_python_version import check_python_version @@ -63,9 +69,13 @@ def setup_logging(debug: bool = False) -> None: else: raise ValueError(f'Bad arguments; run {__file__} -h for more information') - # Calculate core hours and update yaml - expts_dict = calculate_core_hours(expts_dict) - write_monitor_file(yaml_file,expts_dict) + if expts_dict: + # Calculate core hours and update yaml + expts_dict = calculate_core_hours(expts_dict) + write_monitor_file(yaml_file,expts_dict) + + #Call function to print summary + print_WE2E_summary(expts_dict, args.debug) + else: + logging.error(f'No experiments found in provided directory {args.expt_dir}') - #Call function to print summary - print_WE2E_summary(expts_dict, args.debug) diff --git a/tests/WE2E/utils.py b/tests/WE2E/utils.py index fb96dab004..f515eea48d 100755 --- a/tests/WE2E/utils.py +++ b/tests/WE2E/utils.py @@ -53,7 +53,7 @@ def print_WE2E_summary(expts_dict: dict, debug: bool = False): expt_details.append('') expt_details.append('-'*REPORT_WIDTH) expt_details.append(f'Detailed summary of experiment {expt}') - expt_details.append(f"in directory {expts_dict[expt]['expt_dir']}") + expt_details.append(f"in directory {os.path.abspath(expts_dict[expt]['expt_dir'])}") expt_details.append(f'{" "*TASK_COLUMN_WIDTH}| Status | Walltime | Core hours used') expt_details.append('-'*REPORT_WIDTH) @@ -123,6 +123,12 @@ def create_expts_dict(expt_dir: str) -> dict: # Look for FV3LAM_wflow.xml to indicate directories with experiments in them fullpath = os.path.join(expt_dir, item) if not os.path.isdir(fullpath): + # If user is providing an experiment subdir directly, print a warning + if item == "FV3LAM_wflow.xml": + msg = "WARNING: found a rocoto XML in the provided directory!\n" + msg += "This script will only look for experiments in subdirectories\n" + msg += f"of the provided directory {expt_dir}\n" + logging.warning(msg) continue xmlfile = os.path.join(expt_dir, item, 'FV3LAM_wflow.xml') if os.path.isfile(xmlfile): From 9fa7fcc347f3d9dbe0eca86c64ee18eb41cf3d88 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 7 Oct 2023 00:49:39 -0600 Subject: [PATCH 15/87] Fix crontab bug --- ush/get_crontab_contents.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 5e9ccd6fa8..ff743eaf18 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -35,11 +35,10 @@ def get_crontab_contents(called_from_cron, machine, debug): # themselves being called as cron jobs. In that case, we must instead # call the system version of crontab at /usr/bin/crontab. # + crontab_cmd = "crontab" if machine == "CHEYENNE" or machine == "DERECHO": if called_from_cron: crontab_cmd = "/usr/bin/crontab" - else: - crontab_cmd = "crontab" print_info_msg( f""" From 48b67c6a2c6a3ab0e22beb4687ddba8dbffe479d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 7 Oct 2023 01:06:40 -0600 Subject: [PATCH 16/87] Better defaults for config.fire.yaml --- ush/config.fire.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 49b457a475..5972615682 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -4,12 +4,12 @@ metadata: user: RUN_ENVIR: community MACHINE: cheyenne - ACCOUNT: P48503002 + ACCOUNT: ACCOUNT_NUMBER platform: DOMAIN_PREGEN_BASEDIR: "/glade/work/frediani/Projects/UFS-fire/UFS/pregen_dir" workflow: - USE_CRON_TO_RELAUNCH: false - EXPT_SUBDIR: 2020081318_CO-HRRR_3km + USE_CRON_TO_RELAUNCH: true + EXPT_SUBDIR: fire_test_case CCPP_PHYS_SUITE: FV3_HRRR PREDEF_GRID_NAME: SUBCONUS_CO_3km DATE_FIRST_CYCL: '2020081318' @@ -65,7 +65,6 @@ fire: FIRE_LSM_ZCOUPLING_REF: 60.0 rocoto: tasks: - taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 02:00:00 From d34d69df6e3fdfd7e6b4502acec94ef183a67a92 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 9 Nov 2023 14:01:12 -0700 Subject: [PATCH 17/87] First attempt at two-way coupling workflow --- ush/config.fire.yaml | 2 +- ush/create_nems_configure_file.py | 3 ++- ush/generate_FV3LAM_wflow.py | 8 ++++++++ ush/setup.py | 4 ---- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 5972615682..27f0686348 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -56,7 +56,7 @@ fire: FIRE_IGNITION_END_TIME1: 7000 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 1 - FIRE_ATM_FEEDBACK: 0.0 + FIRE_ATM_FEEDBACK: 1.0 FIRE_GROWS_ONLY: 1 FIRE_VISCOSITY: 0.4 FIRE_UPWINDING: 9 diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index 4ec4b6e6f6..dfd5d9dc21 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -61,9 +61,10 @@ def create_nems_configure_file(run_dir,cfg): ATM_diag_line = '' FIRE_petlist_bounds = f'{fire_start} {fire_end}' runseq = [ f" @{cfg['DT_ATMOS']}\n", - " ATM -> FIRE\n", " ATM\n", + " ATM -> FIRE\n", " FIRE\n", + " FIRE -> ATM\n", " @" ] else: EARTH_component_list = 'ATM' diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 890b83c7ae..219e84053a 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -436,6 +436,14 @@ def generate_FV3LAM_wflow( "vsvoo1:0.0", "vsvoo2:0.0", "vsvoo3:0.0", "vsvpo1:0.0", "vsvpo2:0.0", "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] }) + + # If UFS_FIRE, activate appropriate flags + if expt_config['fire'].get('UFS_FIRE'): + gfs_physics_nml_dict.update({ + "cpl_fire": True, + "rrfs_sd": True + }) + settings["gfs_physics_nml"] = gfs_physics_nml_dict # Update levp in external_ic_nml; this should be the only variable that needs changing diff --git a/ush/setup.py b/ush/setup.py index a4881d0a28..2d56929f80 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1566,10 +1566,6 @@ def dict_find(user_dict, substring): if fire_conf["FIRE_NUM_TASKS"] < 1: raise ValueError("FIRE_NUM_TASKS must be > 0") - # Only one-way coupling supported for now - if fire_conf["FIRE_ATM_FEEDBACK"] > 0.0: - raise ValueError("Two-way coupling not supported; FIRE_ATM_FEEDBACK cannot be > 0.0") - if fire_conf["FIRE_UPWINDING"] == 0 and fire_conf["FIRE_VISCOSITY"] == 0.0: raise ValueError("FIRE_VISCOSITY must be > 0.0 if FIRE_UPWINDING == 0") From daf4e0c20e799d3131c5fb11c3cb4290357199e9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 9 Dec 2023 09:26:03 -0700 Subject: [PATCH 18/87] Update nems.configure for new FIRE settings --- ush/create_nems_configure_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index dfd5d9dc21..45000901b6 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -61,10 +61,10 @@ def create_nems_configure_file(run_dir,cfg): ATM_diag_line = '' FIRE_petlist_bounds = f'{fire_start} {fire_end}' runseq = [ f" @{cfg['DT_ATMOS']}\n", - " ATM\n", " ATM -> FIRE\n", + " FIRE -> ATM :remapmethod=conserve\n", + " ATM\n", " FIRE\n", - " FIRE -> ATM\n", " @" ] else: EARTH_component_list = 'ATM' From 6a6acbab17213b070bbecc1aef50416584bf0c7f Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 9 Dec 2023 09:27:15 -0700 Subject: [PATCH 19/87] Update field_table for fire-specific tracer fsmoke --- ush/generate_FV3LAM_wflow.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 219e84053a..a9c79852a4 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -437,12 +437,19 @@ def generate_FV3LAM_wflow( "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] }) - # If UFS_FIRE, activate appropriate flags + # If UFS_FIRE, activate appropriate flags and update FIELD_TABLE if expt_config['fire'].get('UFS_FIRE'): gfs_physics_nml_dict.update({ "cpl_fire": True, - "rrfs_sd": True }) + field_table_append = """# smoke tracer for UFS_FIRE + "TRACER", "atmos_mod", "fsmoke" + "longname", "fire smoke" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=0.0" /\n""" + + with open(FIELD_TABLE_FP, "a+") as file: + file.write(field_table_append) settings["gfs_physics_nml"] = gfs_physics_nml_dict @@ -702,8 +709,8 @@ def generate_FV3LAM_wflow( "-u", settings_str, "-o", FIRE_NML_FP, ] -# if not debug: -# args.append("-q") + if not debug: + args.append("-q") set_namelist(args) # From 624a2216721585610be84717d1c19c3f5fb0a2e9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 9 Dec 2023 09:32:03 -0700 Subject: [PATCH 20/87] Update weather model branch --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 8adbeede1e..e0777e486f 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -11,7 +11,7 @@ required = True protocol = git repo_url = https://github.com/esmf-org/ufs-weather-model # Specify either a branch name or a hash but not both. -branch = feature/ufs_fire +branch = feature/ufs_fire_cpl #hash = 85dc0fd local_path = sorc/ufs-weather-model required = True From 5574530e4badcf86b4227ec444c138a47ab7a7e4 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 12 Dec 2023 14:55:45 -0700 Subject: [PATCH 21/87] Reference config dictionary, not exported variables --- ush/generate_FV3LAM_wflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index a9c79852a4..1c0cb5de1b 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -705,9 +705,9 @@ def generate_FV3LAM_wflow( settings_str = cfg_to_yaml_str(fire_nml_dict) - args=[ "-n", FIRE_NML_BASE_FP, + args=[ "-n", expt_config['workflow']['FIRE_NML_BASE_FP'], "-u", settings_str, - "-o", FIRE_NML_FP, + "-o", expt_config['workflow']['FIRE_NML_FP'], ] if not debug: args.append("-q") From 95d2a7d9749f7cf5222c48fdb8bbd6c78bff0e8f Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 3 Jan 2024 14:28:16 -0700 Subject: [PATCH 22/87] Updates for weather model hash 75d28a14: - Remove deprecated namelist options "fire_grows_only" and "fire_lfn_ext_up" - Add functionality to create_diag_table_file.py that gives the ability to add additional entries to an existing diag_table - When UFS_FIRE is active, add line to diag_table for "fsmoke" tracer --- Externals.cfg | 2 +- parm/diag_table.FV3_HRRR | 3 +++ parm/namelist.fire | 2 -- scripts/exregional_run_fcst.sh | 3 --- ush/config_defaults.yaml | 9 --------- ush/create_diag_table_file.py | 4 +++- 6 files changed, 7 insertions(+), 16 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index e0777e486f..2adf383549 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/esmf-org/ufs-weather-model # Specify either a branch name or a hash but not both. branch = feature/ufs_fire_cpl -#hash = 85dc0fd +#hash = 75d28a14 local_path = sorc/ufs-weather-model required = True diff --git a/parm/diag_table.FV3_HRRR b/parm/diag_table.FV3_HRRR index 0712e55fca..b8198f9832 100755 --- a/parm/diag_table.FV3_HRRR +++ b/parm/diag_table.FV3_HRRR @@ -338,6 +338,9 @@ "gfs_phys", "spp_wts_rad", "spp_wts_rad", "fv3_history", "all", .false., "none", 2 "gfs_phys", "spp_wts_gwd", "spp_wts_gwd", "fv3_history", "all", .false., "none", 2 +# Additional entries from user-specified SRW settings (e.g. UFS-FIRE) +{{ additional_entries }} + #============================================================================================= # #====> This file can be used with diag_manager/v2.0a (or higher) <==== diff --git a/parm/namelist.fire b/parm/namelist.fire index 5f3f22d3a2..48f0482c6c 100644 --- a/parm/namelist.fire +++ b/parm/namelist.fire @@ -33,10 +33,8 @@ fire_wind_height = 5.0, ! height to interpolate winds to for calculating fire spread rate fire_print_msg = 1, ! 1 print fire debugging messages fire_atm_feedback = 1.0, ! real, multiplier for heat fluxes, 1.=normal, 0.=turn off two-way coupling - fire_grows_only = 1, ! if >0 level set function cannot increase = fire can only grow fire_viscosity = 0.4, ! artificial viscosity in level set method (max 1, needed with fire_upwinding=0) fire_upwinding = 9, ! 0=none, 1=standard, 2=godunov, 3=eno, 4=sethian - fire_lfn_ext_up = 1.0, ! 0.=extend level set function at boundary by reflection, 1.=always up fire_lsm_zcoupling = .false., fire_lsm_zcoupling_ref = 60.0, / diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 5e09d5c939..7fb4aefaee 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -676,9 +676,6 @@ if [ "${UFS_FIRE}" = "TRUE" ]; then settings = $settings" - ## TEMPORARY LINK namelist.fire to namelist.input - ln -sf namelist.fire namelist.input - # Link fire input file create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ symlink="geo_em.d01.nc" \ diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index d444e28fd8..e0f8986481 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2768,20 +2768,13 @@ fire: # FIRE_ATM_FEEDBACK # Multiplier for heat fluxes. Use 1.0 for normal two-way coupling. Use 0.0 for one-way coupling. # Intermediate values will vary the amount of forcing provided from the fire to the dynamical core - # NOTE: ONLY ONE-WAY COUPLING IS CURRENTLY SUPPORTED # - # FIRE_GROWS_ONLY - # If >0, level set function cannot increase = fire can only grow - # # FIRE_VISCOSITY # Artificial viscosity in level set method. Maximum 1, required for FIRE_UPWINDING=0 # # FIRE_UPWINDING = 9 # 0=NONE, 1=STANDARD, 2=GODUNOV, 3=ENO, 4=SETHIAN # - # FIRE_LFN_EXT_UP - # 0.=extend level set function at boundary by reflection, 1.=always up - # # FIRE_LSM_ZCOUPLING # # @@ -2806,10 +2799,8 @@ fire: FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 0 FIRE_ATM_FEEDBACK: 0.0 - FIRE_GROWS_ONLY: 1 FIRE_VISCOSITY: 0.4 FIRE_UPWINDING: 9 - FIRE_LFN_EXT_UP: 1.0 FIRE_LSM_ZCOUPLING: False FIRE_LSM_ZCOUPLING_REF: 60.0 FIRE_NUM_TASKS: 0 diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 3ef9ec3901..b59b5726ad 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -63,7 +63,9 @@ def create_diag_table_file(run_dir): verbose=VERBOSE, ) - settings = {"starttime": CDATE, "cres": CRES} + settings = {"starttime": CDATE, "cres": CRES, "additional_entries": ""} + if UFS_FIRE: + settings["additional_entries"] = '"gfs_phys","fsmoke","fsmoke","fv3_history","all",.false.,"none",2' settings_str = cfg_to_yaml_str(settings) print_info_msg( From 6f956b41dd50c0e3eabafac057319939db5cb588 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 3 Jan 2024 14:33:43 -0700 Subject: [PATCH 23/87] Better default values in config.fire.yaml --- ush/config.fire.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 27f0686348..1ad34fe76f 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -3,7 +3,7 @@ metadata: Sample fire config user: RUN_ENVIR: community - MACHINE: cheyenne + MACHINE: derecho ACCOUNT: ACCOUNT_NUMBER platform: DOMAIN_PREGEN_BASEDIR: "/glade/work/frediani/Projects/UFS-fire/UFS/pregen_dir" @@ -20,14 +20,14 @@ workflow: COMPILER: intel task_get_extrn_ics: USE_USER_STAGED_EXTRN_FILES: true - EXTRN_MDL_SOURCE_BASEDIR_ICS: "/glade/scratch/frediani/WRF-Fire/input/HRRR" + EXTRN_MDL_SOURCE_BASEDIR_ICS: "/glade/derecho/scratch/kavulich/FIRE/2024/test_data/HRRR" EXTRN_MDL_NAME_ICS: HRRR FV3GFS_FILE_FMT_ICS: grib2 EXTRN_MDL_FILES_ICS: - '{yyyy}{mm}{dd}.hrrr.t{hh}z.wrfprsf00.grib2' task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true - EXTRN_MDL_SOURCE_BASEDIR_LBCS: "/glade/scratch/frediani/WRF-Fire/input/HRRR" + EXTRN_MDL_SOURCE_BASEDIR_LBCS: "/glade/derecho/scratch/kavulich/FIRE/2024/test_data/HRRR" EXTRN_MDL_FILES_LBCS: - '{yyyy}{mm}{dd}.hrrr.t{hh}z.wrfprsf00.grib2' EXTRN_MDL_NAME_LBCS: HRRR @@ -42,7 +42,7 @@ task_plot_allvars: fire: UFS_FIRE: True FIRE_NUM_TASKS: 1 - FIRE_INPUT_DIR: /glade/scratch/kavulich/FIRE/esmf_fork/ufs-srweather-app/fire_data + FIRE_INPUT_DIR: /glade/derecho/scratch/kavulich/FIRE/2024/test_data DT_FIRE: 0.5 OUTPUT_DT_FIRE: 300 FIRE_NUM_IGNITIONS: 1 @@ -55,12 +55,10 @@ fire: FIRE_IGNITION_START_TIME1: 6480 FIRE_IGNITION_END_TIME1: 7000 FIRE_WIND_HEIGHT: 5.0 - FIRE_PRINT_MSG: 1 + FIRE_PRINT_MSG: 0 FIRE_ATM_FEEDBACK: 1.0 - FIRE_GROWS_ONLY: 1 FIRE_VISCOSITY: 0.4 FIRE_UPWINDING: 9 - FIRE_LFN_EXT_UP: 1.0 FIRE_LSM_ZCOUPLING: False FIRE_LSM_ZCOUPLING_REF: 60.0 rocoto: From 7f159403bcafdd313d9ff6e245d83cbe94c32fd9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 29 Mar 2024 09:16:35 -0600 Subject: [PATCH 24/87] Fix merge problems --- scripts/exregional_run_fcst.sh | 39 ++++++++++++++++++-------------- ush/create_ufs_configure_file.py | 8 +++---- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index f3da17509b..782b854bbc 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -84,7 +84,7 @@ if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} )) FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]} fi - +set -x # #----------------------------------------------------------------------- # @@ -661,35 +661,40 @@ fi # #----------------------------------------------------------------------- # +cp ${FIRE_NML_FP} fire.nml if [ "${UFS_FIRE}" = "TRUE" ]; then FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) # This horrible syntax $((10#$VARNAME)) is to force bash to treat numbers as decimal instead of # trying to octal all up in our business settings=" - 'time': { - 'start_year': $((10#${CDATE:0:4})), - 'start_month': $((10#${CDATE:4:2})), - 'start_day': $((10#${CDATE:6:2})), - 'start_hour': $((10#${CDATE:8:2})), - 'start_minute': 00, - 'start_second': 00, - 'end_year': $((10#${FCST_END_DATE:0:4})), - 'end_month': $((10#${FCST_END_DATE:4:2})), - 'end_day': $((10#${FCST_END_DATE:6:2})), - 'end_hour': $((10#${FCST_END_DATE:8:2})), - 'end_minute': $((10#${FCST_END_DATE:10:2})), - 'end_second': $((10#${FCST_END_DATE:12:2})), - } + &time + start_year = $((10#${CDATE:0:4})), + start_month = $((10#${CDATE:4:2})), + start_day = $((10#${CDATE:6:2})), + start_hour = $((10#${CDATE:8:2})), + start_minute = 00, + start_second = 00, + end_year = $((10#${FCST_END_DATE:0:4})), + end_month = $((10#${FCST_END_DATE:4:2})), + end_day = $((10#${FCST_END_DATE:6:2})), + end_hour = $((10#${FCST_END_DATE:8:2})), + end_minute = $((10#${FCST_END_DATE:10:2})), + end_second = $((10#${FCST_END_DATE:12:2})), + / " - ${USHdir}/set_namelist.py -n "${FIRE_NML_FP}" -u "$settings" -o "${FIRE_NML_FN}" || \ + + echo "$settings" | uw config realize --input-format nml --output-format nml -o "${FIRE_NML_FN}" fire.nml + err=$? + if [ $err -ne 0 ]; then print_err_msg_exit "\ - Call to python script set_namelist.py to update ${FIRE_NML_FN} failed. + Call to uw config realize to update ${FIRE_NML_FN} failed. Parameters passed to this script are: FIRE_NML_FN = \"${FIRE_NML_FN}\" FIRE_NML_FP = \"${FIRE_NML_FP}\" Namelist settings specified on command line: settings = $settings" + fi # Link fire input file create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index a071b48f8f..f74dc26a50 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -109,10 +109,10 @@ def create_ufs_configure_file(run_dir,cfg): "ATM_omp_num_threads_line": ATM_omp_num_threads_line, "ATM_diag_line": ATM_diag_line, "runseq": runseq, - "FIRE_pb": "" - "dt_atmos": DT_ATMOS, - "print_esmf": PRINT_ESMF, - "cpl_aqm": CPL_AQM + "FIRE_pb": "", + "dt_atmos": cfg["DT_ATMOS"], + "print_esmf": cfg["PRINT_ESMF"], + "cpl_aqm": cfg["CPL_AQM"] } if cfg["UFS_FIRE"]: settings["FIRE_pb"] = FIRE_petlist_bounds From 0da4606dbeac8c83824b0e6b63cde1c83fccf652 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 19 Apr 2024 10:57:06 -0600 Subject: [PATCH 25/87] Fix bug in &time namelist section setup: didn't move temporary file back where it should be! --- scripts/exregional_run_fcst.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 782b854bbc..682d6367b0 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -661,8 +661,9 @@ fi # #----------------------------------------------------------------------- # -cp ${FIRE_NML_FP} fire.nml if [ "${UFS_FIRE}" = "TRUE" ]; then + # Due to limitation in uwtools, can only manipulate namelists that end in ".nml" for now, so copy to temp name + cp ${FIRE_NML_FP} fire.nml FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) # This horrible syntax $((10#$VARNAME)) is to force bash to treat numbers as decimal instead of # trying to octal all up in our business @@ -696,6 +697,8 @@ if [ "${UFS_FIRE}" = "TRUE" ]; then $settings" fi + # Move updated namelist back to original name. + mv fire.nml ${FIRE_NML_FP} # Link fire input file create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ symlink="geo_em.d01.nc" \ From 7585150eb386b34b35867cd0f6298fbcd98b7d68 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 22 Apr 2024 11:03:21 -0600 Subject: [PATCH 26/87] *ACTUALLY* fix bug in &time namelist section setup --- scripts/exregional_run_fcst.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 682d6367b0..64273deaa6 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -662,8 +662,6 @@ fi #----------------------------------------------------------------------- # if [ "${UFS_FIRE}" = "TRUE" ]; then - # Due to limitation in uwtools, can only manipulate namelists that end in ".nml" for now, so copy to temp name - cp ${FIRE_NML_FP} fire.nml FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) # This horrible syntax $((10#$VARNAME)) is to force bash to treat numbers as decimal instead of # trying to octal all up in our business @@ -683,8 +681,10 @@ if [ "${UFS_FIRE}" = "TRUE" ]; then end_second = $((10#${FCST_END_DATE:12:2})), / " + # Hopefully future updates to uwtools will eliminate the need for this "temporary file" manipulation + echo "$settings" > fire.update.nml - echo "$settings" | uw config realize --input-format nml --output-format nml -o "${FIRE_NML_FN}" fire.nml + uw config realize --input-format nml --output-format nml -i "${FIRE_NML_FN}" -o "${FIRE_NML_FN}" fire.update.nml err=$? if [ $err -ne 0 ]; then print_err_msg_exit "\ @@ -697,8 +697,6 @@ if [ "${UFS_FIRE}" = "TRUE" ]; then $settings" fi - # Move updated namelist back to original name. - mv fire.nml ${FIRE_NML_FP} # Link fire input file create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ symlink="geo_em.d01.nc" \ From 7ba273464af6180b2e87a4bd1828cbd8a75e7ad2 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 29 Aug 2024 11:13:41 -0600 Subject: [PATCH 27/87] Fix merge issues --- scripts/exregional_run_fcst.sh | 2 +- ush/create_ufs_configure_file.py | 3 +-- ush/generate_FV3LAM_wflow.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 274ff5c604..975de487b4 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -117,7 +117,7 @@ # . $USHdir/source_util_funcs.sh for sect in user nco platform workflow global cpl_aqm_parm constants fixed_files \ - task_get_extrn_lbcs task_run_fcst task_run_post ; do + task_get_extrn_lbcs task_run_fcst task_run_post fire; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index 902e5b7b53..0a4e58340b 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -124,8 +124,7 @@ def create_ufs_configure_file(run_dir,cfg): The variable \"settings\" specifying values to be used in the \"{cfg["UFS_CONFIG_FN"]}\" file has been set as follows:\n {settings=}\n\n""" - ) - verbose=cfg["VERBOSE"], + ), verbose=cfg["VERBOSE"] ) # #----------------------------------------------------------------------- diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index d986a4c844..e87bb85f0c 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -716,7 +716,7 @@ def generate_FV3LAM_wflow( input_format="nml", output_file=expt_config['workflow']['FIRE_NML_FP'], output_format="nml", - supplemental_configs=[fire_nml_dict], + update_config=get_nml_config(fire_nml_dict), ) # From 14a0f893a4ae63e10cb5b391f1c9b952995b51e7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 29 Aug 2024 11:14:50 -0600 Subject: [PATCH 28/87] Cutting the fat out of base utilities boolify and create_symlink_to_file. This gives ~5x speedup for create_symlink_to_file on derecho --- ush/bash_utils/boolify.sh | 18 ------------------ ush/bash_utils/create_symlink_to_file.sh | 15 ++++----------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/ush/bash_utils/boolify.sh b/ush/bash_utils/boolify.sh index 577b7c16eb..d6857e18e0 100644 --- a/ush/bash_utils/boolify.sh +++ b/ush/bash_utils/boolify.sh @@ -11,15 +11,6 @@ function boolify() { # #----------------------------------------------------------------------- # -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# # Get the name of this function and input. # #----------------------------------------------------------------------- @@ -57,13 +48,4 @@ where: echo "FALSE" fi - # - #----------------------------------------------------------------------- - # - # Restore the shell options saved at the beginning of this script/func- - # tion. - # - #----------------------------------------------------------------------- - # - { restore_shell_opts; } > /dev/null 2>&1 } diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh index 0cfcdc9fdf..c8e3181ae2 100644 --- a/ush/bash_utils/create_symlink_to_file.sh +++ b/ush/bash_utils/create_symlink_to_file.sh @@ -31,6 +31,10 @@ target=$1 symlink=$2 relative=${3:-TRUE} relative=$(boolify $relative) +if [ "$relative" != "TRUE" ] && [ "$relative" != "FALSE" ]; then + usage + print_err_msg_exit "'relative' must be set to TRUE or FALSE" +fi # #----------------------------------------------------------------------- # @@ -43,17 +47,6 @@ relative=$(boolify $relative) # #----------------------------------------------------------------------- # -# If "relative" is not set (i.e. if it is set to a null string), reset -# it to a default value of "TRUE". Then check that it is set to a vaild -# value. -# -#----------------------------------------------------------------------- -# - valid_vals_relative=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") - check_var_valid_value "relative" "valid_vals_relative" -# -#----------------------------------------------------------------------- -# # Make sure that the target file exists and is a file. # #----------------------------------------------------------------------- From c0ddb1a8ca0a9a2c42caa0bda879e1561d629363 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 29 Aug 2024 15:48:00 -0600 Subject: [PATCH 29/87] More merge corrections - No more usage function in create_symlink_to_file() - Updates for new uwtools functionality --- scripts/exregional_run_fcst.sh | 13 +++++-------- ush/bash_utils/create_symlink_to_file.sh | 2 -- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 975de487b4..4f5679494c 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -760,7 +760,7 @@ fi # #----------------------------------------------------------------------- # -if [ "${UFS_FIRE}" = "TRUE" ]; then +if [ $(boolify "${UFS_FIRE}") = "TRUE" ]; then FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) # This horrible syntax $((10#$VARNAME)) is to force bash to treat numbers as decimal instead of # trying to octal all up in our business @@ -781,13 +781,13 @@ if [ "${UFS_FIRE}" = "TRUE" ]; then / " # Hopefully future updates to uwtools will eliminate the need for this "temporary file" manipulation - echo "$settings" > fire.update.nml +# echo "$settings" > fire.update.nml - uw config realize --input-format nml --output-format nml -i "${FIRE_NML_FN}" -o "${FIRE_NML_FN}" fire.update.nml + echo $settings | uw config realize --input-format nml --output-format nml --input-file "${FIRE_NML_FP}" -o "${FIRE_NML_FN}" err=$? if [ $err -ne 0 ]; then print_err_msg_exit "\ - Call to uw config realize to update ${FIRE_NML_FN} failed. + Call to uw config realize to create ${FIRE_NML_FN} failed. Parameters passed to this script are: FIRE_NML_FN = \"${FIRE_NML_FN}\" FIRE_NML_FP = \"${FIRE_NML_FP}\" @@ -795,11 +795,8 @@ if [ "${UFS_FIRE}" = "TRUE" ]; then settings = $settings" fi - # Link fire input file - create_symlink_to_file target="${FIRE_INPUT_DIR}/geo_em.d01.nc" \ - symlink="geo_em.d01.nc" \ - relative="FALSE" + create_symlink_to_file ${FIRE_INPUT_DIR}/geo_em.d01.nc geo_em.d01.nc FALSE fi # #----------------------------------------------------------------------- diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh index c8e3181ae2..3f82dfd1fe 100644 --- a/ush/bash_utils/create_symlink_to_file.sh +++ b/ush/bash_utils/create_symlink_to_file.sh @@ -23,7 +23,6 @@ function create_symlink_to_file() { #----------------------------------------------------------------------- # if [[ $# -lt 2 ]]; then - usage print_err_msg_exit "Function create_symlink_to_file() requires at least two arguments" fi @@ -32,7 +31,6 @@ symlink=$2 relative=${3:-TRUE} relative=$(boolify $relative) if [ "$relative" != "TRUE" ] && [ "$relative" != "FALSE" ]; then - usage print_err_msg_exit "'relative' must be set to TRUE or FALSE" fi # From e508aa904c0b6c26ce0aef7af5ac719b5946b136 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 30 Aug 2024 11:44:44 -0600 Subject: [PATCH 30/87] Fix problem with uw namelist update --- scripts/exregional_run_fcst.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 4f5679494c..d13369f195 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -780,10 +780,8 @@ if [ $(boolify "${UFS_FIRE}") = "TRUE" ]; then end_second = $((10#${FCST_END_DATE:12:2})), / " - # Hopefully future updates to uwtools will eliminate the need for this "temporary file" manipulation -# echo "$settings" > fire.update.nml - echo $settings | uw config realize --input-format nml --output-format nml --input-file "${FIRE_NML_FP}" -o "${FIRE_NML_FN}" + echo $settings | uw config realize --update-format nml --input-format nml --output-format nml --input-file "${FIRE_NML_FP}" -o "${FIRE_NML_FN}" err=$? if [ $err -ne 0 ]; then print_err_msg_exit "\ From 904b1da71f5de8ad0f14c2b72300e7c98ed4284c Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 2 Sep 2024 09:34:02 -0600 Subject: [PATCH 31/87] Add ability in retrieve_data.py to use forecast hour templates as well (e.g. {fyyyy}, {fhh}, {fyyyymmdd}, etc) to supplement the current templates based on cycle date --- ush/retrieve_data.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 5acf9d5ce9..7797ed5561 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -206,6 +206,8 @@ def fill_template(template_str, cycle_date, templates_only=False, **kwargs): mem = kwargs.get("mem", "") # ----- + # Set some variables to make the format statement more concise + f_date = cycle_date + dt.timedelta(hours=fcst_hr) cycle_hour = cycle_date.strftime("%H") # One strategy for binning data files at NCEP is to put them into 6 @@ -225,17 +227,26 @@ def fill_template(template_str, cycle_date, templates_only=False, **kwargs): ens_group=ens_group, fcst_hr=fcst_hr, dd=cycle_date.strftime("%d"), + fdd=f_date.strftime("%d"), hh=cycle_hour, + fhh=f_date.strftime("%H"), hh_even=hh_even, jjj=cycle_date.strftime("%j"), + fjjj=f_date.strftime("%j"), mem=mem, min=cycle_date.strftime("%M"), mm=cycle_date.strftime("%m"), + fmm=cycle_date.strftime("%m"), yy=cycle_date.strftime("%y"), + fyy=f_date.strftime("%y"), yyyy=cycle_date.strftime("%Y"), + fyyyy=f_date.strftime("%Y"), yyyymm=cycle_date.strftime("%Y%m"), + fyyyymm=f_date.strftime("%Y%m"), yyyymmdd=cycle_date.strftime("%Y%m%d"), + fyyyymmdd=f_date.strftime("%Y%m%d"), yyyymmddhh=cycle_date.strftime("%Y%m%d%H"), + fyyyymmddhh=f_date.strftime("%Y%m%d%H"), ) if templates_only: From b66988d897f7627c0b8810a352f6e236dca94f08 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 3 Sep 2024 11:41:54 -0600 Subject: [PATCH 32/87] Fix fire default filenames --- ush/config.fire.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 1ad34fe76f..069b31566b 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -24,12 +24,12 @@ task_get_extrn_ics: EXTRN_MDL_NAME_ICS: HRRR FV3GFS_FILE_FMT_ICS: grib2 EXTRN_MDL_FILES_ICS: - - '{yyyy}{mm}{dd}.hrrr.t{hh}z.wrfprsf00.grib2' + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true EXTRN_MDL_SOURCE_BASEDIR_LBCS: "/glade/derecho/scratch/kavulich/FIRE/2024/test_data/HRRR" EXTRN_MDL_FILES_LBCS: - - '{yyyy}{mm}{dd}.hrrr.t{hh}z.wrfprsf00.grib2' + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' EXTRN_MDL_NAME_LBCS: HRRR EXTRN_MDL_LBCS_OFFSET_HRS: 0 LBC_SPEC_INTVL_HRS: 3 From c740fe206ade6ff05ede1ac57fa7960b654256bd Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 12 Sep 2024 10:45:21 -0600 Subject: [PATCH 33/87] Remove mentions of Cheyenne from comments --- ush/get_crontab_contents.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 44aff3db24..fc3a06e3ff 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -15,8 +15,8 @@ def get_crontab_contents(called_from_cron, machine, debug): """ This function returns the contents of the user's cron table, as well as the command used to manipulate the cron table. Typically this latter value will be `crontab`, but on some - platforms the version or location of this may change depending on other circumstances, e.g. on - Cheyenne, this depends on whether a script that wants to call `crontab` is itself being called + platforms the version or location of this may change depending on other circumstances; + this depends on whether a script that wants to call `crontab` is itself being called from a cron job. Args: @@ -28,13 +28,6 @@ def get_crontab_contents(called_from_cron, machine, debug): crontab_contents (str) : String containing the contents of the user's cron table. """ - # - # On Cheyenne, simply typing "crontab" will launch the crontab command - # at "/glade/u/apps/ch/opt/usr/bin/crontab". This is a containerized - # version of crontab that will not work if called from scripts that are - # themselves being called as cron jobs. In that case, we must instead - # call the system version of crontab at /usr/bin/crontab. - # crontab_cmd = "crontab" print_info_msg( From 11fdd1c22dadd06e475e76c8316446d5060525c7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 27 Sep 2024 09:33:55 -0600 Subject: [PATCH 34/87] Update WM to point to main repo and newer hash; this requires an update to build library versions --- Externals.cfg | 6 +++--- modulefiles/build_derecho_intel.lua | 2 +- modulefiles/build_gaea_intel.lua | 2 +- modulefiles/build_hera_gnu.lua | 2 +- modulefiles/build_hera_intel.lua | 2 +- modulefiles/build_hercules_intel.lua | 2 +- modulefiles/build_jet_intel.lua | 2 +- modulefiles/build_orion_intel.lua | 2 +- modulefiles/srw_common.lua | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 408fe7eb16..1b688ca91a 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -9,10 +9,10 @@ required = True [ufs-weather-model] protocol = git -repo_url = https://github.com/esmf-org/ufs-weather-model +repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. -branch = feature/ufs_fire_cpl -#hash = 75d28a14 +#branch = develop +hash = 25ee7f6 local_path = sorc/ufs-weather-model required = True diff --git a/modulefiles/build_derecho_intel.lua b/modulefiles/build_derecho_intel.lua index 491a94f912..1356fdb3e0 100644 --- a/modulefiles/build_derecho_intel.lua +++ b/modulefiles/build_derecho_intel.lua @@ -6,7 +6,7 @@ the CISL machine Derecho (Cray) using Intel@2021.10.0 whatis([===[Loads libraries needed for building the UFS SRW App on Derecho ]===]) prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.10.0")) load(pathJoin("stack-cray-mpich", os.getenv("stack_cray_mpich_ver") or "8.1.25")) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index b47209194c..f54a8fcf54 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -5,7 +5,7 @@ the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0 whatis([===[Loads libraries needed for building the UFS SRW App on Gaea C5 ]===]) -prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index 8854108966..621c7581a6 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera using GN prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/modulefiles") prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/openmpi/modulefiles") -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/ufs-wm-srw-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/upp-addon-env/install/modulefiles/Core") load("stack-gcc/13.3.0") load("stack-openmpi/4.1.6") diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index d8e793044c..d3d20a5bb5 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===]) prepend_path("MODULEPATH","/contrib/sutils/modulefiles") load("sutils") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index b65890f1c4..e01b68905e 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -5,7 +5,7 @@ the MSU machine Hercules using intel-oneapi-compilers/2022.2.1 whatis([===[Loads libraries needed for building the UFS SRW App on Hercules ]===]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.9.0") load("stack-intel-oneapi-mpi/2021.9.0") diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index a0169a684a..78e70e0960 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -5,7 +5,7 @@ the NOAA RDHPC machine Jet using Intel-2021.5.0 whatis([===[Loads libraries needed for building the UFS SRW App on Jet ]===]) -prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.5.0") load("stack-intel-oneapi-mpi/2021.5.1") diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index b2f3d85c00..92d268a4d7 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -5,7 +5,7 @@ the MSU machine Orion using intel-oneapi-compilers/2021.9.0 whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.9.0") load("stack-intel-oneapi-mpi/2021.9.0") diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index cb2047cbe1..d2bdbe6790 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -10,8 +10,8 @@ load("fms/2023.04") load("bacio/2.4.1") load("crtm/2.4.0.1") -load("g2/3.4.5") -load("g2tmpl/1.10.2") +load("g2/3.5.1") +load("g2tmpl/1.13.0") load("ip/4.3.0") load("sp/2.5.0") load("w3emc/2.10.0") From 71b9653f0ce331761967823f45fd97d13bd752b6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 27 Sep 2024 13:28:49 -0600 Subject: [PATCH 35/87] Need new version of UPP as well to correspond with updated WM hash --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 1b688ca91a..aec50632b9 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -21,7 +21,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = be0410e +hash = 81b38a8 local_path = sorc/UPP required = True From 9a48261437ff14dabd7e5f9b5c6ab412c93f0c51 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 27 Sep 2024 15:31:46 -0600 Subject: [PATCH 36/87] Okay, about time we put these build-time settings to use.... - Rename ufs_srweather_app.settings --> build_settings.yaml, make yaml format for easy reading - Add "Application" and "Git hash" to build_settings.yaml - Read build_settings.yaml in setup.py to determine if App was built with fire capabilities --- CMakeLists.txt | 28 +++++++++++++++++++--------- etc/build_settings_template.yaml | 22 ++++++++++++++++++++++ ufs_srweather_app.settings.in | 21 --------------------- ush/setup.py | 7 +++++++ 4 files changed, 48 insertions(+), 30 deletions(-) create mode 100644 etc/build_settings_template.yaml delete mode 100644 ufs_srweather_app.settings.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 30e241daba..44c0667c12 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,12 +8,20 @@ project(ufs-srweather-app VERSION 1.0 LANGUAGES C CXX Fortran) find_package(MPI REQUIRED COMPONENTS C CXX Fortran) # Set extended version info. -SET(SRWA_VERSION_MAJOR 1) -SET(SRWA_VERSION_MINOR 0) +SET(SRWA_VERSION_MAJOR 2) +SET(SRWA_VERSION_MINOR 2) SET(SRWA_VERSION_PATCH 0) SET(SRWA_VERSION_NOTE "-development") SET(SRWA_VERSION ${SRWA_VERSION_MAJOR}.${SRWA_VERSION_MINOR}.${SRWA_VERSION_PATCH}${SRWA_VERSION_NOTE}) +# Get the latest abbreviated commit hash of the working branch +execute_process( + COMMAND git log -1 --format=%h + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + OUTPUT_VARIABLE GIT_HASH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + # A function used to create autotools-style 'yes/no' definitions. # If a variable is set, it 'yes' is returned. Otherwise, 'no' is # returned. @@ -55,7 +63,7 @@ if(NOT DEFINED CMAKE_INSTALL_BINDIR) endif() ##### -# Configure and print the ufs-srweather-app.settings file. +# Configure and print the build settings file. ##### # Determine the configure date. @@ -92,10 +100,12 @@ SET(host_vendor "${osname}") SET(host_os "${osrel}") SET(abs_top_builddir "${CMAKE_CURRENT_BINARY_DIR}") SET(abs_top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}") +SET(application "${APP}") + SET(CC_VERSION "${CMAKE_C_COMPILER}") -# Set values for .settings file. +# Set values for build settings file. SET(CFLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}}") SET(CPPFLAGS "${CMAKE_CPP_FLAGS} ${CMAKE_CPP_FLAGS_${CMAKE_BUILD_TYPE}}") SET(LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS_${CMAKE_BUILD_TYPE}}") @@ -107,20 +117,20 @@ is_enabled(BUILD_SHARED_LIBS enable_shared) is_enabled(STATUS_PARALLEL HAS_PARALLEL) # Generate file from template. -CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/ufs_srweather_app.settings.in" - "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app.settings" +CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/etc/build_settings_template.yaml" + "${CMAKE_CURRENT_BINARY_DIR}/build_settings.yaml" @ONLY) # Read in settings file, print out. # Avoid using system-specific calls so that this # might also work on Windows. -FILE(READ "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app.settings" +FILE(READ "${CMAKE_CURRENT_BINARY_DIR}/build_settings.yaml" UFS-SRWEATHER-APP_SETTINGS) MESSAGE(${UFS-SRWEATHER-APP_SETTINGS}) -# Install ufs_srweather_app.settings file into same location +# Install build settings file into same location # as the app. -INSTALL(FILES "${CMAKE_BINARY_DIR}/ufs_srweather_app.settings" +INSTALL(FILES "${CMAKE_BINARY_DIR}/build_settings.yaml" DESTINATION ${CMAKE_INSTALL_BINDIR}) ##### diff --git a/etc/build_settings_template.yaml b/etc/build_settings_template.yaml new file mode 100644 index 0000000000..785257e154 --- /dev/null +++ b/etc/build_settings_template.yaml @@ -0,0 +1,22 @@ +# UFS Short Range Weather App Configuration Summary +#=================================================== + +# General +#--------- +SRW_Version: @SRWA_VERSION@ +SRW_hash: @GIT_HASH@ +Configured_On: @CONFIG_DATE@ +Host_System: @host_cpu@-@host_vendor@-@host_os@ +Build_Directory: @abs_top_builddir@ +Install_Prefix: @prefix@ +Application: @application@ +# Compiling Options +#------------------- +C_Compiler: @CC_VERSION@ +CFLAGS: @CFLAGS@ +CPPFLAGS: @CPPFLAGS@ +LDFLAGS: @LDFLAGS@ +Shared_Library: @enable_shared@ +Static_Library: @enable_static@ +Extra_libraries: @LIBS@ + diff --git a/ufs_srweather_app.settings.in b/ufs_srweather_app.settings.in deleted file mode 100644 index 7958d2706f..0000000000 --- a/ufs_srweather_app.settings.in +++ /dev/null @@ -1,21 +0,0 @@ -# UFS Short Range Weather App Configuration Summary -=================================================== - -# General ---------- -SRWA Version: @SRWA_VERSION@ -Configured On: @CONFIG_DATE@ -Host System: @host_cpu@-@host_vendor@-@host_os@ -Build Directory: @abs_top_builddir@ -Install Prefix: @prefix@ - -# Compiling Options -------------------- -C Compiler: @CC_VERSION@ -CFLAGS: @CFLAGS@ -CPPFLAGS: @CPPFLAGS@ -LDFLAGS: @LDFLAGS@ -Shared Library: @enable_shared@ -Static Library: @enable_static@ -Extra libraries: @LIBS@ - diff --git a/ush/setup.py b/ush/setup.py index 4dd6d17692..2f83846830 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -374,10 +374,14 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False): # Create a dictionary of config options from defaults, machine, and # user config files. + build_config_fp = os.path.join(USHdir, os.pardir, "exec", "build_settings.yaml") default_config_fp = os.path.join(USHdir, "config_defaults.yaml") user_config_fp = os.path.join(USHdir, user_config_fn) expt_config = load_config_for_setup(USHdir, default_config_fp, user_config_fp) + # Load build settings as a dictionary; will be used later to make sure the build is consistent with the user settings + build_config = load_config_file(build_config_fp) + # Set up some paths relative to the SRW clone expt_config["user"].update(set_srw_paths(USHdir, expt_config)) @@ -1481,8 +1485,11 @@ def dict_find(user_dict, substring): # Check that UFS FIRE settings are correct and consistent # # ----------------------------------------------------------------------- + print(build_config) fire_conf = expt_config["fire"] if fire_conf["UFS_FIRE"]: + if build_config["Application"]!="ATMF": + raise Exception("UFS_FIRE == True but UFS SRW has not been built for fire coupling; see users guide for details") fire_input_file=os.path.join(fire_conf["FIRE_INPUT_DIR"],"geo_em.d01.nc") if not os.path.isfile(fire_input_file): raise FileNotFoundError( From 6ff7a10aa29bc465dcab57401a6f862a6db315fc Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr" Date: Tue, 8 Oct 2024 15:59:42 -0500 Subject: [PATCH 37/87] Dont use pregenerated ICs in sample case since it needs to work on all platforms --- ush/config.fire.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 069b31566b..914656580b 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -5,8 +5,6 @@ user: RUN_ENVIR: community MACHINE: derecho ACCOUNT: ACCOUNT_NUMBER -platform: - DOMAIN_PREGEN_BASEDIR: "/glade/work/frediani/Projects/UFS-fire/UFS/pregen_dir" workflow: USE_CRON_TO_RELAUNCH: true EXPT_SUBDIR: fire_test_case From 3582c684abe094a26072fa921e04b7afd28db80f Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr" Date: Tue, 8 Oct 2024 16:00:00 -0500 Subject: [PATCH 38/87] Add FC definition to build modulefiles to fix fire module build issue --- modulefiles/build_hera_intel.lua | 2 ++ modulefiles/build_hercules_intel.lua | 2 ++ modulefiles/build_jet_intel.lua | 2 ++ modulefiles/build_odin_intel.lua | 2 ++ modulefiles/build_orion_intel.lua | 2 ++ modulefiles/build_singularity_gnu.lua | 2 ++ 6 files changed, 12 insertions(+) diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index d3d20a5bb5..d7ef51b1e7 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -28,6 +28,8 @@ load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.9.0.1")) load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.1.1")) +setenv("FC", "mpiifort") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index e01b68905e..54c82569e9 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -21,6 +21,8 @@ load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.1.1")) setenv("CFLAGS","-diag-disable=10441") setenv("FFLAGS","-diag-disable=10441") +setenv("FC", "mpiifort") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index 78e70e0960..04124e4bf5 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -17,6 +17,8 @@ load("srw_common") load("nccmp/1.9.0.1") load("nco/5.0.6") +setenv("FC", "mpiifort") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") diff --git a/modulefiles/build_odin_intel.lua b/modulefiles/build_odin_intel.lua index f4059a868a..40c3ada501 100644 --- a/modulefiles/build_odin_intel.lua +++ b/modulefiles/build_odin_intel.lua @@ -45,6 +45,8 @@ load("sigio") load("w3emc") load("wgrib2") +setenv("FC", "ftn") + setenv("CMAKE_C_COMPILER","cc") setenv("CMAKE_CXX_COMPILER","CC") setenv("CMAKE_Fortran_COMPILER","ftn") diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index 92d268a4d7..31efcb57c6 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -21,6 +21,8 @@ load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.1.1")) setenv("CFLAGS","-diag-disable=10441") setenv("FFLAGS","-diag-disable=10441") +setenv("FC", "mpiifort") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") diff --git a/modulefiles/build_singularity_gnu.lua b/modulefiles/build_singularity_gnu.lua index 434ac448d4..275842ffc2 100644 --- a/modulefiles/build_singularity_gnu.lua +++ b/modulefiles/build_singularity_gnu.lua @@ -33,6 +33,8 @@ load("gfsio") load("wgrib2") load("upp") +setenv("FC", "mpif90") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpicxx") setenv("CMAKE_Fortran_COMPILER","mpif90") From bae50a382c7d246a095e624cb8e36fd97ad22beb Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr" Date: Sun, 13 Oct 2024 16:15:13 -0500 Subject: [PATCH 39/87] Fix syntax warning for bad regular expression in WE2E test script --- tests/WE2E/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/WE2E/utils.py b/tests/WE2E/utils.py index e64bcd49ce..05fe72ca53 100755 --- a/tests/WE2E/utils.py +++ b/tests/WE2E/utils.py @@ -176,8 +176,8 @@ def calculate_core_hours(expts_dict: dict) -> dict: # Cycle is last 12 characters, task name is rest (minus separating underscore) taskname = task[:-13] # Handle task names that have ensemble and/or fhr info appended with regex - taskname = re.sub('_mem\d{3}', '', taskname) - taskname = re.sub('_f\d{3}', '', taskname) + taskname = re.sub(r'_mem\d{3}', '', taskname) + taskname = re.sub(r'_f\d{3}', '', taskname) nnodes_var = f'NNODES_{taskname.upper()}' if nnodes_var in vdf: nnodes = vdf[nnodes_var] From 235cbf665716fe9f7034d5424f9276d58aea88d4 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 14 Oct 2024 21:15:57 +0000 Subject: [PATCH 40/87] Fix problems with non-fire cases; opened issue about non-functional threading in current develop --- parm/model_configure | 2 -- ush/create_model_configure_file.py | 2 -- ush/create_ufs_configure_file.py | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/parm/model_configure b/parm/model_configure index aeb45f4719..60d288d114 100644 --- a/parm/model_configure +++ b/parm/model_configure @@ -1,5 +1,4 @@ total_member: 1 -PE_MEMBER01: {{ PE_MEMBER01 }} start_year: {{ start_year }} start_month: {{ start_month }} start_day: {{ start_day }} @@ -13,7 +12,6 @@ ENS_SPS: .false. dt_atmos: {{ dt_atmos }} calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: {{ atmos_nthreads }} restart_interval: {{ restart_interval }} output_1st_tstep_rst: .false. write_dopost: {{ write_dopost }} diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index b8767f635a..c607f496ae 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -70,7 +70,6 @@ def create_model_configure_file( # ----------------------------------------------------------------------- # settings = { - "PE_MEMBER01": PE_MEMBER01, "start_year": cdate.year, "start_month": cdate.month, "start_day": cdate.day, @@ -78,7 +77,6 @@ def create_model_configure_file( "nhours_fcst": fcst_len_hrs, "fhrot": fhrot, "dt_atmos": DT_ATMOS, - "atmos_nthreads": OMP_NUM_THREADS_RUN_FCST, "restart_interval": RESTART_INTERVAL, "itasks": ITASKS, "write_dopost": f".{lowercase(str(WRITE_DOPOST))}.", diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index 0a4e58340b..af861fc45e 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -66,8 +66,7 @@ def create_ufs_configure_file(run_dir,cfg): else: EARTH_component_list = 'ATM' ATM_petlist_bounds = f'0 {atm_end}' - ATM_omp_num_threads_line = \ - f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" + ATM_omp_num_threads_line = '' ATM_diag_line = ' Diagnostic = 0' runseq = [ " ATM" ] From e9d88326a8fb88a78fe89c0080796445fbfbd559 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Tue, 15 Oct 2024 16:53:37 +0000 Subject: [PATCH 41/87] Fixes for OMP threading problem from Michael --- ush/config_defaults.yaml | 2 +- ush/create_ufs_configure_file.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 457e2881a0..3b45552e89 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1939,7 +1939,7 @@ task_run_fcst: QUILTING: true PRINT_ESMF: false - PE_MEMBER01: '{{ LAYOUT_Y * LAYOUT_X + fire.FIRE_NUM_TASKS + WRTCMP_write_groups * WRTCMP_write_tasks_per_group if QUILTING else LAYOUT_Y * LAYOUT_X + fire.FIRE_NUM_TASKS }}' + PE_MEMBER01: '{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + fire.FIRE_NUM_TASKS + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + fire.FIRE_NUM_TASKS) }}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index af861fc45e..bb62bec16a 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -54,7 +54,8 @@ def create_ufs_configure_file(run_dir,cfg): elif cfg["UFS_FIRE"]: EARTH_component_list = 'ATM FIRE' ATM_petlist_bounds = f'0 {atm_end}' - ATM_omp_num_threads_line = '' + ATM_omp_num_threads_line = \ + f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" ATM_diag_line = '' FIRE_petlist_bounds = f'{fire_start} {fire_end}' runseq = [ f" @{cfg['DT_ATMOS']}\n", @@ -66,7 +67,8 @@ def create_ufs_configure_file(run_dir,cfg): else: EARTH_component_list = 'ATM' ATM_petlist_bounds = f'0 {atm_end}' - ATM_omp_num_threads_line = '' + ATM_omp_num_threads_line = \ + f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" ATM_diag_line = ' Diagnostic = 0' runseq = [ " ATM" ] From 0ab8f0256645dfc59667bfc6f234be4c8c1d7fdf Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 15 Oct 2024 14:54:25 -0600 Subject: [PATCH 42/87] Move new build_settings_template to sorc/ directory --- CMakeLists.txt | 2 +- {etc => sorc}/build_settings_template.yaml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {etc => sorc}/build_settings_template.yaml (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 44c0667c12..9ac4ae9470 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,7 +117,7 @@ is_enabled(BUILD_SHARED_LIBS enable_shared) is_enabled(STATUS_PARALLEL HAS_PARALLEL) # Generate file from template. -CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/etc/build_settings_template.yaml" +CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/sorc/build_settings_template.yaml" "${CMAKE_CURRENT_BINARY_DIR}/build_settings.yaml" @ONLY) diff --git a/etc/build_settings_template.yaml b/sorc/build_settings_template.yaml similarity index 100% rename from etc/build_settings_template.yaml rename to sorc/build_settings_template.yaml From 55f06c3d557b2103a34b312c91b8ab4aa1d488d3 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 15 Oct 2024 16:20:07 -0600 Subject: [PATCH 43/87] Move ufs_srweather_app_meta.h.in to sorc/ --- CMakeLists.txt | 2 +- ufs_srweather_app_meta.h.in => sorc/ufs_srweather_app_meta.h.in | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ufs_srweather_app_meta.h.in => sorc/ufs_srweather_app_meta.h.in (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ac4ae9470..c5e40a2392 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,7 +137,7 @@ INSTALL(FILES "${CMAKE_BINARY_DIR}/build_settings.yaml" # Create 'ufs_srweather_app_meta.h' include file. ##### configure_file( - ufs_srweather_app_meta.h.in + sorc/ufs_srweather_app_meta.h.in ufs_srweather_app_meta.h @ONLY) FILE(COPY "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app_meta.h" DESTINATION include) diff --git a/ufs_srweather_app_meta.h.in b/sorc/ufs_srweather_app_meta.h.in similarity index 100% rename from ufs_srweather_app_meta.h.in rename to sorc/ufs_srweather_app_meta.h.in From 513160d3e107274d3153d61f8d834580cef7d14e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 15 Oct 2024 16:20:35 -0600 Subject: [PATCH 44/87] Update instructions for custom vertical levels; I completely forgot I made this improvement months ago! --- .../CustomizingTheWorkflow/LAMGrids.rst | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/doc/UsersGuide/CustomizingTheWorkflow/LAMGrids.rst b/doc/UsersGuide/CustomizingTheWorkflow/LAMGrids.rst index 482caf8590..7bc13c0794 100644 --- a/doc/UsersGuide/CustomizingTheWorkflow/LAMGrids.rst +++ b/doc/UsersGuide/CustomizingTheWorkflow/LAMGrids.rst @@ -429,37 +429,24 @@ After hitting ``Enter``, the program will print a ``pmin`` value (e.g., ``pmin= Configure the SRW App ----------------------- -To use the new ``ak``/``bk`` file to define vertical levels in an experiment, users will need to modify the input namelist file (``input.nml.FV3``) and their configuration file (``config.yaml``). - -Modify ``input.nml.FV3`` -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The FV3 namelist file, ``input.nml.FV3``, is located in ``ufs-srweather-app/parm``. Users will need to update the ``levp`` and ``npz`` variables in this file. For ``n`` vertical levels, users should set ``levp=n`` and ``npz=n-1``. For example, a user who wants 128 vertical levels would set ``levp`` and ``npz`` as follows: - -.. code-block:: console - - &external_ic_nml - levp = 128 - - &fv_core_nml - npz = 127 - -Additionally, check that ``external_eta = .true.``. - -.. note:: - - Keep in mind that levels and layers are not the same. In UFS code, ``levp`` is the number of vertical *levels*, and ``npz`` is the number of vertical levels without TOA. Thus, ``npz`` is equivalent to the number of vertical *layers*. For ``v`` vertical *layers*, set ``npz=v`` and ``levp=v+1``. Use the value of ``levp`` as the number of vertical levels when generating ``ak``/``bk``. +To use the new ``ak``/``bk`` file to define vertical levels in an experiment, users will need to modify their configuration file (``config.yaml``). Modify ``config.yaml`` ^^^^^^^^^^^^^^^^^^^^^^^^ -To use the text file produced by ``vcoord_gen`` in the SRW App, users need to set the ``VCOORD_FILE`` variable in their ``config.yaml`` file. Normally, this file is named ``global_hyblev.l65.txt`` and is located in the ``fix_am`` directory on Level 1 systems, but users should adjust the path and name of the file to suit their system. For example, in ``config.yaml``, a user (Jane Smith) might set: +To use the new vertical levels produced by ``vcoord_gen`` in the SRW App, users need to set the ``VCOORD_FILE`` and ``LEVP`` variables in their ``config.yaml`` file. Normally, the vertical coordinate file is named ``global_hyblev.l65.txt`` and is located in the ``fix_am`` directory on Level 1 systems, but users should adjust the path and name of the file to suit their system. Additionally, update the ``LEVP`` variable for the new number of vertical levels. For example, in ``config.yaml``, a user (Jane Smith) might set: .. code-block:: console task_make_ics: + LEVP: 65 VCOORD_FILE: /Users/Jane.Smith/ufs-srweather-app/parm/global_hyblev.L128.txt task_make_lbcs: + LEVP: 65 VCOORD_FILE: /Users/Jane.Smith/ufs-srweather-app/parm/global_hyblev.L128.txt +.. note:: + + Keep in mind that levels and layers are not the same. In UFS code, ``levp`` is the number of vertical *levels*, and ``npz`` is the number of vertical levels without TOA. Thus, ``npz`` in the UFS namelist file is equivalent to the number of vertical *layers*. For ``v`` vertical *layers*, ``npz=v`` and ``levp=v+1``. Use the value of ``levp`` as the number of vertical levels when generating ``ak``/``bk``. + Configure other variables as desired and generate the experiment as described in :numref:`Section %s `. From 36df95be6a87eba1dd2ed65e3e60fe936cb97e1e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 15 Oct 2024 16:57:54 -0600 Subject: [PATCH 45/87] Update fire case for shorter forecast, and to use new variable syntax --- ush/config.fire.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 914656580b..68ec0188a7 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -12,20 +12,20 @@ workflow: PREDEF_GRID_NAME: SUBCONUS_CO_3km DATE_FIRST_CYCL: '2020081318' DATE_LAST_CYCL: '2020081318' - FCST_LEN_HRS: 27 + FCST_LEN_HRS: 6 # 27 hours of LBCs staged on Derecho PREEXISTING_DIR_METHOD: rename VERBOSE: true COMPILER: intel task_get_extrn_ics: USE_USER_STAGED_EXTRN_FILES: true - EXTRN_MDL_SOURCE_BASEDIR_ICS: "/glade/derecho/scratch/kavulich/FIRE/2024/test_data/HRRR" + EXTRN_MDL_SOURCE_BASEDIR_ICS: '{{ fire.FIRE_INPUT_DIR }}/HRRR' EXTRN_MDL_NAME_ICS: HRRR FV3GFS_FILE_FMT_ICS: grib2 EXTRN_MDL_FILES_ICS: - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true - EXTRN_MDL_SOURCE_BASEDIR_LBCS: "/glade/derecho/scratch/kavulich/FIRE/2024/test_data/HRRR" + EXTRN_MDL_SOURCE_BASEDIR_LBCS: '{{ fire.FIRE_INPUT_DIR }}/HRRR' EXTRN_MDL_FILES_LBCS: - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' EXTRN_MDL_NAME_LBCS: HRRR @@ -63,5 +63,5 @@ rocoto: tasks: metatask_run_ensemble: task_run_fcst_mem#mem#: - walltime: 02:00:00 + walltime: 01:00:00 From 6dd02bb0954463a1eb2566a4eb32ad246a8eeef7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 15 Oct 2024 16:58:32 -0600 Subject: [PATCH 46/87] Additional consistency checks in setup.py, remove set -x from run_fcst --- scripts/exregional_run_fcst.sh | 1 - ush/setup.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index d13369f195..c5e7f6601e 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -196,7 +196,6 @@ if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} )) FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]} fi -set -x # #----------------------------------------------------------------------- # diff --git a/ush/setup.py b/ush/setup.py index 5359aa540c..5cdb34ad81 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1527,10 +1527,13 @@ def _dict_find(user_dict, substring): ) ) if fire_conf["FIRE_NUM_TASKS"] < 1: - raise ValueError("FIRE_NUM_TASKS must be > 0") + raise ValueError("FIRE_NUM_TASKS must be > 0 if UFS_FIRE is True") if fire_conf["FIRE_UPWINDING"] == 0 and fire_conf["FIRE_VISCOSITY"] == 0.0: raise ValueError("FIRE_VISCOSITY must be > 0.0 if FIRE_UPWINDING == 0") + else: + if fire_conf["FIRE_NUM_TASKS"] < 1: + logging.warning("UFS_FIRE is not enabled; setting FIRE_NUM_TASKS = 0") # # ----------------------------------------------------------------------- From a92133bee16aa922cf7b2aa3f10b183fbcb59b94 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 07:57:47 -0600 Subject: [PATCH 47/87] Make the linter happy --- ush/create_ufs_configure_file.py | 50 ++++++++++++++------------------ 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index 1e3776da86..a4091a0e03 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -11,12 +11,9 @@ from uwtools.api.template import render from python_utils import ( - cfg_to_yaml_str, flatten_dict, - import_vars, load_yaml_config, print_info_msg, - print_input_args, ) def create_ufs_configure_file(run_dir,cfg): @@ -38,10 +35,10 @@ def create_ufs_configure_file(run_dir,cfg): fire_end = str(int(cfg["PE_MEMBER01"]) - 1) if cfg["CPL_AQM"]: - EARTH_component_list = 'ATM AQM' - ATM_petlist_bounds = '-1 -1' - ATM_omp_num_threads_line = '' - ATM_diag_line = '' + earth_component_list = 'ATM AQM' + atm_petlist_bounds = '-1 -1' + atm_omp_num_threads_line = '' + atm_diag_line = '' runseq = [ f" @{cfg['DT_ATMOS']}\n", " ATM phase1\n", " ATM -> AQM\n", @@ -50,12 +47,12 @@ def create_ufs_configure_file(run_dir,cfg): " ATM phase2\n", " @" ] elif cfg["UFS_FIRE"]: - EARTH_component_list = 'ATM FIRE' - ATM_petlist_bounds = f'0 {atm_end}' - ATM_omp_num_threads_line = \ + earth_component_list = 'ATM FIRE' + atm_petlist_bounds = f'0 {atm_end}' + atm_omp_num_threads_line = \ f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" - ATM_diag_line = '' - FIRE_petlist_bounds = f'{fire_start} {fire_end}' + atm_diag_line = '' + fire_petlist_bounds = f'{fire_start} {fire_end}' runseq = [ f" @{cfg['DT_ATMOS']}\n", " ATM -> FIRE\n", " FIRE -> ATM :remapmethod=conserve\n", @@ -63,11 +60,11 @@ def create_ufs_configure_file(run_dir,cfg): " FIRE\n", " @" ] else: - EARTH_component_list = 'ATM' - ATM_petlist_bounds = f'0 {atm_end}' - ATM_omp_num_threads_line = \ + earth_component_list = 'ATM' + atm_petlist_bounds = f'0 {atm_end}' + atm_omp_num_threads_line = \ f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" - ATM_diag_line = ' Diagnostic = 0' + atm_diag_line = ' Diagnostic = 0' runseq = [ " ATM" ] if cfg["PRINT_ESMF"]: @@ -82,7 +79,7 @@ def create_ufs_configure_file(run_dir,cfg): #----------------------------------------------------------------------- # print_info_msg(f''' - Creating a ufs.configure file (\"{cfg["UFS_CONFIG_FN"]}\") in the specified + Creating a ufs.configure file (\"{cfg["UFS_CONFIG_FN"]}\") in the specified run directory (run_dir): {run_dir=}''', verbose=cfg["VERBOSE"]) # @@ -99,13 +96,12 @@ def create_ufs_configure_file(run_dir,cfg): #----------------------------------------------------------------------- # settings = { - "cpl_aqm": cfg["CPL_AQM"], "ufs_fire": cfg["UFS_FIRE"], "logKindFlag": logkindflag, - "EARTH_cl": EARTH_component_list, - "ATM_pb": ATM_petlist_bounds, - "ATM_omp_num_threads_line": ATM_omp_num_threads_line, - "ATM_diag_line": ATM_diag_line, + "EARTH_cl": earth_component_list, + "ATM_pb": atm_petlist_bounds, + "ATM_omp_num_threads_line": atm_omp_num_threads_line, + "ATM_diag_line": atm_diag_line, "runseq": runseq, "FIRE_pb": "", "dt_atmos": cfg["DT_ATMOS"], @@ -113,9 +109,7 @@ def create_ufs_configure_file(run_dir,cfg): "cpl_aqm": cfg["CPL_AQM"] } if cfg["UFS_FIRE"]: - settings["FIRE_pb"] = FIRE_petlist_bounds - - settings_str = cfg_to_yaml_str(settings) + settings["FIRE_pb"] = fire_petlist_bounds print_info_msg( dedent( @@ -160,6 +154,6 @@ def _parse_args(argv): if __name__ == "__main__": args = _parse_args(sys.argv[1:]) - cfg = load_yaml_config(args.path_to_defns) - cfg = flatten_dict(cfg) - create_ufs_configure_file(run_dir=args.run_dir,cfg=cfg) + conf = load_yaml_config(args.path_to_defns) + confg = flatten_dict(conf) + create_ufs_configure_file(run_dir=args.run_dir,cfg=conf) From 4076da51bad99f88997197d6732f2a9f3a230ce2 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 08:22:36 -0600 Subject: [PATCH 48/87] Fix long line in create diag table script, set build settings to only print in debug mode --- ush/create_diag_table_file.py | 3 ++- ush/setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index ae2a649822..9ae7530b3a 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -60,7 +60,8 @@ def create_diag_table_file(run_dir): settings = {"starttime": CDATE, "cres": CRES, "additional_entries": ""} if UFS_FIRE: - settings["additional_entries"] = '"gfs_phys","fsmoke","fsmoke","fv3_history","all",.false.,"none",2' + settings["additional_entries"] = \ + '"gfs_phys","fsmoke","fsmoke","fv3_history","all",.false.,"none",2' settings_str = cfg_to_yaml_str(settings) print_info_msg( diff --git a/ush/setup.py b/ush/setup.py index 5cdb34ad81..cef6b9edc8 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -406,6 +406,7 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False): # Load build settings as a dictionary; will be used later to make sure the build is consistent with the user settings build_config = load_config_file(build_config_fp) + logger.debug(f"Build configuration\n{build_config}") # Set up some paths relative to the SRW clone expt_config["user"].update(set_srw_paths(USHdir, expt_config)) @@ -1510,7 +1511,6 @@ def _dict_find(user_dict, substring): # Check that UFS FIRE settings are correct and consistent # # ----------------------------------------------------------------------- - print(build_config) fire_conf = expt_config["fire"] if fire_conf["UFS_FIRE"]: if build_config["Application"]!="ATMF": From 683eba0f12435cda59b4d318d782b152b436be0f Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 08:34:22 -0600 Subject: [PATCH 49/87] Fix linting for generate_FV3LAM_wflow.py. Really wish there was a way to run these tests locally... --- ush/generate_FV3LAM_wflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 47ca2cc9a3..e6f6869d1b 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -459,7 +459,7 @@ def generate_FV3LAM_wflow( "units", "kg/kg" "profile_type", "fixed", "surface_value=0.0" /\n""" - with open(FIELD_TABLE_FP, "a+") as file: + with open(FIELD_TABLE_FP, "a+", encoding='UTF-8') as file: file.write(field_table_append) settings["gfs_physics_nml"] = gfs_physics_nml_dict From 997c0972ae72c0200045fff355ba3802540257a6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 08:37:22 -0600 Subject: [PATCH 50/87] Manage externals takes forever and rarely fails, so run that test last --- .github/workflows/python_tests.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index fb0de16910..69b2e30dc1 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -30,10 +30,6 @@ jobs: cache-downloads: true cache-environment: true - - name: Checkout externals - run: | - ./manage_externals/checkout_externals ufs-weather-model - - name: Lint the python code run: | micromamba activate srw_app @@ -58,3 +54,8 @@ jobs: export CI=true export PYTHONPATH=${PWD}/ush python3 -m unittest -b tests/test_python/test_retrieve_data.py + + - name: Checkout externals + run: | + ./manage_externals/checkout_externals ufs-weather-model + From 0042f6c9153c91cde7ffa170ae910ef2786eb22c Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 09:52:09 -0600 Subject: [PATCH 51/87] Update diag_table test --- tests/test_python/test_create_diag_table_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_python/test_create_diag_table_file.py b/tests/test_python/test_create_diag_table_file.py index f7559536f2..d90dcb3cd0 100644 --- a/tests/test_python/test_create_diag_table_file.py +++ b/tests/test_python/test_create_diag_table_file.py @@ -29,3 +29,4 @@ def setUp(self): set_env_var("DIAG_TABLE_TMPL_FP", diag_table_tmpl_fp) set_env_var("CRES", "C48") set_env_var("CDATE", "2021010106") + set_env_var("UFS_FIRE", False) From c4f0fffedf945eb0b208abae6971e5f0c3f31181 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 11:16:55 -0600 Subject: [PATCH 52/87] We don't want to hide output from CI unit tests in case there's a failure --- .github/workflows/python_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index 69b2e30dc1..aa1027fcbb 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -46,14 +46,14 @@ jobs: micromamba activate srw_app export UNIT_TEST=True export PYTHONPATH=$(pwd)/ush - python -m unittest -b tests/test_python/*.py + python -m unittest tests/test_python/*.py - name: Run python functional tests run: | micromamba activate srw_app export CI=true export PYTHONPATH=${PWD}/ush - python3 -m unittest -b tests/test_python/test_retrieve_data.py + python3 -m unittest tests/test_python/test_retrieve_data.py - name: Checkout externals run: | From b63514c2149ebef0be034886899b2cbcb2effcae Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 11:53:47 -0600 Subject: [PATCH 53/87] Need dummy build settings file for test_generate_FV3LAM_wflow.py now --- tests/test_python/test_generate_FV3LAM_wflow.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_python/test_generate_FV3LAM_wflow.py b/tests/test_python/test_generate_FV3LAM_wflow.py index 069660e5d8..f30367690d 100644 --- a/tests/test_python/test_generate_FV3LAM_wflow.py +++ b/tests/test_python/test_generate_FV3LAM_wflow.py @@ -39,6 +39,14 @@ def run_workflow(USHdir, logfile): logfile = "log.generate_FV3LAM_wflow" sed = get_env_var("SED") + # create a dummy build settings file if needed + EXECdir = os.path.join(USHdir, "..", "exec") + build_settings_file = os.path.join(EXECdir, "build_settings.yaml") + if not os.path.exists(build_settings_file): + os.makedirs(EXECdir) + with open(build_settings_file, 'w') as build_settings: + build_settings.write('Application:\n') + # community test case cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") run_command( From 32eee724eb647cdd04344cbbd358b11a39f62151 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 12:13:08 -0600 Subject: [PATCH 54/87] Fix the linter for the test. This is an extremely annoying process! --- tests/test_python/test_generate_FV3LAM_wflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_python/test_generate_FV3LAM_wflow.py b/tests/test_python/test_generate_FV3LAM_wflow.py index f30367690d..4815265ff1 100644 --- a/tests/test_python/test_generate_FV3LAM_wflow.py +++ b/tests/test_python/test_generate_FV3LAM_wflow.py @@ -40,11 +40,11 @@ def run_workflow(USHdir, logfile): sed = get_env_var("SED") # create a dummy build settings file if needed - EXECdir = os.path.join(USHdir, "..", "exec") + EXECdir = os.path.join(USHdir, "..", "exec") build_settings_file = os.path.join(EXECdir, "build_settings.yaml") if not os.path.exists(build_settings_file): os.makedirs(EXECdir) - with open(build_settings_file, 'w') as build_settings: + with open(build_settings_file, 'w', encoding='utf-8') as build_settings: build_settings.write('Application:\n') # community test case From 0b2dd99f05eb0e45e9ac2542aaa026e52eb15c35 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 16 Oct 2024 12:24:54 -0600 Subject: [PATCH 55/87] Forgot you need externals for the unit tests --- .github/workflows/python_tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index aa1027fcbb..113ec3f59c 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -40,6 +40,10 @@ jobs: pylint ush/set_fv3nml*.py pylint ush/update_input_nml.py + - name: Checkout externals + run: | + ./manage_externals/checkout_externals ufs-weather-model + - name: Run python unittests run: | # exclude test_retrieve_data that is tested in functional test @@ -55,7 +59,3 @@ jobs: export PYTHONPATH=${PWD}/ush python3 -m unittest tests/test_python/test_retrieve_data.py - - name: Checkout externals - run: | - ./manage_externals/checkout_externals ufs-weather-model - From c101eb0473241f6ef2689b016096b935fd45c118 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 31 Oct 2024 22:14:50 -0600 Subject: [PATCH 56/87] Add new fire WE2E test; will only work on Derecho for now. Also move "aqm" directory to "special_builds" directory to go with fire --- .../special_builds/config.UFS_FIRE.yaml | 62 +++++++++++++++++++ ...fig.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml | 0 2 files changed, 62 insertions(+) create mode 100644 tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml rename tests/WE2E/test_configs/{aqm => special_builds}/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml (100%) diff --git a/tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml b/tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml new file mode 100644 index 0000000000..9ae35c21af --- /dev/null +++ b/tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml @@ -0,0 +1,62 @@ +metadata: + description: >- + Tests the UFS_FIRE capability +user: + RUN_ENVIR: community +workflow: + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: SUBCONUS_CO_3km + DATE_FIRST_CYCL: '2020081318' + DATE_LAST_CYCL: '2020081318' + FCST_LEN_HRS: 6 # 27 hours of LBCs staged on Derecho + PREEXISTING_DIR_METHOD: rename +task_get_extrn_ics: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_ICS: '{{ fire.FIRE_INPUT_DIR }}/HRRR' + EXTRN_MDL_NAME_ICS: HRRR + FV3GFS_FILE_FMT_ICS: grib2 + EXTRN_MDL_FILES_ICS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' +task_get_extrn_lbcs: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_LBCS: '{{ fire.FIRE_INPUT_DIR }}/HRRR' + EXTRN_MDL_FILES_LBCS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' + EXTRN_MDL_NAME_LBCS: HRRR + EXTRN_MDL_LBCS_OFFSET_HRS: 0 + LBC_SPEC_INTVL_HRS: 3 + FV3GFS_FILE_FMT_LBCS: grib2 +task_run_fcst: + QUILTING: true + PRINT_ESMF: True + OMP_NUM_THREADS_RUN_FCST: 1 +task_plot_allvars: + COMOUT_REF: "" +fire: + UFS_FIRE: True + FIRE_NUM_TASKS: 1 + FIRE_INPUT_DIR: /glade/derecho/scratch/kavulich/FIRE/2024/test_data + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS1: 0.05 + FIRE_IGNITION_START_LAT1: 40.609 + FIRE_IGNITION_START_LON1: -105.879 + FIRE_IGNITION_END_LAT1: 40.609 + FIRE_IGNITION_END_LON1: -105.879 + FIRE_IGNITION_RADIUS1: 250 + FIRE_IGNITION_START_TIME1: 6480 + FIRE_IGNITION_END_TIME1: 7000 + FIRE_WIND_HEIGHT: 5.0 + FIRE_PRINT_MSG: 0 + FIRE_ATM_FEEDBACK: 1.0 + FIRE_VISCOSITY: 0.4 + FIRE_UPWINDING: 9 + FIRE_LSM_ZCOUPLING: False + FIRE_LSM_ZCOUPLING_REF: 60.0 +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 01:00:00 + diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/special_builds/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml similarity index 100% rename from tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml rename to tests/WE2E/test_configs/special_builds/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml From eca0ec4c319c62397cf25f39904ece782285024e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 31 Oct 2024 22:15:05 -0600 Subject: [PATCH 57/87] Fix error introduced when trying to please linter --- ush/create_ufs_configure_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index a4091a0e03..0ffebbd06f 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -156,4 +156,4 @@ def _parse_args(argv): args = _parse_args(sys.argv[1:]) conf = load_yaml_config(args.path_to_defns) confg = flatten_dict(conf) - create_ufs_configure_file(run_dir=args.run_dir,cfg=conf) + create_ufs_configure_file(run_dir=args.run_dir,cfg=confg) From 1191be3da12bba1a94a6cbf4cbf112caee4e10bf Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 31 Oct 2024 22:20:11 -0600 Subject: [PATCH 58/87] Add Fire documentation to Users Guide --- .../BuildingRunningTesting/FIRE.rst | 259 ++++++++++++++++++ .../BuildingRunningTesting/index.rst | 1 + doc/UsersGuide/Reference/Glossary.rst | 3 + doc/conf.py | 3 +- 4 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 doc/UsersGuide/BuildingRunningTesting/FIRE.rst diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst new file mode 100644 index 0000000000..ec37390000 --- /dev/null +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -0,0 +1,259 @@ +.. _UFS_FIRE: + +========================================= +Community Fire Behavior Module (UFS FIRE) +========================================= + +The `Community Fire Behavior Model (CFBM) `_ is a wildland fire model coupled to the UFS Atmospheric Model. The capability to run this code is now available in the UFS Short-Range Weather App for easy use by the community. The `fire_behavior repository `_ is a :term:`submodule` of the UFS Weather Model (WM), coupled through the :term:`NUOPC` Layer to provide direct feedback between the simulated atmosphere and the simulated fire. More information about the CFBM can be found in the :fire-ug:`CFBM Users Guide <>`. + +The biggest difference between the UFS FIRE capability and other modes of the UFS SRW is that a special build flag is required to build the coupled fire behavior code, as described in the instructions below. Aside from that, additional input files, and some fire-specific config settings, building and running is the same as any other use of SRW. + + +.. note:: + + Although this chapter is the primary documentation resource for running the UFS FIRE configuration, users may need to refer to :numref:`Chapter %s ` and :numref:`Chapter %s ` for additional information on building and running the SRW App, respectively. + +Quick Start Guide (UFS FIRE) +===================================== + +Download the Code +------------------- + +Clone the |branch| branch of the authoritative SRW App repository: + +.. code-block:: console + + git clone -b develop https://github.com/ufs-community/ufs-srweather-app + cd ufs-srweather-app + +Checkout Externals +--------------------- + +Users must run the ``checkout_externals`` script to collect (or "check out") the individual components of the SRW App from their respective GitHub repositories. + +.. code-block:: console + + ./manage_externals/checkout_externals + +Build the SRW App with Fire Behavior enabled +-------------------------------------------- + +To build the SRW with fire behavior code, use the following command: + +.. code-block:: console + + ./devbuild.sh -p= -a=ATMF + +where ```` is ``hera``, ``derecho``, or any other Tier 1 platform. The ``-a`` argument indicates the configuration/version of the application to build; in this case, the atmosphere-fire coupling (ATMF). + +If UFS FIRE builds correctly, users should see the standard executables listed in :numref:`Table %s `. There are no additional files expected, since the CFBM is coupled to the UFS weather model via the same ``ufs_model`` executable. + +Load the |wflow_env| Environment +-------------------------------------------- + +Load the python environment for the workflow: + +.. code-block:: console + + module use /path/to/ufs-srweather-app/modulefiles + module load wflow_ + +where ```` is ``hera``, ``derecho``, or any other Tier 1 platform. + +If the console outputs a message, the user should run the commands specified in the message. For example, if the output says: + +.. code-block:: console + + Please do the following to activate conda: + > conda activate srw_app + +then the user should run |activate|. Otherwise, the user can continue with configuring the workflow. + +.. _FIREConfig: + +Configure Experiment +--------------------------- + +Users will need to configure their experiment by setting parameters in the ``config.yaml`` file. To start, users can copy an example experiment setting into ``config.yaml``: + +.. code-block:: console + + cd ush + cp config.fire.yaml config.yaml + +Users will need to change the ``MACHINE`` and ``ACCOUNT`` variables in ``config.yaml`` to match their system. They may also wish to adjust other experiment settings, especially under the ``fire:`` section, described in further detail below. For more information on other configuration settings, see :numref:`Section %s `. + +Activating the fire behavior module is done by setting ``UFS_FIRE: True`` in the ``fire:`` section of your ``config.yaml`` file. If this variable is not specified or set to false, a normal atmospheric simulation will be run, without fire settings. + +.. code-block:: console + + fire: + UFS_FIRE: True + +The fire module has the ability to print out additional messages to the log file for debugging; to enable additional log output (which may slow down the integration considerably, especially at higher levels) set ``FIRE_PRINT_MSG`` > 0 + +.. code-block:: console + + fire: + FIRE_PRINT_MSG: 1 + +Additional boundary conditions file +----------------------------------- +The CFBM, as an independent, coupled component, runs separately from the atmospheric component of the weather model, requires an additional input file (``geo_em.d01.nc``) that contains fire-specific boundary conditions such as fuel properties. On Level 1 systems, users can find an example file in the usual :ref:`input data locations ` under ``LOCATION``. Users can also download the data required for the community experiment from the `UFS SRW App Data Bucket `__. + + +Instructions on how to create this file for your own experiment can be found in the :fire-ug:`CFBM Users Guide `. + +Once the file is acquired/created, you will need to specify its location in your ``config.yaml`` file with the setting ``FIRE_INPUT_DIR``. + +.. code-block:: console + + fire: + FIRE_INPUT_DIR: /directory/containing/geo_em/file + + + +Specifying a fire ignition +--------------------------- + +The CFBM simulates fires by specifying an "ignition" that will then propogate based on the atmospheric conditions and the specified settings. An ignition can either be a "point ignition" (i.e. a disk of fire some specified radius around a single location), or a straight line linear ignition specified by a start and end location and a specified "radius" (width). The ignition can start at the beginning of your simulation, or at some time later as specified. The CFBM can support up to 5 different fire ignitions at different places and times in a given simulation. + +The CFBM settings are controlled by the :term:`namelist` file ``namelist.fire``. The available settings in this file are described in the :fire-ug:`CFBM Users Guide `, and an example file can be found under ``parm/namelist.fire``. However, there is no need to manually provide or edit this file, as the SRW workflow will create the fire namelist using the user settings in ``config.yaml``. + +Example fire configuration +--------------------------- + +Here is one example of settings that can be specified for a UFS FIRE simulation. + +.. code-block:: console + + fire: + UFS_FIRE: True + FIRE_INPUT_DIR: /home/fire_input + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 1800 + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS1: 0.05 + FIRE_IGNITION_START_LAT1: 40.609 + FIRE_IGNITION_START_LON1: -105.879 + FIRE_IGNITION_END_LAT1: 40.609 + FIRE_IGNITION_END_LON1: -105.879 + FIRE_IGNITION_RADIUS1: 250 + FIRE_IGNITION_START_TIME1: 6480 + FIRE_IGNITION_END_TIME1: 7000 + +In this case, a single fire (``FIRE_NUM_IGNITIONS: 1``) of radius 250 meters (``FIRE_IGNITION_RADIUS1: 250``) is ignited at latitude 40.609˚N (``FIRE_IGNITION_START_LAT1: 40.609``), 105.879˚W (``FIRE_IGNITION_START_LON1: -105.879``) 6480 seconds after the start of the simulation (``FIRE_IGNITION_START_TIME1: 6480``) with a rate of spread specified as 0.05 m/s (``FIRE_IGNITION_ROS1: 0.05``). This "ignition" ends 7000 seconds after the start of the simulation (``FIRE_IGNITION_END_TIME1: 7000``), after which the fire behavior is completely governed by the physics of the fire behavior model (integrated every 0.5 seconds as specified by ``OUTPUT_DT_FIRE``), the input fuel conditions, and the simulated atmospheric conditions. + +The CFBM creates output files in :term:`netCDF` format, with the naming scheme ``fire_output_YYYY-MM-DD_hh:mm:ss.nc``. In this case the output files are written every 30 minutes (``OUTPUT_DT_FIRE: 1800``). + +.. note:: + + Any of the settings under :fire-ug:`the ``&fire`` section of the namelist ` can be specified in the SRW App ``config.yaml`` file under the ``fire:`` section, not just the settings described above. However, any additional settings from ``namelist.fire`` will need to be added to ``config_defaults.yaml`` first; otherwise the check for valid SRW options will fail. + + +Generate the Workflow +------------------------ + +Generate the workflow: + +.. code-block:: console + + ./generate_FV3LAM_wflow.py + +Run the Workflow +------------------ + +If ``USE_CRON_TO_RELAUNCH`` is set to true in ``config.yaml``, the workflow will run automatically. If it was set to false, users must submit the workflow manually from the experiment directory: + +.. code-block:: console + + cd ${EXPT_BASEDIR}/${EXPT_SUBDIR} + ./launch_FV3LAM_wflow.sh + +Repeat the launch command regularly until a SUCCESS or FAILURE message appears on the terminal window. See :numref:`Section %s ` for more on the ``${EXPT_BASEDIR}`` and ``${EXPT_SUBDIR}`` variables. + +Users may check experiment status from the experiment directory with either of the following commands: + +.. code-block:: console + + # Check the experiment status (for cron jobs) + rocotostat -w FV3LAM_wflow.xml -d FV3LAM_wflow.db -v 10 + + # Check the experiment status and relaunch the workflow (for manual jobs) + ./launch_FV3LAM_wflow.sh; tail -n 40 log.launch_FV3LAM_wflow + +To see a description of each of the FIRE workflow tasks, see :numref:`Section %s `. + +.. _FIRESuccess: + +Experiment Output +-------------------- + +The workflow run is complete when all tasks display a "SUCCEEDED" message. If everything goes smoothly, users will eventually see a workflow status table similar to the following: + +.. code-block:: console + + CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION + ============================================================================================ + 202302170000 make_grid 47411619 SUCCEEDED 0 1 36.0 + 202302170000 make_orog 47411728 SUCCEEDED 0 1 151.0 + 202302170000 make_sfc_climo 47411801 SUCCEEDED 0 1 58.0 + 202302170000 nexus_gfs_sfc 47411620 SUCCEEDED 0 1 37.0 + 202302170000 nexus_emission_00 47411729 SUCCEEDED 0 1 251.0 + 202302170000 nexus_emission_01 47411730 SUCCEEDED 0 1 250.0 + 202302170000 nexus_emission_02 47411731 SUCCEEDED 0 1 250.0 + 202302170000 nexus_post_split 47412034 SUCCEEDED 0 1 44.0 + 202302170000 fire_emission 47411621 SUCCEEDED 0 1 19.0 + 202302170000 point_source 47411732 SUCCEEDED 0 1 82.0 + 202302170000 aqm_lbcs 47412961 SUCCEEDED 0 1 159.0 + 202302170000 get_extrn_ics 47411622 SUCCEEDED 0 1 314.0 + 202302170000 get_extrn_lbcs 47411623 SUCCEEDED 0 1 0.0 + 202302170000 make_ics_mem000 47659593 SUCCEEDED 0 1 126.0 + 202302170000 make_lbcs_mem000 47659594 SUCCEEDED 0 1 113.0 + 202302170000 run_fcst_mem000 47659742 SUCCEEDED 0 1 763.0 + 202302170000 run_post_mem000_f000 47659910 SUCCEEDED 0 1 30.0 + 202302170000 run_post_mem000_f001 47660029 SUCCEEDED 0 1 30.0 + 202302170000 run_post_mem000_f002 47660030 SUCCEEDED 0 1 31.0 + ... + 202302170000 run_post_mem000_f006 47660110 SUCCEEDED 0 1 29.0 + ============================================================================================ + 202302170600 nexus_gfs_sfc 47659421 SUCCEEDED 0 1 44.0 + 202302170600 nexus_emission_00 47659475 SUCCEEDED 0 1 323.0 + 202302170600 nexus_emission_01 47659476 SUCCEEDED 0 1 323.0 + 202302170600 nexus_emission_02 47659477 SUCCEEDED 0 1 329.0 + 202302170600 nexus_post_split 47659595 SUCCEEDED 0 1 60.0 + 202302170600 fire_emission 47659422 SUCCEEDED 0 1 18.0 + 202302170600 point_source 47659478 SUCCEEDED 0 1 128.0 + 202302170600 aqm_ics 47659597 SUCCEEDED 0 1 159.0 + 202302170600 aqm_lbcs 47659598 SUCCEEDED 0 1 158.0 + 202302170600 get_extrn_ics 47659423 SUCCEEDED 0 1 493.0 + 202302170600 get_extrn_lbcs 47659424 SUCCEEDED 0 1 536.0 + 202302170600 make_ics_mem000 47659594 SUCCEEDED 0 1 134.0 + 202302170600 make_lbcs_mem000 47659596 SUCCEEDED 0 1 112.0 + 202302170600 run_fcst_mem000 47659812 SUCCEEDED 0 1 1429.0 + 202302170600 run_post_mem000_f000 47659998 SUCCEEDED 0 1 30.0 + 202302170600 run_post_mem000_f001 47660042 SUCCEEDED 0 1 31.0 + 202302170600 run_post_mem000_f002 47660043 SUCCEEDED 0 1 29.0 + ... + 202302170600 run_post_mem000_f012 47660134 SUCCEEDED 0 1 30.0 + +.. _FIRE-WE2E: + + +WE2E Test for FIRE +======================= + +Build the app for FIRE: + +.. code-block:: console + + ./devbuild.sh -p=hera -a=ATMF + + +Run the WE2E test: + +.. code-block:: console + + $ cd /path/to/ufs-srweather-app/tests/WE2E + $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q -t UFS_FIRE + diff --git a/doc/UsersGuide/BuildingRunningTesting/index.rst b/doc/UsersGuide/BuildingRunningTesting/index.rst index 5c0efc3c64..a0aa69c85b 100644 --- a/doc/UsersGuide/BuildingRunningTesting/index.rst +++ b/doc/UsersGuide/BuildingRunningTesting/index.rst @@ -13,3 +13,4 @@ Building, Running, and Testing the SRW App Tutorial VXCases AQM + FIRE diff --git a/doc/UsersGuide/Reference/Glossary.rst b/doc/UsersGuide/Reference/Glossary.rst index 7ffc569b21..48fb970cab 100644 --- a/doc/UsersGuide/Reference/Glossary.rst +++ b/doc/UsersGuide/Reference/Glossary.rst @@ -239,6 +239,9 @@ Glossary spack-stack The `spack-stack `_ is a collaborative effort between the NOAA Environmental Modeling Center (EMC), the UCAR Joint Center for Satellite Data Assimilation (JCSDA), and the Earth Prediction Innovation Center (EPIC). *spack-stack* is a repository that provides a :term:`Spack`-based method for building the software stack required for numerical weather prediction (NWP) tools such as the `Unified Forecast System (UFS) `_ and the `Joint Effort for Data assimilation Integration (JEDI) `_ framework. *spack-stack* uses the Spack package manager along with custom Spack configuration files and Python scripts to simplify installation of the libraries required to run various applications. The *spack-stack* can be installed on a range of platforms and comes pre-configured for many systems. Users can install the necessary packages for a particular application and later add the missing packages for another application without having to rebuild the entire stack. To get started, check out the documentation :doc:`here `. + submodule + A `submodule `_ is a git repository linked to another repository as a subdirectory. Many UFS components are linked in this way; for example, the :term:`UPP` repository is a submodule of the :term:`FV3` repository. + tracer According to the American Meteorological Society (AMS) definition, a `tracer `_ is "Any substance in the atmosphere that can be used to track the history [i.e., movement] of an air mass." Tracers are carried around by the motion of the atmosphere (i.e., by :term:`advection`). These substances are usually gases (e.g., water vapor, CO2), but they can also be non-gaseous (e.g., rain drops in microphysics parameterizations). In weather models, temperature (or potential temperature), absolute humidity, and radioactivity are also usually treated as tracers. According to AMS, "The main requirement for a tracer is that its lifetime be substantially longer than the transport process under study." diff --git a/doc/conf.py b/doc/conf.py index f1f094d545..c677204e70 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -308,6 +308,7 @@ def warn_undocumented_members(app, what, name, obj, options, lines): 'srw-repo': ('https://github.com/ufs-community/ufs-srweather-app/%s', '%s'), 'srw-wiki': ('https://github.com/ufs-community/ufs-srweather-app/wiki/%s','%s'), 'uw': ('https://uwtools.readthedocs.io/en/main/%s', '%s'), + 'fire-ug': ('https://mkavulich-fire-behavior.readthedocs.io/en/latest/%s', '%s'), } # -- Options for autoyaml extension --------------------------------------- @@ -316,4 +317,4 @@ def warn_undocumented_members(app, what, name, obj, options, lines): autoyaml_doc_delimiter = "###" # Character(s) which start a documentation comment. autoyaml_comment = "#" #Comment start character(s). autoyaml_level = 6 -#autoyaml_safe_loader = False \ No newline at end of file +#autoyaml_safe_loader = False From 610f04ce1b29f5499c5d791f25ac90ddae7a08ad Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 31 Oct 2024 23:33:32 -0600 Subject: [PATCH 59/87] Details about output; having trouble embedding image though --- .../BuildingRunningTesting/FIRE.rst | 73 ++++++++----------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index ec37390000..b34a584409 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -193,49 +193,36 @@ The workflow run is complete when all tasks display a "SUCCEEDED" message. If ev .. code-block:: console - CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION - ============================================================================================ - 202302170000 make_grid 47411619 SUCCEEDED 0 1 36.0 - 202302170000 make_orog 47411728 SUCCEEDED 0 1 151.0 - 202302170000 make_sfc_climo 47411801 SUCCEEDED 0 1 58.0 - 202302170000 nexus_gfs_sfc 47411620 SUCCEEDED 0 1 37.0 - 202302170000 nexus_emission_00 47411729 SUCCEEDED 0 1 251.0 - 202302170000 nexus_emission_01 47411730 SUCCEEDED 0 1 250.0 - 202302170000 nexus_emission_02 47411731 SUCCEEDED 0 1 250.0 - 202302170000 nexus_post_split 47412034 SUCCEEDED 0 1 44.0 - 202302170000 fire_emission 47411621 SUCCEEDED 0 1 19.0 - 202302170000 point_source 47411732 SUCCEEDED 0 1 82.0 - 202302170000 aqm_lbcs 47412961 SUCCEEDED 0 1 159.0 - 202302170000 get_extrn_ics 47411622 SUCCEEDED 0 1 314.0 - 202302170000 get_extrn_lbcs 47411623 SUCCEEDED 0 1 0.0 - 202302170000 make_ics_mem000 47659593 SUCCEEDED 0 1 126.0 - 202302170000 make_lbcs_mem000 47659594 SUCCEEDED 0 1 113.0 - 202302170000 run_fcst_mem000 47659742 SUCCEEDED 0 1 763.0 - 202302170000 run_post_mem000_f000 47659910 SUCCEEDED 0 1 30.0 - 202302170000 run_post_mem000_f001 47660029 SUCCEEDED 0 1 30.0 - 202302170000 run_post_mem000_f002 47660030 SUCCEEDED 0 1 31.0 - ... - 202302170000 run_post_mem000_f006 47660110 SUCCEEDED 0 1 29.0 - ============================================================================================ - 202302170600 nexus_gfs_sfc 47659421 SUCCEEDED 0 1 44.0 - 202302170600 nexus_emission_00 47659475 SUCCEEDED 0 1 323.0 - 202302170600 nexus_emission_01 47659476 SUCCEEDED 0 1 323.0 - 202302170600 nexus_emission_02 47659477 SUCCEEDED 0 1 329.0 - 202302170600 nexus_post_split 47659595 SUCCEEDED 0 1 60.0 - 202302170600 fire_emission 47659422 SUCCEEDED 0 1 18.0 - 202302170600 point_source 47659478 SUCCEEDED 0 1 128.0 - 202302170600 aqm_ics 47659597 SUCCEEDED 0 1 159.0 - 202302170600 aqm_lbcs 47659598 SUCCEEDED 0 1 158.0 - 202302170600 get_extrn_ics 47659423 SUCCEEDED 0 1 493.0 - 202302170600 get_extrn_lbcs 47659424 SUCCEEDED 0 1 536.0 - 202302170600 make_ics_mem000 47659594 SUCCEEDED 0 1 134.0 - 202302170600 make_lbcs_mem000 47659596 SUCCEEDED 0 1 112.0 - 202302170600 run_fcst_mem000 47659812 SUCCEEDED 0 1 1429.0 - 202302170600 run_post_mem000_f000 47659998 SUCCEEDED 0 1 30.0 - 202302170600 run_post_mem000_f001 47660042 SUCCEEDED 0 1 31.0 - 202302170600 run_post_mem000_f002 47660043 SUCCEEDED 0 1 29.0 - ... - 202302170600 run_post_mem000_f012 47660134 SUCCEEDED 0 1 30.0 + CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION + ================================================================================================================================ + 202008131800 make_grid 6498125 SUCCEEDED 0 1 70.0 + 202008131800 make_orog 6498145 SUCCEEDED 0 1 87.0 + 202008131800 make_sfc_climo 6498172 SUCCEEDED 0 1 90.0 + 202008131800 get_extrn_ics 6498126 SUCCEEDED 0 1 46.0 + 202008131800 get_extrn_lbcs 6498127 SUCCEEDED 0 1 46.0 + 202008131800 make_ics_mem000 6498202 SUCCEEDED 0 1 91.0 + 202008131800 make_lbcs_mem000 6498203 SUCCEEDED 0 1 106.0 + 202008131800 run_fcst_mem000 6498309 SUCCEEDED 0 1 1032.0 + 202008131800 run_post_mem000_f000 6498336 SUCCEEDED 0 1 75.0 + 202008131800 run_post_mem000_f001 6498387 SUCCEEDED 0 1 76.0 + 202008131800 run_post_mem000_f002 6498408 SUCCEEDED 0 1 75.0 + 202008131800 run_post_mem000_f003 6498409 SUCCEEDED 0 1 75.0 + 202008131800 run_post_mem000_f004 6498432 SUCCEEDED 0 1 64.0 + 202008131800 run_post_mem000_f005 6498433 SUCCEEDED 0 1 77.0 + 202008131800 run_post_mem000_f006 6498435 SUCCEEDED 0 1 74.0 + 202008131800 integration_test_mem000 6498434 SUCCEEDED 0 1 27.0 + +In addition to the standard UFS and UPP output described elsewhere in this users guide, the UFS_FIRE runs produce additional output files :numref:`described above `: + +.. code-block:: console + + $ cd /path/to/expt_dir/experiment + $ ls 2020081318/fire_output* + fire_output_2020-08-13_18:00:00.nc fire_output_2020-08-13_19:30:00.nc fire_output_2020-08-13_21:00:00.nc fire_output_2020-08-13_22:30:00.nc + fire_output_2020-08-13_18:30:00.nc fire_output_2020-08-13_20:00:00.nc fire_output_2020-08-13_21:30:00.nc fire_output_2020-08-13_23:00:00.nc + fire_output_2020-08-13_19:00:00.nc fire_output_2020-08-13_20:30:00.nc fire_output_2020-08-13_22:00:00.nc fire_output_2020-08-13_23:30:00.nc + + These files contain output directly from the fire model (hence why they are at a greater frequency), including variables such as the fire perimeter and area, smoke emitted, and fuel percentage burnt. .. _FIRE-WE2E: From 227edf61e878107d4bf1797366bc1e5fa2f875aa Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sun, 3 Nov 2024 19:17:12 -0700 Subject: [PATCH 60/87] Fix default number of cores; fire cores should not be multiplied by OMP --- ush/config_defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index f87e4e6ad8..44bd90598a 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1928,7 +1928,7 @@ task_run_fcst: QUILTING: true PRINT_ESMF: false - PE_MEMBER01: '{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + fire.FIRE_NUM_TASKS + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + fire.FIRE_NUM_TASKS) }}' + PE_MEMBER01: '{{ fire.FIRE_NUM_TASKS + OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X) + fire.FIRE_NUM_TASKS }}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" From 65ffd9f594b2fce77a6481d9b107886f00ef091b Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sun, 3 Nov 2024 19:17:58 -0700 Subject: [PATCH 61/87] Don't specify locations of data for WE2E test --- tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml b/tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml index 9ae35c21af..41061e6d4d 100644 --- a/tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml +++ b/tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml @@ -12,14 +12,12 @@ workflow: PREEXISTING_DIR_METHOD: rename task_get_extrn_ics: USE_USER_STAGED_EXTRN_FILES: true - EXTRN_MDL_SOURCE_BASEDIR_ICS: '{{ fire.FIRE_INPUT_DIR }}/HRRR' EXTRN_MDL_NAME_ICS: HRRR FV3GFS_FILE_FMT_ICS: grib2 EXTRN_MDL_FILES_ICS: - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true - EXTRN_MDL_SOURCE_BASEDIR_LBCS: '{{ fire.FIRE_INPUT_DIR }}/HRRR' EXTRN_MDL_FILES_LBCS: - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' EXTRN_MDL_NAME_LBCS: HRRR From 01aefef474ebdfc54e3f347fce637d438d685fe6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sun, 3 Nov 2024 19:18:22 -0700 Subject: [PATCH 62/87] Fix merge for ufs.configure file --- ush/create_ufs_configure_file.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index 712d004224..9117eca07e 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -100,15 +100,17 @@ def create_ufs_configure_file(run_dir,cfg): "logKindFlag": logkindflag, "EARTH_cl": earth_component_list, "ATM_pb": atm_petlist_bounds, - "AQM_pb": aqm_petlist_bounds, "ATM_omp_num_threads_line": atm_omp_num_threads_line, "ATM_diag_line": atm_diag_line, "runseq": runseq, + "AQM_pb": "", "FIRE_pb": "", "dt_atmos": cfg["DT_ATMOS"], "print_esmf": cfg["PRINT_ESMF"], "cpl_aqm": cfg["CPL_AQM"] } + if cfg["CPL_AQM"]: + settings["AQM_pb"] = aqm_petlist_bounds if cfg["UFS_FIRE"]: settings["FIRE_pb"] = fire_petlist_bounds From 061ffd503578e8566dbe0bce093d833da5a36ae9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 1 Nov 2024 16:13:36 -0600 Subject: [PATCH 63/87] Add broken image for demonstration --- doc/UsersGuide/BuildingRunningTesting/FIRE.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index b34a584409..95b939a775 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -224,6 +224,10 @@ In addition to the standard UFS and UPP output described elsewhere in this users These files contain output directly from the fire model (hence why they are at a greater frequency), including variables such as the fire perimeter and area, smoke emitted, and fuel percentage burnt. +.. image:: https://private-user-images.githubusercontent.com/12705538/382190299-de126b41-c4ed-4cd8-b392-1e75dcded686.png + :alt: Image of the simulated fire area from an example run + :align: center + .. _FIRE-WE2E: From 28c19e52e572e021c9aae062a18b0ae651688846 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 4 Nov 2024 08:27:07 -0700 Subject: [PATCH 64/87] Minor fixes to FIRE chapter: Remove irrelevant sentence about tasks, fix indentation, fix intra-chapter link --- doc/UsersGuide/BuildingRunningTesting/FIRE.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index 95b939a775..50d025da31 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -182,8 +182,6 @@ Users may check experiment status from the experiment directory with either of t # Check the experiment status and relaunch the workflow (for manual jobs) ./launch_FV3LAM_wflow.sh; tail -n 40 log.launch_FV3LAM_wflow -To see a description of each of the FIRE workflow tasks, see :numref:`Section %s `. - .. _FIRESuccess: Experiment Output @@ -212,7 +210,7 @@ The workflow run is complete when all tasks display a "SUCCEEDED" message. If ev 202008131800 run_post_mem000_f006 6498435 SUCCEEDED 0 1 74.0 202008131800 integration_test_mem000 6498434 SUCCEEDED 0 1 27.0 -In addition to the standard UFS and UPP output described elsewhere in this users guide, the UFS_FIRE runs produce additional output files :numref:`described above `: +In addition to the standard UFS and UPP output described elsewhere in this users guide, the UFS_FIRE runs produce additional output files :ref:`described above `: .. code-block:: console @@ -222,7 +220,7 @@ In addition to the standard UFS and UPP output described elsewhere in this users fire_output_2020-08-13_18:30:00.nc fire_output_2020-08-13_20:00:00.nc fire_output_2020-08-13_21:30:00.nc fire_output_2020-08-13_23:00:00.nc fire_output_2020-08-13_19:00:00.nc fire_output_2020-08-13_20:30:00.nc fire_output_2020-08-13_22:00:00.nc fire_output_2020-08-13_23:30:00.nc - These files contain output directly from the fire model (hence why they are at a greater frequency), including variables such as the fire perimeter and area, smoke emitted, and fuel percentage burnt. +These files contain output directly from the fire model (hence why they are at a greater frequency), including variables such as the fire perimeter and area, smoke emitted, and fuel percentage burnt. .. image:: https://private-user-images.githubusercontent.com/12705538/382190299-de126b41-c4ed-4cd8-b392-1e75dcded686.png :alt: Image of the simulated fire area from an example run From 124cebc1f80dbca97849b27fa858d447711a9097 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 4 Nov 2024 08:44:50 -0700 Subject: [PATCH 65/87] Fixed image of fire output --- doc/UsersGuide/BuildingRunningTesting/FIRE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index 50d025da31..4fc2f2e3ea 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -222,7 +222,7 @@ In addition to the standard UFS and UPP output described elsewhere in this users These files contain output directly from the fire model (hence why they are at a greater frequency), including variables such as the fire perimeter and area, smoke emitted, and fuel percentage burnt. -.. image:: https://private-user-images.githubusercontent.com/12705538/382190299-de126b41-c4ed-4cd8-b392-1e75dcded686.png +.. image:: https://github.com/ufs-community/ufs-srweather-app/wiki/FIRE/ncview.emis_smoke_trim.png :alt: Image of the simulated fire area from an example run :align: center From 3e5fa8fc27e9309364bfcff555b8bb4d070dea14 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Tue, 5 Nov 2024 11:04:45 -0700 Subject: [PATCH 66/87] Apply suggestions from code review Co-authored-by: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> --- doc/UsersGuide/BuildingRunningTesting/FIRE.rst | 6 +++--- ush/create_ufs_configure_file.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index 4fc2f2e3ea..92ef63d6fc 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -6,7 +6,7 @@ Community Fire Behavior Module (UFS FIRE) The `Community Fire Behavior Model (CFBM) `_ is a wildland fire model coupled to the UFS Atmospheric Model. The capability to run this code is now available in the UFS Short-Range Weather App for easy use by the community. The `fire_behavior repository `_ is a :term:`submodule` of the UFS Weather Model (WM), coupled through the :term:`NUOPC` Layer to provide direct feedback between the simulated atmosphere and the simulated fire. More information about the CFBM can be found in the :fire-ug:`CFBM Users Guide <>`. -The biggest difference between the UFS FIRE capability and other modes of the UFS SRW is that a special build flag is required to build the coupled fire behavior code, as described in the instructions below. Aside from that, additional input files, and some fire-specific config settings, building and running is the same as any other use of SRW. +The biggest difference between the UFS FIRE capability and other modes of the UFS SRW is that a special build flag is required to build the coupled fire behavior code, as described in the instructions below. Aside from that, the need for additional input files, and some fire-specific config settings, configuring and running an experiment is the same as any other use of SRW. .. note:: @@ -35,7 +35,7 @@ Users must run the ``checkout_externals`` script to collect (or "check out") the ./manage_externals/checkout_externals -Build the SRW App with Fire Behavior enabled +Build the SRW App with Fire Behavior Enabled -------------------------------------------- To build the SRW with fire behavior code, use the following command: @@ -51,7 +51,7 @@ If UFS FIRE builds correctly, users should see the standard executables listed i Load the |wflow_env| Environment -------------------------------------------- -Load the python environment for the workflow: +Load the appropriate modules for the workflow: .. code-block:: console diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index 9117eca07e..3d06d8b67b 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -125,8 +125,8 @@ def create_ufs_configure_file(run_dir,cfg): # #----------------------------------------------------------------------- # - # Call uwtools "render" set_template function from workflow_tools to fill in jinja template - # from config template and settings variable to create ufs.configure file for this experiment + # Call the uwtools "render" function to fill in jinja expressions contained in the UFS Configure file template + # with the values from the settings variable to create ufs.configure file for this experiment # #----------------------------------------------------------------------- # From ef727654ed03a449478186391e1282dc0cb809eb Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 5 Nov 2024 11:07:02 -0700 Subject: [PATCH 67/87] Make the linter happy; --- ush/create_ufs_configure_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index 3d06d8b67b..5bec56fb62 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -125,8 +125,9 @@ def create_ufs_configure_file(run_dir,cfg): # #----------------------------------------------------------------------- # - # Call the uwtools "render" function to fill in jinja expressions contained in the UFS Configure file template - # with the values from the settings variable to create ufs.configure file for this experiment + # Call the uwtools "render" function to fill in jinja expressions contained in the UFS Configure + # file template with the values from the settings variable to create ufs.configure file for this + # experiment # #----------------------------------------------------------------------- # From becf0e7aea67d365753fa74b8aa1b1e479328088 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 5 Nov 2024 11:07:36 -0700 Subject: [PATCH 68/87] Better description for -a build flag --- devbuild.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index 6918a1e229..78c0004511 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -15,8 +15,10 @@ OPTIONS compiler to use; default depends on platform (e.g. intel | gnu | cray | gccgfortran) -a, --app=APPLICATION - weather model application to build; for example, ATMAQ for Online-CMAQ - (e.g. ATM | ATMAQ | ATMF | ATMW | S2S | S2SW) + weather model application to build; supported SRW options are + ATM (default) Atmosphere only + ATMAQ Online-CMAQ (air quality) + ATMF UFS_FIRE (coupled Community Fire Behavior Model) --ccpp="CCPP_SUITE1,CCPP_SUITE2..." CCPP suites (CCPP_SUITES) to include in build; delimited with ',' --enable-options="OPTION1,OPTION2,..." From 7937dde28ff7f63b389b42f4ffa920d15f766b53 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 5 Nov 2024 11:11:53 -0700 Subject: [PATCH 69/87] Revert move of aqm test directory, make new "fire" test directory, rename fire test to indicate its one-way coupled --- .../config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml | 0 .../config.UFS_FIRE_one-way-coupled.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/WE2E/test_configs/{special_builds => aqm}/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml (100%) rename tests/WE2E/test_configs/{special_builds/config.UFS_FIRE.yaml => fire/config.UFS_FIRE_one-way-coupled.yaml} (100%) diff --git a/tests/WE2E/test_configs/special_builds/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml similarity index 100% rename from tests/WE2E/test_configs/special_builds/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml rename to tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml diff --git a/tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml similarity index 100% rename from tests/WE2E/test_configs/special_builds/config.UFS_FIRE.yaml rename to tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml From 85ca43eb8ee2939198c3281ec30bec774c73dafa Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 5 Nov 2024 11:19:17 -0700 Subject: [PATCH 70/87] Consistent order in PE_MEMBER01 definition --- ush/config_defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 44bd90598a..bd15083749 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1928,7 +1928,7 @@ task_run_fcst: QUILTING: true PRINT_ESMF: false - PE_MEMBER01: '{{ fire.FIRE_NUM_TASKS + OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X) + fire.FIRE_NUM_TASKS }}' + PE_MEMBER01: '{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) + fire.FIRE_NUM_TASKS if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X) + fire.FIRE_NUM_TASKS }}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" From cf429cb0a2ee5772a06a91e94913c8347be858d0 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 5 Nov 2024 12:20:46 -0700 Subject: [PATCH 71/87] Instead of a separate entry for all these duplicated namelist settings, make all the ignition-specific variables lists of length FIRE_NUM_IGNITIONS --- ush/config.fire.yaml | 16 ++--- ush/config_defaults.yaml | 109 +++++++++++++++++++++-------------- ush/generate_FV3LAM_wflow.py | 18 +++++- ush/setup.py | 22 ++++++- 4 files changed, 110 insertions(+), 55 deletions(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 68ec0188a7..1e476470a7 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -44,14 +44,14 @@ fire: DT_FIRE: 0.5 OUTPUT_DT_FIRE: 300 FIRE_NUM_IGNITIONS: 1 - FIRE_IGNITION_ROS1: 0.05 - FIRE_IGNITION_START_LAT1: 40.609 - FIRE_IGNITION_START_LON1: -105.879 - FIRE_IGNITION_END_LAT1: 40.609 - FIRE_IGNITION_END_LON1: -105.879 - FIRE_IGNITION_RADIUS1: 250 - FIRE_IGNITION_START_TIME1: 6480 - FIRE_IGNITION_END_TIME1: 7000 + FIRE_IGNITION_ROS: 0.05 + FIRE_IGNITION_START_LAT: 40.609 + FIRE_IGNITION_START_LON: -105.879 + FIRE_IGNITION_END_LAT: 40.609 + FIRE_IGNITION_END_LON: -105.879 + FIRE_IGNITION_RADIUS: 250 + FIRE_IGNITION_START_TIME: 6480 + FIRE_IGNITION_END_TIME: 7000 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 0 FIRE_ATM_FEEDBACK: 1.0 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index bd15083749..d298245f14 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2695,32 +2695,20 @@ fire: # OUTPUT_DT_FIRE # The fire behavior component’s output timestep # - # FIRE_NUM_IGNITIONS - # Number of fire ignitions - # - # FIRE_IGNITION_ROS1 - # - # - # FIRE_IGNITION_START_LAT1 - # - # - # FIRE_IGNITION_START_LON1 - # - # - # FIRE_IGNITION_END_LAT1 - # - # - # FIRE_IGNITION_END_LON1 - # - # - # FIRE_IGNITION_RADIUS1 - # - # - # FIRE_IGNITION_START_TIME1 - # - # - # FIRE_IGNITION_END_TIME1 + # FIRE_NUM_TASKS + # Number of MPI tasks assigned to the FIRE_BEHAVIOR component. # + + UFS_FIRE: False + FIRE_INPUT_DIR: "" + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 + FIRE_NUM_TASKS: 0 + + + # The following options control namelist values in the Community Fire + # Behavior Model. See the users guide for more information. + # ---------------------------------------------------------------------- # # FIRE_WIND_HEIGHT # Height to interpolate winds to for calculating fire spread rate @@ -2740,30 +2728,54 @@ fire: # FIRE_VISCOSITY # Artificial viscosity in level set method. Maximum 1, required for FIRE_UPWINDING=0 # - # FIRE_UPWINDING = 9 - # 0=NONE, 1=STANDARD, 2=GODUNOV, 3=ENO, 4=SETHIAN + # FIRE_UPWINDING + # Upwinding scheme used for calculating the normal spread of the fire front + # 0 = Central Difference + # 1 = Standard + # 2 = Godunov + # 3 = ENO1 + # 4 = Sethian + # 5 = 2nd-order Sethian + # 6 = WENO3 + # 7 = WENO5 + # 8 = Hybrid WENO3/ENO1 + # 9 = Hybrid WENO5/ENO1 # # FIRE_LSM_ZCOUPLING - # + # When true, uses FIRE_LSM_ZCOUPLING_REF instead of FIRE_WIND_HEIGHT as a reference height + # to calculate the logarithmic surface layer wind profile # # FIRE_LSM_ZCOUPLING_REF + # Reference height from which the velocity at FIRE_WIND_HEIGHT is calculated using a logarithmic profile + # + # FIRE_NUM_IGNITIONS + # Number of fire ignitions. + # + # NOTE: If FIRE_NUM_IGNITIONS > 1, the following variables should be lists with one entry for each ignition + # FIRE_IGNITION_ROS # - # FIRE_NUM_TASKS - # Number of MPI tasks assigned to the FIRE_BEHAVIOR component. + # + # FIRE_IGNITION_START_LAT + # + # + # FIRE_IGNITION_START_LON + # + # + # FIRE_IGNITION_END_LAT + # + # + # FIRE_IGNITION_END_LON + # + # + # FIRE_IGNITION_RADIUS + # + # + # FIRE_IGNITION_START_TIME + # + # + # FIRE_IGNITION_END_TIME + # - UFS_FIRE: False - FIRE_INPUT_DIR: "" - DT_FIRE: 0.5 - OUTPUT_DT_FIRE: 300 - FIRE_NUM_IGNITIONS: 1 - FIRE_IGNITION_ROS1: 0.05 - FIRE_IGNITION_START_LAT1: 40.609 - FIRE_IGNITION_START_LON1: -105.879 - FIRE_IGNITION_END_LAT1: 40.609 - FIRE_IGNITION_END_LON1: -105.879 - FIRE_IGNITION_RADIUS1: 250 - FIRE_IGNITION_START_TIME1: 6480 - FIRE_IGNITION_END_TIME1: 7000 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 0 FIRE_ATM_FEEDBACK: 0.0 @@ -2771,7 +2783,16 @@ fire: FIRE_UPWINDING: 9 FIRE_LSM_ZCOUPLING: False FIRE_LSM_ZCOUPLING_REF: 60.0 - FIRE_NUM_TASKS: 0 + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS: 0.05 + FIRE_IGNITION_START_LAT: 40.609 + FIRE_IGNITION_START_LON: -105.879 + FIRE_IGNITION_END_LAT: 40.609 + FIRE_IGNITION_END_LON: -105.879 + FIRE_IGNITION_RADIUS: 250 + FIRE_IGNITION_START_TIME: 6480 + FIRE_IGNITION_END_TIME: 7000 + rocoto: attrs: "" diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index e6f6869d1b..65d62cf39f 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -699,9 +699,17 @@ def generate_FV3LAM_wflow( fire_nml_dict['atm']['interval_atm'] = expt_config['task_run_fcst']['DT_ATMOS'] fire_nml_dict['atm']['kde'] = expt_config['task_make_ics']['LEVP'] # Fill in &fire and static &time variables + + # These settings must be handled specially below + each_ignit = ["FIRE_IGNITION_ROS", "FIRE_IGNITION_START_LAT", "FIRE_IGNITION_START_LON", + "FIRE_IGNITION_RADIUS", "FIRE_IGNITION_START_TIME", "FIRE_IGNITION_END_TIME"] + + # These settings do not get added to namelist, or are handled elsewhere + pass_settings = ["UFS_FIRE", "FIRE_INPUT_DIR", "FIRE_NUM_TASKS"] + pass_settings.extend(each_ignit) + for setting in expt_config['fire']: - # Would like to use pattern matching here but don't want to force Python 3.10 - if setting in ["UFS_FIRE", "FIRE_INPUT_DIR", "FIRE_NUM_TASKS"]: + if setting in pass_settings: pass elif setting == "DT_FIRE": fire_nml_dict['time']['dt'] = expt_config['fire'][setting] @@ -712,6 +720,12 @@ def generate_FV3LAM_wflow( # and enter into namelist.fire's &fire section fire_nml_dict['fire'][setting.lower()] = expt_config['fire'][setting] + # The variables specific to each ignition need special handling: SRW uses a list, but the + # fire model has these settings as separate namelist entries + for i in range(expt_config['fire']['FIRE_NUM_IGNITIONS'] - 1): + for setting in each_ignit: + fire_nml_dict['fire'][f"{setting.lower()}{i+1}"] = expt_config['fire'][setting][i] + realize( input_config=expt_config['workflow']['FIRE_NML_BASE_FP'], input_format="nml", diff --git a/ush/setup.py b/ush/setup.py index cef6b9edc8..888a725700 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1528,6 +1528,26 @@ def _dict_find(user_dict, substring): ) if fire_conf["FIRE_NUM_TASKS"] < 1: raise ValueError("FIRE_NUM_TASKS must be > 0 if UFS_FIRE is True") + elif fire_conf["FIRE_NUM_TASKS"] > 1: + raise ValueError("FIRE_NUM_TASKS > 1 not yet supported") + + if fire_conf["FIRE_NUM_IGNITIONS"] > 5: + raise ValueError(f"Only 5 or fewer fire ignitions supported") + + if fire_conf["FIRE_NUM_IGNITIONS"] > 1: + # These settings all need to be lists for multiple fire ignitions + each_fire = ["FIRE_IGNITION_ROS", "FIRE_IGNITION_START_LAT", "FIRE_IGNITION_START_LON", + "FIRE_IGNITION_END_LAT", "FIRE_IGNITION_END_LON", "FIRE_IGNITION_RADIUS", + "FIRE_IGNITION_START_TIME", "FIRE_IGNITION_END_TIME"] + for setting in each_fire: + if not isinstance(setting, list): + logging.critical(f"{fire_conf['FIRE_NUM_IGNITIONS']=}") + logging.critical(f"{fire_conf[setting]=}") + raise ValueError(f"For FIRE_NUM_IGNITIONS > 1, {setting} must be a list of the same length") + if len(setting) != fire_conf["FIRE_NUM_IGNITIONS"]: + logging.critical(f"{fire_conf['FIRE_NUM_IGNITIONS']=}") + logging.critical(f"{fire_conf[setting]=}") + raise ValueError(f"For FIRE_NUM_IGNITIONS > 1, {setting} must be a list of the same length") if fire_conf["FIRE_UPWINDING"] == 0 and fire_conf["FIRE_VISCOSITY"] == 0.0: raise ValueError("FIRE_VISCOSITY must be > 0.0 if FIRE_UPWINDING == 0") @@ -1538,7 +1558,7 @@ def _dict_find(user_dict, substring): # # ----------------------------------------------------------------------- # - # Generate var_defns.sh file in the EXPTDIR. This file contains all + # Generate var_defns.yaml file in the EXPTDIR. This file contains all # the user-specified settings from expt_config. # # ----------------------------------------------------------------------- From b51d1cdd5635b2702bf91a7a0f9631bf91cfbd82 Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr." Date: Wed, 6 Nov 2024 20:33:53 +0000 Subject: [PATCH 72/87] Update build_settings to include "Machine" value, add check for machine consistency in setup.py, convert some logging calls to specific logger object --- CMakeLists.txt | 2 +- devbuild.sh | 1 + sorc/build_settings_template.yaml | 1 + ush/setup.py | 31 +++++++++++++++++++------------ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c5e40a2392..cf8180097d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,7 +101,7 @@ SET(host_os "${osrel}") SET(abs_top_builddir "${CMAKE_CURRENT_BINARY_DIR}") SET(abs_top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}") SET(application "${APP}") - +SET(machine "${BUILD_MACHINE}") SET(CC_VERSION "${CMAKE_C_COMPILER}") diff --git a/devbuild.sh b/devbuild.sh index 78c0004511..332abb49b5 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -365,6 +365,7 @@ fi # cmake settings CMAKE_SETTINGS="\ + -DBUILD_MACHINE=${MACHINE}\ -DCMAKE_BUILD_TYPE=${BUILD_TYPE}\ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}\ -DCMAKE_INSTALL_BINDIR=${BIN_DIR}\ diff --git a/sorc/build_settings_template.yaml b/sorc/build_settings_template.yaml index 785257e154..792ee540c4 100644 --- a/sorc/build_settings_template.yaml +++ b/sorc/build_settings_template.yaml @@ -5,6 +5,7 @@ #--------- SRW_Version: @SRWA_VERSION@ SRW_hash: @GIT_HASH@ +Machine: @machine@ Configured_On: @CONFIG_DATE@ Host_System: @host_cpu@-@host_vendor@-@host_os@ Build_Directory: @abs_top_builddir@ diff --git a/ush/setup.py b/ush/setup.py index 888a725700..7ad5f05cc4 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -406,7 +406,14 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False): # Load build settings as a dictionary; will be used later to make sure the build is consistent with the user settings build_config = load_config_file(build_config_fp) - logger.debug(f"Build configuration\n{build_config}") + logger.debug(f"Read build configuration from {build_config_fp}\n{build_config}") + + # Fail if build machine and config machine are inconsistent + if build_config["Machine"].upper() != expt_config["user"]["MACHINE"]: + logger.critical("ERROR: Machine in build settings file != machine specified in config file") + logger.critical(f"build machine: {build_config['Machine']}") + logger.critical(f"config machine: {expt_config['user']['MACHINE']}") + raise ValueError("Check config settings for correct value for 'machine'") # Set up some paths relative to the SRW clone expt_config["user"].update(set_srw_paths(USHdir, expt_config)) @@ -1485,8 +1492,8 @@ def _dict_find(user_dict, substring): if workflow_config["SDF_USES_THOMPSON_MP"]: - logging.debug(f'Selected CCPP suite ({workflow_config["CCPP_PHYS_SUITE"]}) uses Thompson MP') - logging.debug(f'Setting up links for additional fix files') + logger.debug(f'Selected CCPP suite ({workflow_config["CCPP_PHYS_SUITE"]}) uses Thompson MP') + logger.debug(f'Setting up links for additional fix files') # If the model ICs or BCs are not from RAP or HRRR, they will not contain aerosol # climatology data needed by the Thompson scheme, so we need to provide a separate file @@ -1502,8 +1509,8 @@ def _dict_find(user_dict, substring): for fix_file in fixed_files["THOMPSON_FIX_FILES"]: fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"].append(f"{fix_file} | {fix_file}") - logging.debug(f'New fix file list:\n{fixed_files["FIXgsm_FILES_TO_COPY_TO_FIXam"]=}') - logging.debug(f'New fix file mapping:\n{fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"]=}') + logger.debug(f'New fix file list:\n{fixed_files["FIXgsm_FILES_TO_COPY_TO_FIXam"]=}') + logger.debug(f'New fix file mapping:\n{fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"]=}') # ----------------------------------------------------------------------- @@ -1540,20 +1547,20 @@ def _dict_find(user_dict, substring): "FIRE_IGNITION_END_LAT", "FIRE_IGNITION_END_LON", "FIRE_IGNITION_RADIUS", "FIRE_IGNITION_START_TIME", "FIRE_IGNITION_END_TIME"] for setting in each_fire: - if not isinstance(setting, list): - logging.critical(f"{fire_conf['FIRE_NUM_IGNITIONS']=}") - logging.critical(f"{fire_conf[setting]=}") + if not isinstance(fire_conf[setting], list): + logger.critical(f"{fire_conf['FIRE_NUM_IGNITIONS']=}") + logger.critical(f"{fire_conf[setting]=}") raise ValueError(f"For FIRE_NUM_IGNITIONS > 1, {setting} must be a list of the same length") - if len(setting) != fire_conf["FIRE_NUM_IGNITIONS"]: - logging.critical(f"{fire_conf['FIRE_NUM_IGNITIONS']=}") - logging.critical(f"{fire_conf[setting]=}") + if len(fire_conf[setting]) != fire_conf["FIRE_NUM_IGNITIONS"]: + logger.critical(f"{fire_conf['FIRE_NUM_IGNITIONS']=}") + logger.critical(f"{fire_conf[setting]=}") raise ValueError(f"For FIRE_NUM_IGNITIONS > 1, {setting} must be a list of the same length") if fire_conf["FIRE_UPWINDING"] == 0 and fire_conf["FIRE_VISCOSITY"] == 0.0: raise ValueError("FIRE_VISCOSITY must be > 0.0 if FIRE_UPWINDING == 0") else: if fire_conf["FIRE_NUM_TASKS"] < 1: - logging.warning("UFS_FIRE is not enabled; setting FIRE_NUM_TASKS = 0") + logger.warning("UFS_FIRE is not enabled; setting FIRE_NUM_TASKS = 0") # # ----------------------------------------------------------------------- From 6029253a60acd4f12c1c3eb5dd32af574a61070b Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr." Date: Wed, 6 Nov 2024 20:35:09 +0000 Subject: [PATCH 73/87] - Update fire test for new setting names; add new multi-fire test - Fix missing values for fire namelist logic in workflow generation --- ...ig.UFS_FIRE_multifire_one-way-coupled.yaml | 76 +++++++++++++++++++ .../fire/config.UFS_FIRE_one-way-coupled.yaml | 18 ++--- ush/generate_FV3LAM_wflow.py | 10 ++- 3 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml new file mode 100644 index 0000000000..3c24404f82 --- /dev/null +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml @@ -0,0 +1,76 @@ +metadata: + description: >- + Tests the UFS_FIRE capability for a single fire with one-way (ATM-->FIRE) coupling +user: + RUN_ENVIR: community +workflow: + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: SUBCONUS_CO_3km + DATE_FIRST_CYCL: '2020081318' + DATE_LAST_CYCL: '2020081318' + FCST_LEN_HRS: 6 # 27 hours of LBCs staged on Derecho + PREEXISTING_DIR_METHOD: rename +task_get_extrn_ics: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_NAME_ICS: HRRR + FV3GFS_FILE_FMT_ICS: grib2 + EXTRN_MDL_FILES_ICS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' +task_get_extrn_lbcs: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_FILES_LBCS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' + EXTRN_MDL_NAME_LBCS: HRRR + EXTRN_MDL_LBCS_OFFSET_HRS: 0 + LBC_SPEC_INTVL_HRS: 3 + FV3GFS_FILE_FMT_LBCS: grib2 +task_run_fcst: + QUILTING: true + PRINT_ESMF: True + OMP_NUM_THREADS_RUN_FCST: 1 +task_plot_allvars: + COMOUT_REF: "" +fire: + UFS_FIRE: True + FIRE_NUM_TASKS: 1 + FIRE_INPUT_DIR: /glade/derecho/scratch/kavulich/FIRE/2024/test_data + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 + FIRE_NUM_IGNITIONS: 2 + FIRE_IGNITION_ROS: + - 0.05 + - 0.1 + FIRE_IGNITION_START_LAT: + - 40.609 + - 40.666 + FIRE_IGNITION_START_LON: + - -105.879 + - -105.95 + FIRE_IGNITION_END_LAT: + - 40.609 + - 40.678 + FIRE_IGNITION_END_LON: + - -105.879 + - -105.94 + FIRE_IGNITION_RADIUS: + - 250 + - 100 + FIRE_IGNITION_START_TIME: + - 6480 + - 3600 + FIRE_IGNITION_END_TIME: + - 7000 + - 3720 + FIRE_WIND_HEIGHT: 5.0 + FIRE_PRINT_MSG: 0 + FIRE_ATM_FEEDBACK: 1.0 + FIRE_VISCOSITY: 0.4 + FIRE_UPWINDING: 9 + FIRE_LSM_ZCOUPLING: False + FIRE_LSM_ZCOUPLING_REF: 60.0 +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 01:00:00 + diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml index 41061e6d4d..0d0de9fb48 100644 --- a/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml @@ -1,6 +1,6 @@ metadata: description: >- - Tests the UFS_FIRE capability + Tests the UFS_FIRE capability for a single fire with one-way (ATM-->FIRE) coupling user: RUN_ENVIR: community workflow: @@ -37,14 +37,14 @@ fire: DT_FIRE: 0.5 OUTPUT_DT_FIRE: 300 FIRE_NUM_IGNITIONS: 1 - FIRE_IGNITION_ROS1: 0.05 - FIRE_IGNITION_START_LAT1: 40.609 - FIRE_IGNITION_START_LON1: -105.879 - FIRE_IGNITION_END_LAT1: 40.609 - FIRE_IGNITION_END_LON1: -105.879 - FIRE_IGNITION_RADIUS1: 250 - FIRE_IGNITION_START_TIME1: 6480 - FIRE_IGNITION_END_TIME1: 7000 + FIRE_IGNITION_ROS: 0.05 + FIRE_IGNITION_START_LAT: 40.609 + FIRE_IGNITION_START_LON: -105.879 + FIRE_IGNITION_END_LAT: 40.609 + FIRE_IGNITION_END_LON: -105.879 + FIRE_IGNITION_RADIUS: 250 + FIRE_IGNITION_START_TIME: 6480 + FIRE_IGNITION_END_TIME: 7000 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 0 FIRE_ATM_FEEDBACK: 1.0 diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 65d62cf39f..1c202fec46 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -691,6 +691,7 @@ def generate_FV3LAM_wflow( #----------------------------------------------------------------------- # if expt_config['fire'].get('UFS_FIRE'): + logging.debug("Setting fire namelist values") fire_nml_dict = {} fire_nml_dict['atm'] = {} fire_nml_dict['time'] = {} @@ -702,7 +703,8 @@ def generate_FV3LAM_wflow( # These settings must be handled specially below each_ignit = ["FIRE_IGNITION_ROS", "FIRE_IGNITION_START_LAT", "FIRE_IGNITION_START_LON", - "FIRE_IGNITION_RADIUS", "FIRE_IGNITION_START_TIME", "FIRE_IGNITION_END_TIME"] + "FIRE_IGNITION_RADIUS", "FIRE_IGNITION_START_TIME", "FIRE_IGNITION_END_TIME", + "FIRE_IGNITION_END_LAT", "FIRE_IGNITION_END_LON"] # These settings do not get added to namelist, or are handled elsewhere pass_settings = ["UFS_FIRE", "FIRE_INPUT_DIR", "FIRE_NUM_TASKS"] @@ -722,9 +724,11 @@ def generate_FV3LAM_wflow( # The variables specific to each ignition need special handling: SRW uses a list, but the # fire model has these settings as separate namelist entries - for i in range(expt_config['fire']['FIRE_NUM_IGNITIONS'] - 1): + for i in range(expt_config['fire']['FIRE_NUM_IGNITIONS']): for setting in each_ignit: - fire_nml_dict['fire'][f"{setting.lower()}{i+1}"] = expt_config['fire'][setting][i] + nmle = f"{setting.lower()}{i+1}" + fire_nml_dict['fire'][nmle] = expt_config['fire'][setting][i] + print(f"{fire_nml_dict['fire'][nmle]=}") realize( input_config=expt_config['workflow']['FIRE_NML_BASE_FP'], From d4043bd2e36748d37ee9883e990f3271d2241f61 Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr." Date: Thu, 7 Nov 2024 00:10:42 +0000 Subject: [PATCH 74/87] Need to convert fire namelist settings to a list if not provided as a list --- ush/generate_FV3LAM_wflow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 1c202fec46..b825454db4 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -726,6 +726,10 @@ def generate_FV3LAM_wflow( # fire model has these settings as separate namelist entries for i in range(expt_config['fire']['FIRE_NUM_IGNITIONS']): for setting in each_ignit: + # If not a list, convert to a 1-element list + if not isinstance(expt_config['fire'][setting], list): + expt_config['fire'][setting] = [ expt_config['fire'][setting] ] + nmle = f"{setting.lower()}{i+1}" fire_nml_dict['fire'][nmle] = expt_config['fire'][setting][i] print(f"{fire_nml_dict['fire'][nmle]=}") From 5aa39cdff0faace7853c54c73229c15c5ecc411e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 6 Nov 2024 18:07:52 -0700 Subject: [PATCH 75/87] Update fire instructions in users guide for latest commits; add descriptions of config options --- .../BuildingRunningTesting/FIRE.rst | 24 ++-- .../CustomizingTheWorkflow/ConfigWorkflow.rst | 111 ++++++++++++++++++ 2 files changed, 124 insertions(+), 11 deletions(-) diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index 92ef63d6fc..114d1f6bc5 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -118,12 +118,12 @@ Specifying a fire ignition The CFBM simulates fires by specifying an "ignition" that will then propogate based on the atmospheric conditions and the specified settings. An ignition can either be a "point ignition" (i.e. a disk of fire some specified radius around a single location), or a straight line linear ignition specified by a start and end location and a specified "radius" (width). The ignition can start at the beginning of your simulation, or at some time later as specified. The CFBM can support up to 5 different fire ignitions at different places and times in a given simulation. -The CFBM settings are controlled by the :term:`namelist` file ``namelist.fire``. The available settings in this file are described in the :fire-ug:`CFBM Users Guide `, and an example file can be found under ``parm/namelist.fire``. However, there is no need to manually provide or edit this file, as the SRW workflow will create the fire namelist using the user settings in ``config.yaml``. +The CFBM settings are controlled by the :term:`namelist` file ``namelist.fire``. The available settings in this file are described in the :fire-ug:`CFBM Users Guide `, and an example file can be found under ``parm/namelist.fire``. However, there is no need to manually provide or edit this file, as the SRW workflow will create the fire namelist using the user settings in ``config.yaml``. The fire-specific options in SRW are documented in :numref:`Section %s `. Example fire configuration --------------------------- -Here is one example of settings that can be specified for a UFS FIRE simulation. +Here is one example of settings that can be specified for a UFS FIRE simulation: .. code-block:: console @@ -133,16 +133,16 @@ Here is one example of settings that can be specified for a UFS FIRE simulation. DT_FIRE: 0.5 OUTPUT_DT_FIRE: 1800 FIRE_NUM_IGNITIONS: 1 - FIRE_IGNITION_ROS1: 0.05 - FIRE_IGNITION_START_LAT1: 40.609 - FIRE_IGNITION_START_LON1: -105.879 - FIRE_IGNITION_END_LAT1: 40.609 - FIRE_IGNITION_END_LON1: -105.879 - FIRE_IGNITION_RADIUS1: 250 - FIRE_IGNITION_START_TIME1: 6480 - FIRE_IGNITION_END_TIME1: 7000 + FIRE_IGNITION_ROS: 0.05 + FIRE_IGNITION_START_LAT: 40.609 + FIRE_IGNITION_START_LON: -105.879 + FIRE_IGNITION_END_LAT: 40.609 + FIRE_IGNITION_END_LON: -105.879 + FIRE_IGNITION_RADIUS: 250 + FIRE_IGNITION_START_TIME: 6480 + FIRE_IGNITION_END_TIME: 7000 -In this case, a single fire (``FIRE_NUM_IGNITIONS: 1``) of radius 250 meters (``FIRE_IGNITION_RADIUS1: 250``) is ignited at latitude 40.609˚N (``FIRE_IGNITION_START_LAT1: 40.609``), 105.879˚W (``FIRE_IGNITION_START_LON1: -105.879``) 6480 seconds after the start of the simulation (``FIRE_IGNITION_START_TIME1: 6480``) with a rate of spread specified as 0.05 m/s (``FIRE_IGNITION_ROS1: 0.05``). This "ignition" ends 7000 seconds after the start of the simulation (``FIRE_IGNITION_END_TIME1: 7000``), after which the fire behavior is completely governed by the physics of the fire behavior model (integrated every 0.5 seconds as specified by ``OUTPUT_DT_FIRE``), the input fuel conditions, and the simulated atmospheric conditions. +In this case, a single fire (``FIRE_NUM_IGNITIONS: 1``) of radius 250 meters (``FIRE_IGNITION_RADIUS: 250``) is ignited at latitude 40.609˚N (``FIRE_IGNITION_START_LAT: 40.609``), 105.879˚W (``FIRE_IGNITION_START_LON: -105.879``) 6480 seconds after the start of the simulation (``FIRE_IGNITION_START_TIME: 6480``) with a rate of spread specified as 0.05 m/s (``FIRE_IGNITION_ROS: 0.05``). This "ignition" ends 7000 seconds after the start of the simulation (``FIRE_IGNITION_END_TIME: 7000``), after which the fire behavior is completely governed by the physics of the fire behavior model (integrated every 0.5 seconds as specified by ``OUTPUT_DT_FIRE``), the input fuel conditions, and the simulated atmospheric conditions. The CFBM creates output files in :term:`netCDF` format, with the naming scheme ``fire_output_YYYY-MM-DD_hh:mm:ss.nc``. In this case the output files are written every 30 minutes (``OUTPUT_DT_FIRE: 1800``). @@ -150,6 +150,8 @@ The CFBM creates output files in :term:`netCDF` format, with the naming scheme ` Any of the settings under :fire-ug:`the ``&fire`` section of the namelist ` can be specified in the SRW App ``config.yaml`` file under the ``fire:`` section, not just the settings described above. However, any additional settings from ``namelist.fire`` will need to be added to ``config_defaults.yaml`` first; otherwise the check for valid SRW options will fail. +To specify multiple fire ignitions (``FIRE_NUM_IGNITIONS > 1``), the above settings will need to be specified as a list, with one entry per ignition. See :numref:`Section %s ` for more details. + Generate the Workflow ------------------------ diff --git a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst index 31cee09eeb..832581767e 100644 --- a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1807,6 +1807,117 @@ Non-default parameters for coupled Air Quality Modeling (AQM) tasks are set in t ``NEXUS_GFS_SFC_ARCHV_DIR``: (Default: "/NCEPPROD/hpssprod/runhistory") Path to archive directory for gfs surface files on HPSS. +.. _fire-parameters: + +Community Fire Behavior Model Parameters +======================================== + +Non-default parameters for the Community Fire Behavior Model (CFBM) in SRW are set in the ``fire:`` section of the ``config.yaml`` file. + +``UFS_FIRE``: (Default: false) + Flag for turning on CFBM fire simulation + +``FIRE_INPUT_DIR``: (Default: "") + Directory where fire input file (geo_em.d01.nc) can be found + +``DT_FIRE``: (Default: 0.5) + The fire behavior component’s integration timestep in seconds + +``OUTPUT_DT_FIRE``: (Default: 300) + The fire behavior component’s output timestep in seconds + +``FIRE_NUM_TASKS``: (Default: 0) + Number of MPI tasks assigned to the FIRE_BEHAVIOR component. Currently only 1 task is supported. + +.. note:: + The following options control namelist values in the ``&fire`` section of the Community Fire + Behavior Model. See the :fire-ug:`CFBM Users Guide ` for more information. + +``FIRE_PRINT_MSG``: (Default: 0) + Debug print level for the weather model fire core. Levels greater than 1 will print extra + messages to the log file at run time. + + 0: no extra prints + + 1: Extra prints + + 2: More extra prints + + 3: Even more extra prints + +``FIRE_WIND_HEIGHT``: (Default: 5.0) + Height to interpolate winds to for calculating fire spread rate + +``FIRE_ATM_FEEDBACK``: (Default: 0.0) + Multiplier for heat fluxes. Use 1.0 for normal two-way coupling. Use 0.0 for one-way coupling. + Intermediate values will vary the amount of forcing provided from the fire to the dynamical core. + +``FIRE_VISCOSITY``: (Default: 0.4) + Artificial viscosity in level set method. Maximum value of 1. Required for ``FIRE_UPWINDING=0`` + +``FIRE_UPWINDING``: (Default: 9) + Upwinding scheme used for calculating the normal spread of the fire front. More detailed descriptions + of these options can be found in the :fire-ug:`CFBM Users Guide `. + + 0 = Central Difference + + 1 = Standard + + 2 = Godunov + + 3 = ENO1 + + 4 = Sethian + + 5 = 2nd-order Sethian + + 6 = WENO3 + + 7 = WENO5 + + 8 = Hybrid WENO3/ENO1 + + 9 = Hybrid WENO5/ENO1 + +``FIRE_LSM_ZCOUPLING`` (Default: false) + When true, uses ``FIRE_LSM_ZCOUPLING_REF`` instead of ``FIRE_WIND_HEIGHT`` as a reference height + to calculate the logarithmic surface layer wind profile + +``FIRE_LSM_ZCOUPLING_REF`` (Default: 60.0) + Reference height from which the velocity at FIRE_WIND_HEIGHT is calculated using a logarithmic profile + + +``FIRE_NUM_IGNITIONS`` (Default: 1) + Number of fire ignitions. + +.. note:: + If ``FIRE_NUM_IGNITIONS > 1``, the following variables should be lists with one entry for each ignition + +``FIRE_IGNITION_ROS`` (Default: 0.0) + Ignition rate of spread (Rothermel parameterization) + +``FIRE_IGNITION_START_LAT`` (Default: 40.609) + Latitude for start of ignition(s) + +``FIRE_IGNITION_START_LON`` (Default: -105.879) + Longitude for start of ignition(s) + +``FIRE_IGNITION_END_LAT`` (Default: 40.609) + Latitude for end of ignition(s) + +``FIRE_IGNITION_END_LON`` (Default: -105.879) + Longitude for end of ignition(s) + +``FIRE_IGNITION_RADIUS`` (Default: 250) + Radius of ignition area in meters + +``FIRE_IGNITION_START_TIME`` (Default: 6480) + Start time of ignition(s) in seconds (counting from the beginning of the simulation) + +``FIRE_IGNITION_START_TIME`` (Default: 7000) + End time of ignition(s) in seconds (counting from the beginning of the simulation) + + Rocoto Parameters =================== From f44556621acefe78d88600b2b8da7f057bcc5206 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 6 Nov 2024 18:08:16 -0700 Subject: [PATCH 76/87] Initial commit of fire capability does not support two-way coupling --- ush/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/setup.py b/ush/setup.py index 7ad5f05cc4..0f2af305ec 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1556,6 +1556,9 @@ def _dict_find(user_dict, substring): logger.critical(f"{fire_conf[setting]=}") raise ValueError(f"For FIRE_NUM_IGNITIONS > 1, {setting} must be a list of the same length") + if fire_conf["FIRE_ATM_FEEDBACK"] > 0.0: + raise ValueError("FIRE_ATM_FEEDBACK > 0 (two-way coupling) not supported in UFS yet") + if fire_conf["FIRE_UPWINDING"] == 0 and fire_conf["FIRE_VISCOSITY"] == 0.0: raise ValueError("FIRE_VISCOSITY must be > 0.0 if FIRE_UPWINDING == 0") else: From c77770f3235511e480364c4f20558fe30878c9f7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 6 Nov 2024 18:14:54 -0700 Subject: [PATCH 77/87] Add LEVP setting documentation --- .../CustomizingTheWorkflow/ConfigWorkflow.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst index 832581767e..8b9e7648a7 100644 --- a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1027,9 +1027,15 @@ FVCOM Parameters ``FVCOM_FILE``: (Default: "fvcom.nc") Name of the file located in ``FVCOM_DIR`` that has :term:`FVCOM` data interpolated to the FV3-LAM grid. This file will be copied later to a new location, and the name will be changed to ``fvcom.nc`` if a name other than ``fvcom.nc`` is selected. -Vertical Coordinate File Parameter +Vertical Coordinate Parameters ------------------------------------ +``LEVP``: (Default: 65) + Number of vertical levels in the atmosphere. In order to change this + number, the user will additionally need to create a vertical coordinate + distribution file; this process is described in :numref:`Section %s ` + This value should be the same in both ``make_ics`` and ``make_lbcs``. + ``VCOORD_FILE``: (Default: ``"{{ workflow.FIXam }}/global_hyblev.l65.txt"``) Full path to the file used to set the vertical coordinates in FV3. This file should be the same in both ``make_ics`` and ``make_lbcs``. @@ -1047,9 +1053,15 @@ Non-default parameters for the ``make_lbcs`` task are set in the ``task_make_lbc ``OMP_STACKSIZE_MAKE_LBCS``: (Default: "1024m") Controls the size of the stack for threads created by the OpenMP implementation. -Vertical Coordinate File Parameter +Vertical Coordinate Parameters ------------------------------------ +``LEVP``: (Default: 65) + Number of vertical levels in the atmosphere. In order to change this + number, the user will additionally need to create a vertical coordinate + distribution file; this process is described in :numref:`Section %s ` + This value should be the same in both ``make_ics`` and ``make_lbcs``. + ``VCOORD_FILE``: (Default: ``"{{ workflow.FIXam }}/global_hyblev.l65.txt"``) Full path to the file used to set the vertical coordinates in FV3. This file should be the same in both ``make_ics`` and ``make_lbcs``. From 49ee2c634772df0e5634600611461162512529f9 Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr." Date: Thu, 7 Nov 2024 01:53:42 +0000 Subject: [PATCH 78/87] Remove default time values from namelist.fire --- parm/namelist.fire | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/parm/namelist.fire b/parm/namelist.fire index 48f0482c6c..4ef039996b 100644 --- a/parm/namelist.fire +++ b/parm/namelist.fire @@ -1,16 +1,4 @@ &time - start_year = 2020 - start_month = 08 - start_day = 13 - start_hour = 18 - start_minute = 00 - start_second = 00 - end_year = 2020 - end_month = 08 - end_day = 14 - end_hour = 21 - end_minute = 00 - end_second = 00 dt = 0.5 interval_output = 300 / From feed835025ddd69a0733775e2214b0108634cbea Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr." Date: Thu, 7 Nov 2024 01:53:58 +0000 Subject: [PATCH 79/87] Fix generate workflow unit test --- tests/test_python/test_generate_FV3LAM_wflow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_python/test_generate_FV3LAM_wflow.py b/tests/test_python/test_generate_FV3LAM_wflow.py index 4815265ff1..aa4e038f7a 100644 --- a/tests/test_python/test_generate_FV3LAM_wflow.py +++ b/tests/test_python/test_generate_FV3LAM_wflow.py @@ -39,12 +39,13 @@ def run_workflow(USHdir, logfile): logfile = "log.generate_FV3LAM_wflow" sed = get_env_var("SED") - # create a dummy build settings file if needed + # create a build settings file if needed EXECdir = os.path.join(USHdir, "..", "exec") build_settings_file = os.path.join(EXECdir, "build_settings.yaml") if not os.path.exists(build_settings_file): - os.makedirs(EXECdir) + os.makedirs(EXECdir, exist_ok=True) with open(build_settings_file, 'w', encoding='utf-8') as build_settings: + build_settings.write('Machine: linux\n') build_settings.write('Application:\n') # community test case From f1b721b62e709ff59675891956eeba6029225b34 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 7 Nov 2024 07:54:49 -0700 Subject: [PATCH 80/87] Fix test description and running instructions --- doc/UsersGuide/BuildingRunningTesting/FIRE.rst | 13 ++++++++++--- .../config.UFS_FIRE_multifire_one-way-coupled.yaml | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index 114d1f6bc5..8f92012a65 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -231,7 +231,7 @@ These files contain output directly from the fire model (hence why they are at a .. _FIRE-WE2E: -WE2E Test for FIRE +WE2E Tests for FIRE ======================= Build the app for FIRE: @@ -241,10 +241,17 @@ Build the app for FIRE: ./devbuild.sh -p=hera -a=ATMF -Run the WE2E test: +Run the WE2E tests: .. code-block:: console $ cd /path/to/ufs-srweather-app/tests/WE2E - $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q -t UFS_FIRE + $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q -t fire + +You can also run each test individually if needed: + + $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q -t UFS_FIRE_one-way-coupled + $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q -t UFS_FIRE_multifire_one-way-coupled + + diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml index 3c24404f82..6c23c61c0a 100644 --- a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml @@ -1,6 +1,6 @@ metadata: description: >- - Tests the UFS_FIRE capability for a single fire with one-way (ATM-->FIRE) coupling + Tests the UFS_FIRE capability for multiple fire ignitions with one-way (ATM-->FIRE) coupling user: RUN_ENVIR: community workflow: From 824054986af2d81c72c06bc2abb153ad1b1b40e6 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Thu, 7 Nov 2024 10:25:55 -0700 Subject: [PATCH 81/87] Update config.fire.yaml Initial capability needs FIRE_ATM_FEEDBACK=0.0 --- ush/config.fire.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml index 1e476470a7..fc986f6a6a 100644 --- a/ush/config.fire.yaml +++ b/ush/config.fire.yaml @@ -54,7 +54,7 @@ fire: FIRE_IGNITION_END_TIME: 7000 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 0 - FIRE_ATM_FEEDBACK: 1.0 + FIRE_ATM_FEEDBACK: 0.0 FIRE_VISCOSITY: 0.4 FIRE_UPWINDING: 9 FIRE_LSM_ZCOUPLING: False From a12471435c0cccd21b2d76f1fd9bb4b05f35646c Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Thu, 7 Nov 2024 10:30:15 -0700 Subject: [PATCH 82/87] Update config.UFS_FIRE_multifire_one-way-coupled.yaml FIRE_ATM_FEEDBACK = 0.0 --- .../fire/config.UFS_FIRE_multifire_one-way-coupled.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml index 6c23c61c0a..99da59d4f2 100644 --- a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml @@ -63,7 +63,7 @@ fire: - 3720 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 0 - FIRE_ATM_FEEDBACK: 1.0 + FIRE_ATM_FEEDBACK: 0.0 FIRE_VISCOSITY: 0.4 FIRE_UPWINDING: 9 FIRE_LSM_ZCOUPLING: False From e67173fb206c3d38bf78318b8eebfa5c4d0be035 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Thu, 7 Nov 2024 10:30:59 -0700 Subject: [PATCH 83/87] Update config.UFS_FIRE_one-way-coupled.yaml FIRE_ATM_FEEDBACK = 0.0 --- .../WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml index 0d0de9fb48..377f8176fc 100644 --- a/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml @@ -47,7 +47,7 @@ fire: FIRE_IGNITION_END_TIME: 7000 FIRE_WIND_HEIGHT: 5.0 FIRE_PRINT_MSG: 0 - FIRE_ATM_FEEDBACK: 1.0 + FIRE_ATM_FEEDBACK: 0.0 FIRE_VISCOSITY: 0.4 FIRE_UPWINDING: 9 FIRE_LSM_ZCOUPLING: False From 966326ecba5d8b3db23f0f00b41282f2a6b5ef78 Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr." Date: Fri, 8 Nov 2024 16:20:24 +0000 Subject: [PATCH 84/87] Point to official CFBM users guide location now that its published --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index c677204e70..c77e8ab113 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -308,7 +308,7 @@ def warn_undocumented_members(app, what, name, obj, options, lines): 'srw-repo': ('https://github.com/ufs-community/ufs-srweather-app/%s', '%s'), 'srw-wiki': ('https://github.com/ufs-community/ufs-srweather-app/wiki/%s','%s'), 'uw': ('https://uwtools.readthedocs.io/en/main/%s', '%s'), - 'fire-ug': ('https://mkavulich-fire-behavior.readthedocs.io/en/latest/%s', '%s'), + 'fire-ug': ('https://fire-behavior.readthedocs.io/en/latest/%s', '%s'), } # -- Options for autoyaml extension --------------------------------------- From a12b8e126745593cbaa3f8231e033614f2e47064 Mon Sep 17 00:00:00 2001 From: "Michael J. Kavulich, Jr." Date: Fri, 8 Nov 2024 20:17:20 +0000 Subject: [PATCH 85/87] Final updates for fire tests - New variable to WE2E test data, we should use this to consolidate test variables in the future - Shorten fire tests to 3 hours - Fix extra prints in workflow generation --- .../fire/config.UFS_FIRE_multifire_one-way-coupled.yaml | 4 ++-- .../test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml | 4 ++-- ush/generate_FV3LAM_wflow.py | 1 - ush/machine/derecho.yaml | 1 + ush/machine/gaea.yaml | 1 + ush/machine/hera.yaml | 1 + ush/machine/hercules.yaml | 1 + ush/machine/jet.yaml | 1 + ush/machine/noaacloud.yaml | 1 + ush/machine/odin.yaml | 1 + ush/machine/orion.yaml | 1 + ush/machine/stampede.yaml | 1 + ush/machine/wcoss2.yaml | 1 + 13 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml index 99da59d4f2..6bd217914f 100644 --- a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml @@ -8,7 +8,7 @@ workflow: PREDEF_GRID_NAME: SUBCONUS_CO_3km DATE_FIRST_CYCL: '2020081318' DATE_LAST_CYCL: '2020081318' - FCST_LEN_HRS: 6 # 27 hours of LBCs staged on Derecho + FCST_LEN_HRS: 3 # 27 hours of LBCs staged on Derecho PREEXISTING_DIR_METHOD: rename task_get_extrn_ics: USE_USER_STAGED_EXTRN_FILES: true @@ -33,7 +33,7 @@ task_plot_allvars: fire: UFS_FIRE: True FIRE_NUM_TASKS: 1 - FIRE_INPUT_DIR: /glade/derecho/scratch/kavulich/FIRE/2024/test_data + FIRE_INPUT_DIR: '{{ platform.WE2E_TEST_DATA }}/UFS_FIRE/{{ workflow.DATE_FIRST_CYCL }}' DT_FIRE: 0.5 OUTPUT_DT_FIRE: 300 FIRE_NUM_IGNITIONS: 2 diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml index 377f8176fc..756dac8135 100644 --- a/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml @@ -8,7 +8,7 @@ workflow: PREDEF_GRID_NAME: SUBCONUS_CO_3km DATE_FIRST_CYCL: '2020081318' DATE_LAST_CYCL: '2020081318' - FCST_LEN_HRS: 6 # 27 hours of LBCs staged on Derecho + FCST_LEN_HRS: 3 # 27 hours of LBCs staged on Derecho PREEXISTING_DIR_METHOD: rename task_get_extrn_ics: USE_USER_STAGED_EXTRN_FILES: true @@ -33,7 +33,7 @@ task_plot_allvars: fire: UFS_FIRE: True FIRE_NUM_TASKS: 1 - FIRE_INPUT_DIR: /glade/derecho/scratch/kavulich/FIRE/2024/test_data + FIRE_INPUT_DIR: '{{ platform.WE2E_TEST_DATA }}/UFS_FIRE/{{ workflow.DATE_FIRST_CYCL }}' DT_FIRE: 0.5 OUTPUT_DT_FIRE: 300 FIRE_NUM_IGNITIONS: 1 diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index b825454db4..2b2f85c8a7 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -732,7 +732,6 @@ def generate_FV3LAM_wflow( nmle = f"{setting.lower()}{i+1}" fire_nml_dict['fire'][nmle] = expt_config['fire'][setting][i] - print(f"{fire_nml_dict['fire'][nmle]=}") realize( input_config=expt_config['workflow']['FIRE_NML_BASE_FP'], diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index 8bc768732f..d8a3e8f4d4 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: pbspro + WE2E_TEST_DATA: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 92d33d7ad2..91e248e57b 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: slurm + WE2E_TEST_DATA: /gpfs/f5/epic/world-shared/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /gpfs/f5/epic/world-shared/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /gpfs/f5/epic/world-shared/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /gpfs/f5/epic/world-shared/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 189689f30d..1ca55ae270 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 40 SCHED: slurm + WE2E_TEST_DATA: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index 6a325094da..eddf307091 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 80 SCHED: slurm + WE2E_TEST_DATA: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 847530e4eb..b14a4ab9ff 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 24 SCHED: slurm + WE2E_TEST_DATA: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/noaacloud.yaml b/ush/machine/noaacloud.yaml index 50de28e751..2b27c0c139 100644 --- a/ush/machine/noaacloud.yaml +++ b/ush/machine/noaacloud.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: '{{ 44 if (user.ACCOUNT == "cz-epic") else 36 if (user.ACCOUNT == "ca-epic") else 28 }}' SCHED: slurm + WE2E_TEST_DATA: /contrib/EPIC/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/odin.yaml b/ush/machine/odin.yaml index 8cd1627da1..6bf51d707b 100644 --- a/ush/machine/odin.yaml +++ b/ush/machine/odin.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 24 SCHED: slurm + WE2E_TEST_DATA: /scratch/ywang/UFS_SRW_App/develop DOMAIN_PREGEN_BASEDIR: /scratch/ywang/UFS_SRW_App/develop/FV3LAM_pregen PARTITION_DEFAULT: workq QUEUE_DEFAULT: workq diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 5467160167..285eb34ee2 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 40 SCHED: slurm + WE2E_TEST_DATA: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/stampede.yaml b/ush/machine/stampede.yaml index 270cd3d593..cb6f1216eb 100644 --- a/ush/machine/stampede.yaml +++ b/ush/machine/stampede.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 68 SCHED: slurm + WE2E_TEST_DATA: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop DOMAIN_PREGEN_BASEDIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/FV3LAM_pregen PARTITION_DEFAULT: normal QUEUE_DEFAULT: normal diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index b8c3625dff..6adb36e6e5 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: pbspro + WE2E_TEST_DATA: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop TEST_CCPA_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc From c2c48f7ed6e1e4fc4750f795b1731b2fefa5d389 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 12 Nov 2024 09:37:23 -0700 Subject: [PATCH 86/87] Fix for jenkins test exec directory location --- ush/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/setup.py b/ush/setup.py index 0f2af305ec..deaebbf7c8 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -399,12 +399,12 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False): # Create a dictionary of config options from defaults, machine, and # user config files. - build_config_fp = os.path.join(USHdir, os.pardir, "exec", "build_settings.yaml") default_config_fp = os.path.join(USHdir, "config_defaults.yaml") user_config_fp = os.path.join(USHdir, user_config_fn) expt_config = load_config_for_setup(USHdir, default_config_fp, user_config_fp) # Load build settings as a dictionary; will be used later to make sure the build is consistent with the user settings + build_config_fp = os.path.join(expt_config["user"].get("EXECdir"), "build_settings.yaml") build_config = load_config_file(build_config_fp) logger.debug(f"Read build configuration from {build_config_fp}\n{build_config}") From 543f584c0d898f61d37ecdc29323b7f4ed7d87a8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 13 Nov 2024 12:40:27 -0700 Subject: [PATCH 87/87] Fix link to fire UG --- doc/UsersGuide/BuildingRunningTesting/FIRE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index 8f92012a65..09a22975a6 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -148,7 +148,7 @@ The CFBM creates output files in :term:`netCDF` format, with the naming scheme ` .. note:: - Any of the settings under :fire-ug:`the ``&fire`` section of the namelist ` can be specified in the SRW App ``config.yaml`` file under the ``fire:`` section, not just the settings described above. However, any additional settings from ``namelist.fire`` will need to be added to ``config_defaults.yaml`` first; otherwise the check for valid SRW options will fail. + Any of the settings under :fire-ug:`the &fire section of the namelist ` can be specified in the SRW App ``config.yaml`` file under the ``fire:`` section, not just the settings described above. However, any additional settings from ``namelist.fire`` will need to be added to ``config_defaults.yaml`` first; otherwise the check for valid SRW options will fail. To specify multiple fire ignitions (``FIRE_NUM_IGNITIONS > 1``), the above settings will need to be specified as a list, with one entry per ignition. See :numref:`Section %s ` for more details.