diff --git a/Makefile b/Makefile index 9b321be28..efc76c1f1 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ TARGETS = clean-devenv devshell docs env format lint meta package test typec export RECIPE_DIR := $(shell cd ./recipe && pwd) -clean = $(shell $(CONDA_EXE) env remove -n DEV-$(call val,name)) +clean = $(info $(shell $(CONDA_EXE) env remove -n DEV-$(call val,name))) spec = $(call val,name)$(2)$(call val,version)$(2)$(call val,$(1)) val = $(shell jq -r .$(1) $(METAJSON)) diff --git a/docs/conf.py b/docs/conf.py index c56003d01..2fb3178e3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,7 +10,7 @@ with open("../recipe/meta.json", "r", encoding="utf-8") as f: _metadata = json.loads(f.read()) -autodoc_mock_imports = ["f90nml", "jsonschema", "lxml"] +autodoc_mock_imports = ["f90nml", "iotaa", "jsonschema", "lxml"] copyright = str(dt.datetime.now().year) extensions = ["sphinx.ext.autodoc", "sphinx.ext.extlinks", "sphinx.ext.intersphinx"] extlinks_detect_hardcoded_links = True diff --git a/docs/index.rst b/docs/index.rst index e56502227..53ea4afe4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -96,17 +96,15 @@ Do you already have a Rocoto XML but don't want to run Rocoto to make sure it wo The Drivers ----------- -The uwtools driver(s) live right there beside the rest of the tools in the CLI and API. These tools will be under development for the foreseeable future, but we do have a forecast driver currently available in beta testing mode. +Drivers for NWP components are available as top-level CLI modes and API modules. -Forecast -^^^^^^^^ +FV3 +^^^ -| **CLI**: ``uw forecast -h`` -| **API**: ``import uwtools.api.drivers.forecast`` +| **CLI**: ``uw fv3 -h`` +| **API**: ``import uwtools.api.drivers.fv3`` -This driver is the first of its kind (with many others to come) and takes a few pieces of information from the user --- the model, the time, and a structured YAML --- and runs a forecast via batch job or as an executable. That simple. - -We've helped by providing a JSON Schema that allows you to validate your YAML to ensure you've got it right! +Provided with a valid UW YAML configuration file and a forecast-cycle value, ``uw fv3`` can prepare a fully provisioned FV3 run directory, execute FV3 directly, or can submit an FV3 batch job to an HPC scheduler. Over time, we'll add many other drivers to support a variety of UFS components from pre-processing to post-processing, along with many data assimilation components. diff --git a/docs/sections/user_guide/api/forecast.rst b/docs/sections/user_guide/api/forecast.rst deleted file mode 100644 index 7ab68efd8..000000000 --- a/docs/sections/user_guide/api/forecast.rst +++ /dev/null @@ -1,5 +0,0 @@ -``uwtools.api.forecast`` -======================== - -.. automodule:: uwtools.api.forecast - :members: diff --git a/docs/sections/user_guide/api/fv3.rst b/docs/sections/user_guide/api/fv3.rst new file mode 100644 index 000000000..4a5e70954 --- /dev/null +++ b/docs/sections/user_guide/api/fv3.rst @@ -0,0 +1,5 @@ +``uwtools.api.fv3`` +======================== + +.. automodule:: uwtools.api.fv3 + :members: diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index 4b6acfc97..2eb583e9b 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -3,7 +3,7 @@ API .. toctree:: config - forecast + fv3 logging rocoto template diff --git a/docs/sections/user_guide/cli/index.rst b/docs/sections/user_guide/cli/index.rst index e2fc943ac..ba5377559 100644 --- a/docs/sections/user_guide/cli/index.rst +++ b/docs/sections/user_guide/cli/index.rst @@ -5,6 +5,6 @@ CLI :maxdepth: 1 mode_config - mode_forecast + mode_fv3 mode_rocoto mode_template diff --git a/docs/sections/user_guide/cli/mode_config.rst b/docs/sections/user_guide/cli/mode_config.rst index cc52f806e..4818b5a4f 100644 --- a/docs/sections/user_guide/cli/mode_config.rst +++ b/docs/sections/user_guide/cli/mode_config.rst @@ -28,7 +28,7 @@ The ``uw`` mode for handling configuration files (configs). ``compare`` ----------- -The ``compare`` mode lets users compare two config files. +The ``compare`` mode lets users compare two config files. .. code-block:: text @@ -148,7 +148,7 @@ The examples that follow use namelist files ``values1.nml`` and ``values2.nml``, ``realize`` ----------- -In ``uw`` terminology, to realize a configuration file is to transform it from its raw form into its final, usable state. The ``realize`` mode can build a complete config file from two or more separate files. +In ``uw`` terminology, to realize a configuration file is to transform it from its raw form into its final, usable state. The ``realize`` mode can build a complete config file from two or more separate files. .. code-block:: text diff --git a/docs/sections/user_guide/cli/mode_forecast.rst b/docs/sections/user_guide/cli/mode_forecast.rst deleted file mode 100644 index bb3d9c854..000000000 --- a/docs/sections/user_guide/cli/mode_forecast.rst +++ /dev/null @@ -1,88 +0,0 @@ -Mode ``forecast`` -================= - -The ``uw`` mode for configuring and running forecasts. - -.. code-block:: text - - $ uw forecast --help - usage: uw forecast [-h] MODE ... - - Configure and run forecasts - - Optional arguments: - -h, --help - Show help and exit - - Positional arguments: - MODE - run - Run a forecast - -``run`` -------- - -.. code-block:: text - - $ uw forecast run --help - usage: uw forecast run --config-file PATH --cycle CYCLE --model {FV3} [-h] [--batch-script PATH] - [--dry-run] [--quiet] [--verbose] - - Run a forecast - - Required arguments: - --config-file PATH, -c PATH - Path to config file - --cycle CYCLE - The cycle in ISO8601 format - --model {FV3} - Model name - - Optional arguments: - -h, --help - Show help and exit - --batch-script PATH - Path to output batch file (defaults to stdout) - --dry-run - Only log info, making no changes - --debug - Print all log messages, plus any unhandled exception's stack trace (implies --verbose) - --quiet, -q - Print no logging messages - --verbose, -v - Print all logging messages - -.. _cli_forecast_run_examples: - -Examples -^^^^^^^^ - -The examples use a configuration file named ``config.yaml``. Its contents are described in depth in Section :ref:`forecast_yaml`. - -* Run an FV3 forecast on an interactive node - - .. code-block:: sh - - $ uw forecast run -c config.yaml --cycle 2024-01-09T12 --model FV3 - - The forecast will run on the node where you have invoked this command. Optionally, capture the output in a log file using shell redirection. - -* Run an FV3 forecast using a batch system - - .. code-block:: sh - - $ uw forecast run -c config.yaml --cycle 2024-01-09T12 --model FV3 --batch-script submit_fv3.sh - - This command writes a file named ``submit_fv3.sh`` and submits it to the batch system. - -* With the ``--dry-run`` flag specified, nothing is written to ``stdout``, but a report of all the directories, files, symlinks, etc., that would have been created are logged to ``stderr``. None of these artifacts will actually be created and no jobs will be executed or submitted to the batch system. - - .. code-block:: sh - - $ uw forecast run -c config.yaml --cycle 2024-01-09T12 --model FV3 --batch-script --dry-run - -* Request verbose log output - - .. code-block:: sh - - $ uw forecast run -c config.yaml --cycle 2024-01-09T12 --model FV3 -v diff --git a/docs/sections/user_guide/cli/mode_fv3.rst b/docs/sections/user_guide/cli/mode_fv3.rst new file mode 100644 index 000000000..5caa6d635 --- /dev/null +++ b/docs/sections/user_guide/cli/mode_fv3.rst @@ -0,0 +1,103 @@ +Mode ``fv3`` +============ + +The ``uw`` mode for configuring and running FV3. + +.. code-block:: text + + $ uw fv3 --help + usage: uw fv3 [-h] TASK ... + + Execute FV3 tasks + + Optional arguments: + -h, --help + Show help and exit + + Positional arguments: + TASK + boundary_files + The FV3 lateral boundary-condition files + diag_table + The FV3 diag_table file + field_table + The FV3 field_table file + files_copied + Files copied for FV3 run + files_linked + Files linked for FV3 run + model_configure + The FV3 model_configure file + namelist_file + The FV3 namelist file + provisioned_run_directory + The run directory provisioned with all required content + restart_directory + The FV3 RESTART directory + run + FV3 run execution + runscript + A runscript suitable for submission to the scheduler + +All tasks take the same arguments. For example: + +.. code-block:: text + + $ uw fv3 run --help + usage: uw fv3 run --config-file PATH --cycle CYCLE [-h] [--batch] [--dry-run] [--debug] [--quiet] + [--verbose] + + FV3 run execution + + Required arguments: + --config-file PATH, -c PATH + Path to config file + --cycle CYCLE + The cycle in ISO8601 format + + Optional arguments: + -h, --help + Show help and exit + --batch + Submit run to batch scheduler + --dry-run + Only log info, making no changes + --debug + Print all log messages, plus any unhandled exception's stack trace (implies --verbose) + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages + +Examples +^^^^^^^^ + +The examples use a configuration file named ``config.yaml``. Its contents are described in depth in section :ref:`fv3_yaml`. + +* Run FV3 on an interactive node + + .. code-block:: text + + $ uw fv3 run --config-file config.yaml --cycle 2024-02-11T12 + + The driver creates a ``runscript`` file in the directory specified by ``run_dir`` in the config and runs it, executing FV3. + +* Run FV3 via a batch job + + .. code-block:: text + + $ uw fv3 run --config-file config.yaml --cycle 2024-02-11T12 --batch + + The driver creates a ``runscript`` file in the directory specified by ``run_dir`` in the config and submits it to the batch system. Running with ``--batch`` requires a correctly configured ``platform`` block in ``config,yaml``, as well as appropriate settings in the ``execution`` block under ``fv3``. + +* Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any. + + .. code-block:: text + + $ uw fv3 run --config-file config.yaml --cycle 2024-02-11T12 --batch --dry-run + +* The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an FV3 run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file: + + .. code-block: text + + $ uw fv3 provisioned_run_directory --config-file config.yaml --cycle 2024-02-11T12 --batch diff --git a/docs/sections/user_guide/installation.rst b/docs/sections/user_guide/installation.rst index e8755759b..b9561d8b8 100644 --- a/docs/sections/user_guide/installation.rst +++ b/docs/sections/user_guide/installation.rst @@ -1,8 +1,8 @@ Installation ============ -.. note:: - +.. note:: + Developers should visit the :doc:`Developer Setup <../contributor_guide/developer_setup>` section located in the :doc:`Contributor Guide <../contributor_guide/index>`. The recommended installation mechanism uses the Python package and virtual-environment manager :conda:`conda<>`. Specifically, these instructions assume use of the :miniforge:`Miniforge<>` variant of :miniconda:`Miniconda<>`, built to use, by default, packages from the :conda-forge:`conda-forge<>` project. Users of the original :miniconda:`Miniconda<>` or the :anaconda:`Anaconda distribution<>` should add the flags ``-c conda-forge --override-channels`` to ``conda`` commands to specify the required package channels. diff --git a/docs/sections/user_guide/uw_yaml/field_table_yaml.rst b/docs/sections/user_guide/uw_yaml/field_table.rst similarity index 100% rename from docs/sections/user_guide/uw_yaml/field_table_yaml.rst rename to docs/sections/user_guide/uw_yaml/field_table.rst diff --git a/docs/sections/user_guide/uw_yaml/forecast_yaml.rst b/docs/sections/user_guide/uw_yaml/forecast_yaml.rst deleted file mode 100644 index d28601065..000000000 --- a/docs/sections/user_guide/uw_yaml/forecast_yaml.rst +++ /dev/null @@ -1,240 +0,0 @@ -.. _forecast_yaml: - -Forecast YAML -============= - -The structured YAML to run a forecast is described below. It is enforced via JSON Schema. - -In this block, there are entries that define where the forecast will run (``run_directory:``), what will run (``executable:``), the data files that should be staged (``cycle_dependent:`` and ``static:``), and blocks that correspond to the configuration files required by the forecast model (``fd_ufs:``, ``field_table:``, ``namelist:``, etc.). Each of the configuration file blocks will allow the user to set a template file as input, or to define a configuration file in its native key/value pair format with an option to update the values (``update_values:``) contained in the original input file (``base_file:``). - -The configuration files required by the UFS Weather Model are documented :weather-model-io:`here`. - -The ``forecast:`` block ------------------------ - -This section describes the specifics of the FV3 atmosphere forecast component. - -.. code-block:: yaml - - forecast: - cycle_dependent: - INPUT/gfs_data.nc: /path/to/gfs_data.nc - INPUT/sfc_data.nc: /path/to/sfc_data.nc - INPUT/gfs_ctrl.nc: /path/to/gfs_ctrl.nc - diag_table: /path/to/diag_table/template/file - domain: regional - executable: fv3.exe - fd_ufs: - base_file: /path/to/base/fd_ufs.yaml - field_table: - base_file: /path/to/field_table.yaml - update_values: - liq_wat: - longname: cloud water mixing ratio - units: kg/kg - profile_type: - name: fixed - surface_value: 2.0 - length: 12 - model_configure: - base_file: /path/to/base/model_configure - update_values: - write_dopost: .false. - namelist: - base_file: /path/to/base/input.nml - update_values: - fv_core_nml: - k_split: 2 - n_split: 6 - run_dir: /path/to/forecast/run/directory/{yyyymmddhh} - static: - fv3.exe: /path/to/executable/ufs_model - INPUT/grid_spec.nc: /path/to/grid_spec.nc - ... - data_table: /path/to/data_table - eta_micro_lookup.data: /path/to/noahmptable.dat - noahmptable.tbl: /path/to/noahmptable.tbl - ufs_configure: /path/to/template/ufs.configure - -.. _updating_values: - -Updating Values -^^^^^^^^^^^^^^^ - -Many of the blocks describe configuration files needed by the UFS Weather Model, i.e. ``namelist:``, ``fd_ufs:``, ``model_configure:``. The ``base_file:`` entry in a given block is required to initially stage the file; it can then be modified via an ``update_values:`` block. - -To ensure the correct values are updated, the hierarchy of entries in the base file must be mirrored under the ``update_values:`` block. Multiple entries within a block may be updated and they need not follow the same order as those in the base file. For example, the base file named ``people.yaml`` may contain: - -.. code-block:: yaml - - person: - age: 19 - address: - city: Boston - number: 12 - state: MA - street: Acorn St - name: Jane - -Then the entries in the ``update_values:`` YAML block would override this base file with the entries: - -.. code-block:: yaml - - base_file: people.yaml - update_values: - person: - address: - street: Main St - number: 99 - -The contents of the staged ``people.yaml`` that results: - -.. code-block:: yaml - - person: - age: 19 - address: - city: Boston - number: 99 - state: MA - street: Main St - name: Jane - - -UW YAML Keys -^^^^^^^^^^^^ - -``cycle_dependent:`` -"""""""""""""""""""" - -This block contains a set of files to stage in the run directory: File names as they appear in the run directory are keys and their source paths are the values. Source paths can be provided as a single string path, or a list of paths to be staged in a common directory under their original names. - - .. warning:: The current version does not support adding cycle information to the content of the files, and this information must be hard-coded in the YAML file. - -``diag_table:`` -""""""""""""""" - -The path to the input Jinja2 template for the ``diag_table`` file. - -The diag_table is described :weather-model-io:`here`. - -``domain:`` -""""""""""" - -A switch to differentiate between a global or regional configuration. Accepted values are ``global`` and ``regional``. - -``executable:`` -""""""""""""""" - -The path to the compiled executable. - -``fd_ufs:`` -"""""""""""" - -This block requires a ``base_file:`` entry that contains the path to the YAML file. An optional ``update_values:`` block may be provided to update any values contained in the base file. Please see the :ref:`updating_values` section for providing information in these entries. - -The ``fd_ufs.yaml`` file is a structured YAML used by the FV3 weather model. The tested version can be found in the :ufs-weather-model:`ufs-weather-model repository`. The naming convention for the dictionary entries are documented :cmeps:`here<>`. - -``field_table:`` -"""""""""""""""" - -The block requires a ``base_file:`` entry that contains the path to the YAML file. An optional ``update_values:`` block may be provided to update any values contained in the base file. Please see the :ref:`updating_values` section for providing information in these entries. - -If a predefined field table (i.e., not a configurable YAML) is to be used, include it in the ``static:`` block. - -The documentation for the ``field_table`` file is :weather-model-io:`here`. Information on how to structure the UW YAML for configuring a ``field_table`` is in the :ref:`defining_a_field_table` Section. - -``length:`` -""""""""""" - -The length of the forecast in hours. - -``model_configure:`` -"""""""""""""""""""" - -The block requires a ``base_file:`` entry that contains the path to the YAML file. An optional ``update_values:`` block may be provided to update any values contained in the base file. Please see the :ref:`updating_values` section for providing information in these entries. - -The documentation for the ``model_configure`` file is :weather-model-io:`here`. - -``namelist:`` -""""""""""""" - -The block requires a ``base_file:`` entry that contains the path to the namelist file. An optional ``update_values:`` block may be provided to update any values contained in the base file. Please see the :ref:`updating_values` section for providing information in these entries. - -The documentation for the FV3 namelist, ``input.nml`` is :weather-model-io:`here`. - -``run_dir:`` -"""""""""""" - -The path where the forecast input data will be staged and output data will appear after a successful forecast. - -``static:`` -""""""""""" - -This block contains a set of files to stage in the run directory: file names as they appear in the run directory are keys and their source paths are the values. Source paths can be provided as a single string path, or a list of paths to be staged in a common directory under their original names. - -``ufs_configure:`` - -""""""""""""""""""" - -The path to the input Jinja2 template for the ``ufs.configure`` file. - -The documentation for the ``ufs.configure`` file is :weather-model-io:`here`. - -The ``platform:`` block ------------------------ - -This block describes necessary facts about the computational platform. - -.. code-block:: yaml - - platform: - mpicmd: srun # required - scheduler: slurm - -``mpicmd:`` -^^^^^^^^^^^ -The MPI command used to run the model executable. Typical options are ``srun``, ``mpirun``, ``mpiexec``, etc. System administrators should be able to advise the appropriate choice, if needed. - -``scheduler:`` -^^^^^^^^^^^^^^ -The name of the batch system. Supported options are ``lsf``, ``pbs``, and ``slurm``. - -The ``preprocessing:`` block ----------------------------- - -.. code-block:: yaml - - preprocessing: - lateral_boundary_conditions: - interval_hours: 3 # optional, default - offset: 0 # optional, default - output_file_path: # required - -``lateral_boundary_conditions:`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The optional block describes how the lateral boundary conditions have been prepared for a limited-area configuration of the model forecast. It is required for a limited-area forecast. The following entries in its subtree are used for the forecast: - -``interval_hours:`` -""""""""""""""""""" -The integer number of hours setting how frequently the lateral boundary conditions will be used in the model forecast. - -``offset:`` -""""""""""" -The integer number of hours setting how many hours earlier the external model used for boundary conditions started compared to the desired forecast cycle. - -``output_file_path:`` -""""""""""""""""""""""""" -The path to the lateral boundary conditions files prepared for the forecast. It accepts the integer ``forecast_hour`` as a Python template, e.g., ``/path/to/srw.t00z.gfs_bndy.tile7.f{forecast_hour:03d}.nc``. - -The ``user:`` block -------------------- - -.. code-block:: yaml - - user: - account: my_account # optional - -``account:`` -^^^^^^^^^^^^ -The user account associated with the batch system. diff --git a/docs/sections/user_guide/uw_yaml/fv3.rst b/docs/sections/user_guide/uw_yaml/fv3.rst new file mode 100644 index 000000000..1818a49ec --- /dev/null +++ b/docs/sections/user_guide/uw_yaml/fv3.rst @@ -0,0 +1,246 @@ +.. _fv3_yaml: + +FV3 YAML +======== + +The structured YAML to run FV3 is described below. It is validated by JSON Schema. The configuration files required by the UFS Weather Model are documented :weather-model-io:`here`. + +The ``fv3:`` and ``platform:`` Blocks +------------------------------------- + +Here are prototype UW YAML ``fv3:`` and ``platform:`` blocks, explained in detail below: + +.. code-block:: yaml + + fv3: + diag_table: /path/to/diag_table_to_use + domain: regional + execution: + batchargs: + walltime: "00:10:00" + executable: ufs_model + mpiargs: + - "--export=NONE" + mpicmd: srun + threads: 1 + field_table: + base_file: /path/to/field_table_to_use + update_values: + liq_wat: + longname: cloud water mixing ratio + units: kg/kg + profile_type: + name: fixed + surface_value: 2.0 + files_to_copy: + INPUT/gfs_data.nc: /path/to/gfs_data.nc + INPUT/sfc_data.nc: /path/to/sfc_data.nc + INPUT/gfs_ctrl.nc: /path/to/gfs_ctrl.nc + ... + files_to_link: + co2historicaldata_2010.txt: src/uwtools/drivers/global_co2historicaldata_2010.txt + co2historicaldata_2011.txt: src/uwtools/drivers/global_co2historicaldata_2011.txt + ... + lateral_boundary_conditions: + interval_hours: 3 + offset: 0 + path: gfs_bndy.tile{tile}.f{forecast_hour}.nc + length: 12 + model_configure: + base_file: /path/to/model_configure_to_use + update_values: + write_dopost: .false. + namelist: + base_file: /path/to/base/input.nml + update_values: + fv_core_nml: + k_split: 2 + n_split: 6 + run_dir: /path/to/runs/{{ cycle.strftime('%Y%m%d%H') }} + platform: + account: user_account + scheduler: slurm + +.. _updating_values: + +Updating Values +--------------- + +Some blocks support ``base_file:`` and ``update_values:`` entries. A ``base_file:`` entry specifies the path to a file to use as a basis for a particular runtime file. An ``update_values:`` entry specifies changes/additions to the base file. At least one of ``base_file:`` or ``update_values:`` must be provided. If only ``base_file:`` is provided, the file will be used as-is. If only ``update_values:`` is provided, it will completely define the runtime file in question. If both are provided, ``update_values:`` is used to modify the contents of ``base_file:``. The hierarchy of entries in the ``update_values:`` block must mirror that in the ``base_file:``. For example, a ``base_file:`` named ``people.yaml`` might contain: + +.. code-block:: yaml + + person: + age: 19 + address: + city: Boston + number: 12 + state: MA + street: Acorn St + name: Jane + +A compatible YAML block updating the person's street address might then contain: + +.. code-block:: yaml + + base_file: people.yaml + update_values: + person: + address: + street: Main St + number: 99 + +The result would be: + +.. code-block:: yaml + + person: + age: 19 + address: + city: Boston + number: 99 + state: MA + street: Main St + name: Jane + +UW YAML for the ``fv3:`` Block +------------------------------ + +diag_table: +^^^^^^^^^^^ + +The path to the ``diag_table`` file. It does not currently support edits, so must be pre-configured as needed. See FV3 ``diag_table`` documentation :weather-model-io:`here`. + +domain: +^^^^^^^ + +Accepted values are ``global`` and ``regional``. + +execution: +^^^^^^^^^^ + +batchargs: +"""""""""" + +These entries map to job-scheduler directives sent to e.g. Slurm when a batch job is submitted via the ``--batch`` CLI switch or the ``batch=True`` API argument. The only **required** entry is ``walltime``. + +Shorthand names are provided for certain directives for each scheduler, and can be specified as-so along with appropriate values. Recognized names for each scheduler are: + +* LSF: ``jobname``, ``memory``, ``nodes``, ``queue``, ``shell``, ``stdout``, ``tasks_per_node``, ``threads``, ``walltime`` +* PBS: ``debug``, ``jobname``, ``memory``, ``nodes``, ``queue``, ``shell``, ``stdout``, ``tasks_per_node``, ``threads``, ``walltime`` +* Slurm: ``cores``, ``exclusive``, ``export``, ``jobname``, ``memory``, ``nodes``, ``partition``, ``queue``, ``rundir``, ``stderr``, ``stdout``, ``tasks_per_node``, ``threads``, ``walltime`` + +Other, arbitrary directive key-value pairs can be provided exactly as they should appear in the batch runscript. For example + +.. code-block:: yaml + + --nice: 100 + +could be specified to have the Slurm directive + +.. code-block: text + + #SBATCH --nice=100 + +included in the batch runscript. + +executable: +""""""""""" + +The name of or path to the FV3 executable binary. + +mpiargs: +"""""""" + +An **array** of string arguments that should follow the MPI launch program (``mpiexec``, ``srun``, et al.) on the command line. + +mpicmd: +""""""" + +The MPI launch program (``mpiexec``, ``srun``, et al.) + +threads: +"""""""" + +The number of OpenMP threads to use when running FV3. + +field_table: +^^^^^^^^^^^^ + +Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_values` for details). See FV3 ``field_table`` documentation :weather-model-io:`here`, or :ref:`defining_a_field_table` for UW YAML-specific details. + +files_to_copy: +^^^^^^^^^^^^^^ + +Defines files to be copied to the run directory. Keys in the ``files_to_copy:`` YAML map specify destination paths relative to the run directory, and values specify source paths. Both keys and values may contain Jinja2 expressions using a ``cycle`` variable, which is a Python ``datetime`` object corresponding to the FV3 cycle being run. This supports specification of cycle-specific filenames/paths. For example, a key-value pair + +.. code-block: yaml + + gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc: /some/path/{{ cycle.strftime('%Y%m%d')}}/{{ cycle.strftime('%H') }}/gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc + +would be rendered as + +.. code-block: yaml + + gfs.t18z.atmanl.nc: /some/path/20240212/18/gfs.t18z.atmanl.nc + +for the ``2024-02-12T18`` cycle. + +files_to_link: +^^^^^^^^^^^^^^ + +Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies. + + +lateral_boundary_conditions: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Describes how the lateral boundary conditions have been prepared for a limited-area configuration of the FV3 forecast. + +interval_hours: +""""""""""""""" + +How frequently the lateral boundary conditions will be used in the FV3 forecast, in integer hours. + +offset: +""""""" + +How many hours earlier the external model used for boundary conditions started compared to the desired forecast cycle, in integer hours. + +path: +""""" + +An absolute-path template to the lateral boundary condition files prepared for the forecast. The Python ``int`` variable ``forecast_hour`` will be interpolated into, e.g., ``/path/to/srw.t00z.gfs_bndy.tile7.f{forecast_hour:03d}.nc``. Note that this is a Python string template rather than a Jinja2 template. + +length: +""""""" + +The length of the forecast in integer hours. + +model_configure: +"""""""""""""""" + +Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_values` for details). See FV3 ``model_configure`` documentation :weather-model-io:`here`. + +namelist: +""""""""" + +Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_values` for details). See FV3 ``model_configure`` documentation :weather-model-io:`here`. + +run_dir: +"""""""" + +The path to the directory where FV3 will find its inputs, configuration files, etc., and where it will write its output. + +UW YAML for the ``platform:`` Block +----------------------------------- + +account: +^^^^^^^^ + +The account name to use when requesting resources from the batch job scheduler. + +scheduler: +^^^^^^^^^^ + +One of ``lsf``, ``pbs``, or ``slurm``. diff --git a/docs/sections/user_guide/uw_yaml/index.rst b/docs/sections/user_guide/uw_yaml/index.rst index a0fe1a7de..4719887f4 100644 --- a/docs/sections/user_guide/uw_yaml/index.rst +++ b/docs/sections/user_guide/uw_yaml/index.rst @@ -4,6 +4,6 @@ UW YAML .. toctree:: :maxdepth: 1 - field_table_yaml - forecast_yaml - rocoto_yaml + field_table + fv3 + rocoto diff --git a/docs/sections/user_guide/uw_yaml/rocoto_yaml.rst b/docs/sections/user_guide/uw_yaml/rocoto.rst similarity index 99% rename from docs/sections/user_guide/uw_yaml/rocoto_yaml.rst rename to docs/sections/user_guide/uw_yaml/rocoto.rst index 90a305f3a..a26c334a5 100644 --- a/docs/sections/user_guide/uw_yaml/rocoto_yaml.rst +++ b/docs/sections/user_guide/uw_yaml/rocoto.rst @@ -250,7 +250,7 @@ This translates to Rocoto XML (whitespace added for readability): -This example of Rocoto XML will be expanded during the workflow's execution to generate three individual tasks: ``hello``, ``hola``, and ``bonjour``. +This example of Rocoto XML will be expanded during the workflow's execution to generate three individual tasks: ``hello``, ``hola``, and ``bonjour``. UW YAML Definitions ------------------- diff --git a/recipe/channels b/recipe/channels index bdff4d5cf..f19240f68 100644 --- a/recipe/channels +++ b/recipe/channels @@ -1 +1,2 @@ conda-forge +maddenp diff --git a/recipe/meta.json b/recipe/meta.json index e3498a3e3..ccf16bac3 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -8,6 +8,7 @@ "coverage =7.3.*", "docformatter =1.7.*", "f90nml =1.4.*", + "iotaa =0.7.1.*", "isort =5.13.*", "jinja2 =3.1.*", "jq =1.7.*", @@ -23,6 +24,7 @@ ], "run": [ "f90nml =1.4.*", + "iotaa =0.7.1.*", "jinja2 =3.1.*", "jsonschema =4.20.*", "lxml =4.9.*", diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 883c24ee5..3c3279d1b 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -12,6 +12,7 @@ requirements: - pip run: - f90nml 1.4.* + - iotaa 0.7.1.* - jinja2 3.1.* - jsonschema 4.20.* - lxml 4.9.* diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index 4dc645a60..2e5f4ea6f 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -166,8 +166,6 @@ def _ensure_config_arg_type( # Import-time code -# pylint: disable=duplicate-code - # The following statements dynamically interpolate values into functions' docstrings, which will not # work if the docstrings are inlined in the functions. They must remain separate statements to avoid # hardcoding values into them. diff --git a/src/uwtools/api/forecast.py b/src/uwtools/api/forecast.py deleted file mode 100644 index 0eb21418d..000000000 --- a/src/uwtools/api/forecast.py +++ /dev/null @@ -1,41 +0,0 @@ -import datetime as dt - -from uwtools.drivers.forecast import CLASSES as _CLASSES -from uwtools.types import DefinitePath, OptionalPath - - -def run( # pylint: disable=missing-function-docstring - model: str, - cycle: dt.datetime, - config_file: DefinitePath, - batch_script: OptionalPath = None, - dry_run: bool = False, -) -> bool: - forecast_class = _CLASSES[model] - return forecast_class( - batch_script=batch_script, - config_file=config_file, - dry_run=dry_run, - ).run(cycle=cycle) - - -# The following statement dynamically interpolates values into run()'s docstring, which will not -# work if the docstring is inlined in the function. It must remain a separate statement to avoid -# hardcoding values into it. - -run.__doc__ = """ -Run a forecast model. - -If ``batch_script`` is specified, a batch script will be written that, when submitted to the appropriate -scheduler, will run the forecast on batch resources. When not specified, the forecast will be run -immediately on the current system, without creation of a batch script. - -:param model: One of: {models} -:param cycle: The cycle to run -:param config_file: Path to config file for the forecast run -:param batch_script: Path to a batch script to write -:param dry_run: Do not run forecast, just report what would have been done -:return: Success status of requested operation (immediate run or batch-script creation) -""".format( - models=", ".join(list(_CLASSES.keys())) -).strip() diff --git a/src/uwtools/api/fv3.py b/src/uwtools/api/fv3.py new file mode 100644 index 000000000..5b79ac03d --- /dev/null +++ b/src/uwtools/api/fv3.py @@ -0,0 +1,41 @@ +import datetime as dt +from typing import Dict + +import iotaa + +from uwtools.drivers.fv3 import FV3 +from uwtools.types import DefinitePath + + +def execute( + task: str, + config_file: DefinitePath, + cycle: dt.datetime, + batch: bool = False, + dry_run: bool = False, +) -> bool: + """ + Execute an FV3 task. + + If ``batch`` is specified, a runscript will be written and submitted to the batch system. + Otherwise, the forecast will be run directly on the current system. + + :param task: The task to execute + :param config_file: Path to UW YAML config file + :param cycle: The cycle to run + :param batch: Submit run to the batch system + :param dry_run: Do not run forecast, just report what would have been done + :return: True if task completes without raising an exception + """ + obj = FV3(config_file=config_file, cycle=cycle, batch=batch, dry_run=dry_run) + getattr(obj, task)() + return True + + +def tasks() -> Dict[str, str]: + """ + Returns a mapping from task names to their one-line descriptions. + """ + return { + task: getattr(FV3, task).__doc__.strip().split("\n")[0] for task in iotaa.tasknames(FV3) + } diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index ad25977f0..6f8c4dd0a 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -14,12 +14,11 @@ from typing import Any, Callable, Dict, List, Tuple import uwtools.api.config -import uwtools.api.forecast +import uwtools.api.fv3 import uwtools.api.rocoto import uwtools.api.template import uwtools.config.jinja2 import uwtools.rocoto -from uwtools.drivers.forecast import CLASSES as FORECAST_CLASSES from uwtools.logging import log, setup_logging from uwtools.utils.file import FORMAT, get_file_format @@ -50,7 +49,7 @@ def main() -> None: log.debug("Command: %s %s", Path(sys.argv[0]).name, " ".join(sys.argv[1:])) modes = { STR.config: _dispatch_config, - STR.forecast: _dispatch_forecast, + STR.fv3: _dispatch_fv3, STR.rocoto: _dispatch_rocoto, STR.template: _dispatch_template, } @@ -72,7 +71,7 @@ def _add_subparser_config(subparsers: Subparsers) -> ModeChecks: """ parser = _add_subparser(subparsers, STR.config, "Handle configs") _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.action) + subparsers = _add_subparsers(parser, STR.action, STR.action.upper()) return { STR.compare: _add_subparser_config_compare(subparsers), STR.realize: _add_subparser_config_realize(subparsers), @@ -199,61 +198,54 @@ def _dispatch_config_validate(args: Args) -> bool: return uwtools.api.config.validate(schema_file=args[STR.schemafile], config=args[STR.infile]) -# Mode forecast +# Mode fv3 -def _add_subparser_forecast(subparsers: Subparsers) -> ModeChecks: +def _add_subparser_fv3(subparsers: Subparsers) -> ModeChecks: """ - Subparser for mode: forecast + Subparser for mode: fv3 :param subparsers: Parent parser's subparsers, to add this subparser to. """ - parser = _add_subparser(subparsers, STR.forecast, "Configure and run forecasts") + parser = _add_subparser(subparsers, STR.fv3, "Execute FV3 tasks") _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.action) + subparsers = _add_subparsers(parser, STR.action, STR.task.upper()) return { - STR.run: _add_subparser_forecast_run(subparsers), + task: _add_subparser_fv3_task(subparsers, task, helpmsg) + for task, helpmsg in uwtools.api.fv3.tasks().items() } -def _add_subparser_forecast_run(subparsers: Subparsers) -> ActionChecks: +def _add_subparser_fv3_task(subparsers: Subparsers, task: str, helpmsg: str) -> ActionChecks: """ - Subparser for mode: forecast run + Subparser for mode: fv3 :param subparsers: Parent parser's subparsers, to add this subparser to. + :param task: The task to add a subparser for. + :param helpmsg: Help message for task. """ - parser = _add_subparser(subparsers, STR.run, "Run a forecast") + parser = _add_subparser(subparsers, task, helpmsg.rstrip(".")) required = parser.add_argument_group(TITLE_REQ_ARG) _add_arg_config_file(required) _add_arg_cycle(required) - _add_arg_model(required, choices=list(FORECAST_CLASSES.keys())) optional = _basic_setup(parser) - _add_arg_batch_script(optional) + _add_arg_batch(optional) _add_arg_dry_run(optional) checks = _add_args_verbosity(optional) return checks -def _dispatch_forecast(args: Args) -> bool: +def _dispatch_fv3(args: Args) -> bool: """ - Dispatch logic for forecast mode. + Dispatch logic for fv3 mode. :param args: Parsed command-line args. """ - return {STR.run: _dispatch_forecast_run}[args[STR.action]](args) - - -def _dispatch_forecast_run(args: Args) -> bool: - """ - Dispatch logic for forecast run action. - - :param args: Parsed command-line args. - """ - return uwtools.api.forecast.run( - model=args[STR.model], - cycle=args[STR.cycle], + return uwtools.api.fv3.execute( + task=args[STR.action], config_file=args[STR.cfgfile], - batch_script=args[STR.batch_script], + cycle=args[STR.cycle], + batch=args[STR.batch], dry_run=args[STR.dryrun], ) @@ -269,7 +261,7 @@ def _add_subparser_rocoto(subparsers: Subparsers) -> ModeChecks: """ parser = _add_subparser(subparsers, STR.rocoto, "Realize and validate Rocoto XML Documents") _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.action) + subparsers = _add_subparsers(parser, STR.action, STR.action.upper()) return { STR.realize: _add_subparser_rocoto_realize(subparsers), STR.validate: _add_subparser_rocoto_validate(subparsers), @@ -346,7 +338,7 @@ def _add_subparser_template(subparsers: Subparsers) -> ModeChecks: """ parser = _add_subparser(subparsers, STR.template, "Handle templates") _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.action) + subparsers = _add_subparsers(parser, STR.action, STR.action.upper()) return { STR.render: _add_subparser_template_render(subparsers), STR.translate: _add_subparser_template_translate(subparsers), @@ -435,14 +427,11 @@ def _dispatch_template_translate(args: Args) -> bool: # pylint: disable=missing-function-docstring -def _add_arg_batch_script(group: Group, required: bool = False) -> None: +def _add_arg_batch(group: Group) -> None: group.add_argument( - _switch(STR.batch_script), - help="Path to output batch file (defaults to stdout)", - metavar="PATH", - required=required, - default=None, - type=str, + _switch(STR.batch), + action="store_true", + help="Submit run to batch scheduler", ) @@ -468,7 +457,7 @@ def _add_arg_cycle(group: Group) -> None: def _add_arg_debug(group: Group) -> None: group.add_argument( - "--debug", + _switch(STR.debug), action="store_true", help=""" Print all log messages, plus any unhandled exception's stack trace (implies --verbose) @@ -536,16 +525,6 @@ def _add_arg_key_eq_val_pairs(group: Group) -> None: ) -def _add_arg_model(group: Group, choices: List[str]) -> None: - group.add_argument( - _switch(STR.model), - choices=choices, - help="Model name", - required=True, - type=str, - ) - - def _add_arg_output_file(group: Group, required: bool = False) -> None: group.add_argument( _switch(STR.outfile), @@ -676,15 +655,17 @@ def _add_subparser(subparsers: Subparsers, name: str, helpmsg: str) -> Parser: return parser -def _add_subparsers(parser: Parser, dest: str) -> Subparsers: +def _add_subparsers(parser: Parser, dest: str, metavar: str) -> Subparsers: """ Add subparsers to a parser. - :parm parser: The parser to add subparsers to. + :param parser: The parser to add subparsers to. + :param dest: Name of parser attribute to store subparser under. + :param metavar: Name for hierarchy of subparsers as shown by --help. :return: The new subparsers object. """ return parser.add_subparsers( - dest=dest, metavar="MODE", required=True, title="Positional arguments" + dest=dest, metavar=metavar, required=True, title="Positional arguments" ) @@ -756,10 +737,10 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: description="Unified Workflow Tools", add_help=False, formatter_class=_formatter ) _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.mode) + subparsers = _add_subparsers(parser, STR.mode, STR.mode.upper()) checks = { STR.config: _add_subparser_config(subparsers), - STR.forecast: _add_subparser_forecast(subparsers), + STR.fv3: _add_subparser_fv3(subparsers), STR.rocoto: _add_subparser_rocoto(subparsers), STR.template: _add_subparser_template(subparsers), } @@ -783,7 +764,7 @@ class STR: """ action: str = "action" - batch_script: str = "batch_script" + batch: str = "batch" cfgfile: str = "config_file" compare: str = "compare" config: str = "config" @@ -794,7 +775,7 @@ class STR: file1path: str = "file_1_path" file2fmt: str = "file_2_format" file2path: str = "file_2_path" - forecast: str = "forecast" + fv3: str = "fv3" help: str = "help" infile: str = "input_file" infmt: str = "input_format" @@ -810,6 +791,8 @@ class STR: run: str = "run" schemafile: str = "schema_file" suppfiles: str = "supplemental_files" + task: str = "task" + tasks: str = "tasks" template: str = "template" translate: str = "translate" validate: str = "validate" diff --git a/src/uwtools/config/formats/base.py b/src/uwtools/config/formats/base.py index 5e0e769d7..0b393dfbc 100644 --- a/src/uwtools/config/formats/base.py +++ b/src/uwtools/config/formats/base.py @@ -10,7 +10,7 @@ import yaml -from uwtools.config.jinja2 import dereference +from uwtools.config import jinja2 from uwtools.config.support import INCLUDE_TAG, depth, log_and_error from uwtools.exceptions import UWConfigError from uwtools.logging import log @@ -159,7 +159,7 @@ def depth(self) -> int: """ return depth(self.data) - def dereference(self) -> None: + def dereference(self, context: Optional[dict] = None) -> None: """ Render as much Jinja2 syntax as possible. """ @@ -171,7 +171,7 @@ def logstate(state: str) -> None: while True: logstate("current") - new = dereference(val=self.data, context={**os.environ, **self.data}) + new = jinja2.dereference(val=self.data, context=context or {**os.environ, **self.data}) assert isinstance(new, dict) if new == self.data: break diff --git a/src/uwtools/config/formats/ini.py b/src/uwtools/config/formats/ini.py index a5cd53e99..c2d2125ab 100644 --- a/src/uwtools/config/formats/ini.py +++ b/src/uwtools/config/formats/ini.py @@ -1,5 +1,3 @@ -# pylint: disable=duplicate-code - import configparser from io import StringIO from typing import Optional, Union diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index fc9abc14f..bda1dfd3b 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -127,7 +127,7 @@ def dereference(val: _ConfigVal, context: dict, local: Optional[dict] = None) -> """ rendered: _ConfigVal = val # fall-back value if isinstance(val, dict): - return {k: dereference(v, context, local=val) for k, v in val.items()} + return {dereference(k, context): dereference(v, context, local=val) for k, v in val.items()} if isinstance(val, list): return [dereference(v, context) for v in val] if isinstance(val, str): diff --git a/src/uwtools/config/validator.py b/src/uwtools/config/validator.py index 12ddc2b6e..3aa1718b5 100644 --- a/src/uwtools/config/validator.py +++ b/src/uwtools/config/validator.py @@ -36,11 +36,7 @@ def validate_yaml( for error in errors: for line in str(error).split("\n"): log.error(line) - # It's pointless to evaluate an invalid config, so return now if that's the case. - if errors: - return False - # If no issues were detected, report success. - return True + return not bool(errors) # Private functions diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index a68c85b8e..295650d47 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -1,177 +1,146 @@ """ -Provides an abstract class representing drivers for various NWP tools. +An abstract class for component drivers. """ - -import os -import shutil +import re from abc import ABC, abstractmethod -from collections.abc import Mapping -from datetime import datetime from pathlib import Path -from typing import Any, Dict, Type, Union +from textwrap import dedent +from typing import Any, Dict, List, Optional, Type from uwtools.config import validator from uwtools.config.formats.base import Config from uwtools.config.formats.yaml import YAMLConfig +from uwtools.exceptions import UWConfigError from uwtools.logging import log -from uwtools.scheduler import BatchScript, JobScheduler -from uwtools.types import DefinitePath, OptionalPath +from uwtools.scheduler import JobScheduler +from uwtools.types import DefinitePath class Driver(ABC): """ - An abstract class representing drivers for various NWP tools. + An abstract class for component drivers. """ - def __init__( - self, - config_file: DefinitePath, - dry_run: bool = False, - batch_script: OptionalPath = None, - ): - """ - Initialize the driver. + def __init__(self, config_file: DefinitePath, dry_run: bool = False, batch: bool = False): """ + A component driver. - self._config_file = config_file - self._dry_run = dry_run - self._batch_script = batch_script + :param config_file: Path to config file. + :param dry_run: Run in dry-run mode? + :param batch: Run component via the batch system? + """ + self._config = YAMLConfig(config=config_file) + self._config.dereference() self._validate() - self._experiment_config = YAMLConfig(config=config_file) - self._platform_config = self._experiment_config.get("platform", {}) - self._config: Dict[str, Any] = {} - - # Public methods + self._dry_run = dry_run + self._batch = batch - @abstractmethod - def batch_script(self) -> BatchScript: + @staticmethod + def _create_user_updated_config( + config_class: Type[Config], config_values: dict, path: Path + ) -> None: """ - Create a script for submission to the batch scheduler. + Create a config from a base file, user-provided values, or a combination of the two. - :return: The batch script object with all run commands needed for executing the program. + :param config_class: The Config subclass matching the config type. + :param config_values: The configuration object to update base values with. + :param path: Path to dump file to. """ + path.parent.mkdir(parents=True, exist_ok=True) + user_values = config_values.get("update_values", {}) + if base_file := config_values.get("base_file"): + config_obj = config_class(base_file) + config_obj.update_values(user_values) + config_obj.dereference() + config_obj.dump(path) + else: + config_class.dump_dict(cfg=user_values, path=path) + log.debug(f"Wrote config to {path}") + @property @abstractmethod - def output(self) -> None: + def _driver_config(self) -> Dict[str, Any]: """ - ??? + Returns the config block specific to this driver. """ + @property @abstractmethod - def requirements(self) -> None: + def _resources(self) -> Dict[str, Any]: """ - ??? + Returns configuration data for the FV3 runscript. """ - @abstractmethod - def resources(self) -> Mapping: + @property + def _runcmd(self) -> str: """ - Parses the config and returns a formatted dictionary for the batch script. + Returns the full command-line component invocation. """ + execution = self._driver_config.get("execution", {}) + mpiargs = execution.get("mpiargs", []) + components = [ + execution["mpicmd"], # MPI run program + *[str(x) for x in mpiargs], # MPI arguments + execution["executable"], # component executable name + ] + return " ".join(filter(None, components)) - @abstractmethod - def run(self, cycle: datetime) -> bool: + def _runscript( + self, + execution: List[str], + envcmds: Optional[List[str]] = None, + envvars: Optional[Dict[str, str]] = None, + scheduler: Optional[JobScheduler] = None, + ) -> str: """ - Run the NWP tool. + Returns a driver runscript. - :param cycle: The time stamp of the cycle to run. - :return: Did the driver exit with success status? + :param execution: Statements to execute. + :param envcmds: Shell commands to set up runtime environment. + :param envvars: Environment variables to set in runtime environment. + :param scheduler: A job-scheduler object. """ + # Render script sections into a template, remove any extraneous newlines related to elided + # sections, then return the resulting string. + template = """ + #!/bin/bash - def run_cmd(self) -> str: - """ - The command-line command to run the NWP tool. + {directives} - :return: Collated string that contains MPI command, runtime arguments, and exec name. - """ - components = [ - self._platform_config["mpicmd"], # MPI run program - *[ - str(x) for x in self._config.get("runtime_info", {}).get("mpi_args", []) - ], # MPI arguments - self._config["executable"], # NWP tool executable name - ] - return " ".join(filter(None, components)) + {envcmds} - @property - def scheduler(self) -> JobScheduler: - """ - The job scheduler specified by the platform information. + {envvars} - :return: The scheduler object + {execution} """ - return JobScheduler.get_scheduler(self.resources()) + rs = dedent(template).format( + directives="\n".join(scheduler.directives if scheduler else ""), + envcmds="\n".join(envcmds or []), + envvars="\n".join([f"export {k}={v}" for k, v in (envvars or {}).items()]), + execution="\n".join(execution), + ) + return re.sub(r"\n\n\n+", "\n\n", rs.strip()) @property - @abstractmethod - def schema_file(self) -> Path: - """ - The path to the file containing the schema to validate the config file against. + def _scheduler(self) -> JobScheduler: """ - - @staticmethod - def stage_files( - run_directory: Path, - files_to_stage: Dict[str, Union[list, str]], - link_files: bool = False, - dry_run: bool = False, - ) -> None: + Returns the job scheduler specified by the platform information. """ - Creates destination files in run directory and copies or links contents from the source path - provided. Source paths could be provided as a single path or a list of paths to be staged in - a common directory. - - :param run_directory: Path of desired run directory. - :param files_to_stage: File names in the run directory (keys) and their source paths - (values). - :param link_files: Whether to link or copy the files. - """ - link_or_copy = os.symlink if link_files else shutil.copyfile - for dst_rel_path, src_path_or_paths in files_to_stage.items(): - dst_path = run_directory / dst_rel_path - if isinstance(src_path_or_paths, list): - Driver.stage_files( - dst_path, - {os.path.basename(src): src for src in src_path_or_paths}, - link_files, - ) - else: - if dry_run: - msg = f"File {src_path_or_paths} would be staged as {dst_path}" - log.info(msg) - else: - msg = f"File {src_path_or_paths} staged as {dst_path}" - log.info(msg) - link_or_copy(src_path_or_paths, dst_path) # type: ignore - - # Private methods + return JobScheduler.get_scheduler(self._resources) - @staticmethod - def _create_user_updated_config( - config_class: Type[Config], config_values: dict, output_path: OptionalPath - ) -> None: + @abstractmethod + def _validate(self) -> None: """ - Create a config from a base file, user-provided values, of a combination of the two. - - :param config_class: The Config subclass matching the config type. - :param config_values: The configuration object to update base values with. - :param output_path: Optional path to dump file to. + Perform all necessary schema validation. """ - user_values = config_values.get("update_values", {}) - if base_file := config_values.get("base_file"): - config_obj = config_class(base_file) - config_obj.update_values(user_values) - config_obj.dereference() - config_obj.dump(output_path) - else: - config_class.dump_dict(cfg=user_values, path=output_path) - if output_path: - log.info(f"Wrote config to {output_path}") - def _validate(self) -> bool: + def _validate_one(self, schema_file: Path) -> None: """ - Validate the user-supplied config file. + Validate the config. - :return: Was the input configuration file valid against its schema? + :param schema_file: The schema file to validate the config against. + :raises: UWConfigError if config fails validation. """ - return validator.validate_yaml(config=self._config_file, schema_file=self.schema_file) + log.info("Validating config per %s", schema_file) + if not validator.validate_yaml(config=self._config, schema_file=schema_file): + raise UWConfigError("YAML validation errors") diff --git a/src/uwtools/drivers/forecast.py b/src/uwtools/drivers/forecast.py deleted file mode 100644 index 23d7e4382..000000000 --- a/src/uwtools/drivers/forecast.py +++ /dev/null @@ -1,295 +0,0 @@ -""" -Drivers for forecast models. -""" - -import sys -from collections.abc import Mapping -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Tuple - -from uwtools.config.formats.fieldtable import FieldTableConfig -from uwtools.config.formats.nml import NMLConfig -from uwtools.config.formats.yaml import YAMLConfig -from uwtools.drivers.driver import Driver -from uwtools.logging import log -from uwtools.scheduler import BatchScript -from uwtools.types import DefinitePath, ExistAct, OptionalPath -from uwtools.utils.file import handle_existing, resource_pathobj, validate_existing_action -from uwtools.utils.processing import execute - - -class FV3Forecast(Driver): - """ - A driver for the FV3 forecast model. - """ - - def __init__( - self, - config_file: DefinitePath, - dry_run: bool = False, - batch_script: OptionalPath = None, - ): - """ - Initialize the Forecast Driver. - """ - - super().__init__(config_file=config_file, dry_run=dry_run, batch_script=batch_script) - self._config = self._experiment_config["forecast"] - - # Public methods - - def batch_script(self) -> BatchScript: - """ - Prepare batch script contents for interaction with system scheduler. - - :return: The batch script object with all run commands needed for executing the program. - """ - pre_run = self._mpi_env_variables("\n") - batch_script = self.scheduler.batch_script - batch_script.append(pre_run) - batch_script.append(self.run_cmd()) - return batch_script - - @staticmethod - def create_directory_structure( - run_directory: DefinitePath, exist_act: str = ExistAct.delete, dry_run: bool = False - ) -> None: - """ - Collects the name of the desired run directory, and has an optional flag for what to do if - the run directory specified already exists. Creates the run directory and adds - subdirectories INPUT and RESTART. Verifies creation of all directories. - - :param run_directory: Path of desired run directory. - :param exist_act: Action when run directory exists: "delete" (default), "quit", or "rename" - """ - - validate_existing_action( - exist_act, valid_actions=[ExistAct.delete, ExistAct.quit, ExistAct.rename] - ) - - run_directory = Path(run_directory) - - # Exit program with error if caller specified the "quit" action. - - if exist_act == ExistAct.quit and run_directory.is_dir(): - log.critical(f"Option {exist_act} specified, exiting") - sys.exit(1) - - # Handle a potentially pre-existing directory appropriately. - - if dry_run and run_directory.is_dir(): - log.info(f"Would {exist_act} directory") - else: - handle_existing(run_directory, exist_act) - - # Create new run directory with two required subdirectories. - - for subdir in ("INPUT", "RESTART"): - path = run_directory / subdir - if dry_run: - log.info("Would create directory: %s", path) - else: - log.info("Creating directory: %s", path) - path.mkdir(parents=True) - - def create_field_table(self, output_path: OptionalPath) -> None: - """ - Uses the forecast config object to create a Field Table. - - :param output_path: Optional location of output field table. - """ - self._create_user_updated_config( - config_class=FieldTableConfig, - config_values=self._config.get("field_table", {}), - output_path=output_path, - ) - - def create_model_configure(self, output_path: OptionalPath) -> None: - """ - Uses the forecast config object to create a model_configure. - - :param output_path: Optional location of the output model_configure file. - """ - self._create_user_updated_config( - config_class=YAMLConfig, - config_values=self._config.get("model_configure", {}), - output_path=output_path, - ) - - def create_namelist(self, output_path: OptionalPath) -> None: - """ - Uses an object with user supplied values and an optional namelist base file to create an - output namelist file. Will "dereference" the base file. - - :param output_path: Optional location of output namelist. - """ - self._create_user_updated_config( - config_class=NMLConfig, - config_values=self._config.get("namelist", {}), - output_path=output_path, - ) - - def output(self) -> None: - """ - ??? - """ - - def prepare_directories(self) -> Path: - """ - Prepares the run directory and stages static and cycle-dependent files. - - :return: Path to the run directory. - """ - run_directory = Path(self._config["run_dir"]) - self.create_directory_structure(run_directory, ExistAct.delete, dry_run=self._dry_run) - self._prepare_config_files(run_directory) - self._config["cycle_dependent"].update(self._define_boundary_files()) - for file_category in ["static", "cycle_dependent"]: - self.stage_files( - run_directory, self._config[file_category], link_files=True, dry_run=self._dry_run - ) - return run_directory - - def requirements(self) -> None: - """ - ??? - """ - - def resources(self) -> Mapping: - """ - Parses the experiment configuration to provide the information needed for the batch script. - - :return: A formatted dictionary needed to create a batch script - """ - - return { - "account": self._experiment_config["platform"]["account"], - "scheduler": self._experiment_config["platform"]["scheduler"], - **self._config["jobinfo"], - } - - def run(self, cycle: datetime) -> bool: - """ - Runs FV3 either locally or via a batch-script submission. - - :param cycle: The forecast cycle to run. - :return: Did the batch submission or FV3 run exit with success status? - """ - status, output = ( - self._run_via_batch_submission() - if self._batch_script - else self._run_via_local_execution() - ) - if self._dry_run: - for line in output: - log.info(line) - return status - - @property - def schema_file(self) -> Path: - """ - The path to the file containing the schema to validate the config file against. - """ - return resource_pathobj("FV3Forecast.jsonschema") - - # Private methods - - def _boundary_hours(self, lbcs_config: Dict) -> tuple[int, int, int]: - """ - Prepares parameters to generate the lateral boundary condition (LBCS) forecast hours from an - external input data source, e.g. GFS, RAP, etc. - - :return: The offset hours between the cycle and the external input data, the hours between - LBC ingest, and the last hour of the external input data forecast - """ - offset = abs(lbcs_config["offset"]) - end_hour = self._config["length"] + offset + 1 - return offset, lbcs_config["interval_hours"], end_hour - - def _define_boundary_files(self) -> Dict[str, str]: - """ - Maps the prepared boundary conditions to the appropriate hours for the forecast. - - :return: A dict of boundary file names mapped to source input file paths - """ - boundary_files = {} - lbcs_config = self._experiment_config["preprocessing"]["lateral_boundary_conditions"] - boundary_file_path = lbcs_config["output_file_path"] - offset, interval, endhour = self._boundary_hours(lbcs_config) - tiles = [7] if self._config["domain"] == "global" else range(1, 7) - for tile in tiles: - for boundary_hour in range(offset, endhour, interval): - forecast_hour = boundary_hour - offset - link_name = f"INPUT/gfs_bndy.tile{tile}.{forecast_hour:03d}.nc" - boundary_file_path = boundary_file_path.format( - tile=tile, - forecast_hour=boundary_hour, - ) - boundary_files[link_name] = boundary_file_path - - return boundary_files - - def _mpi_env_variables(self, delimiter: str = " ") -> str: - """ - Set the environment variables needed for the MPI job. - - :return: A bash string of environment variables - """ - envvars = { - "KMP_AFFINITY": "scatter", - "OMP_NUM_THREADS": self._config.get("runtime_info", {}).get("threads", 1), - "OMP_STACKSIZE": "512m", - "MPI_TYPE_DEPTH": 20, - "ESMF_RUNTIME_COMPLIANCECHECK": "OFF:depth=4", - } - return delimiter.join([f"{k}={v}" for k, v in envvars.items()]) - - def _prepare_config_files(self, run_directory: Path) -> None: - """ - Collect all the configuration files needed for FV3. - """ - if self._dry_run: - for call in ("field_table", "model_configure", "input.nml"): - log.info(f"Would prepare: {run_directory}/{call}") - else: - self.create_field_table(run_directory / "field_table") - self.create_model_configure(run_directory / "model_configure") - self.create_namelist(run_directory / "input.nml") - - def _run_via_batch_submission(self) -> Tuple[bool, List[str]]: - """ - Prepares and submits a batch script. - - :return: A tuple containing the success status of submitting the job to the batch system, - and a list of strings that make up the batch script. - """ - run_directory = self.prepare_directories() - batch_script = self.batch_script() - batch_lines = ["Batch script:", *str(batch_script).split("\n")] - if self._dry_run: - return True, batch_lines - assert self._batch_script is not None - outpath = run_directory / self._batch_script - batch_script.dump(outpath) - return self.scheduler.submit_job(outpath), batch_lines - - def _run_via_local_execution(self) -> Tuple[bool, List[str]]: - """ - Collects the necessary MPI environment variables in order to construct full run command, - then executes said command. - - :return: A tuple containing a boolean of the success status of the FV3 run and a list of - strings that make up the full command line. - """ - run_directory = self.prepare_directories() - pre_run = self._mpi_env_variables(" ") - full_cmd = f"{pre_run} {self.run_cmd()}" - command_lines = ["Command:", *full_cmd.split("\n")] - if self._dry_run: - return True, command_lines - success, _ = execute(cmd=full_cmd, cwd=run_directory, log_output=True) - return success, command_lines - - -CLASSES = {"FV3": FV3Forecast} diff --git a/src/uwtools/drivers/fv3.py b/src/uwtools/drivers/fv3.py new file mode 100644 index 000000000..1ce5ee207 --- /dev/null +++ b/src/uwtools/drivers/fv3.py @@ -0,0 +1,323 @@ +""" +A driver for the FV3 model. +""" + +import os +import stat +from datetime import datetime +from pathlib import Path +from shutil import copy +from typing import Any, Dict + +from iotaa import asset, dryrun, external, task, tasks + +from uwtools.config.formats.fieldtable import FieldTableConfig +from uwtools.config.formats.nml import NMLConfig +from uwtools.config.formats.yaml import YAMLConfig +from uwtools.drivers.driver import Driver +from uwtools.logging import log +from uwtools.types import DefinitePath +from uwtools.utils.file import resource_pathobj +from uwtools.utils.processing import execute + + +class FV3(Driver): + """ + A driver for the FV3 model. + """ + + def __init__( + self, config_file: DefinitePath, cycle: datetime, dry_run: bool = False, batch: bool = False + ): + """ + The FV3 driver. + + :param config_file: Path to config file. + :param cycle: The forecast cycle. + :param dry_run: Run in dry-run mode? + :param batch: Run component via the batch system? + """ + super().__init__(config_file=config_file, dry_run=dry_run, batch=batch) + self._config.dereference(context={"cycle": cycle}) + if self._dry_run: + dryrun() + self._cycle = cycle + self._rundir = Path(self._driver_config["run_dir"]) + + # Workflow tasks + + @tasks + def boundary_files(self): + """ + The FV3 lateral boundary-condition files. + """ + yield self._taskname("lateral boundary condition files") + lbcs = self._driver_config["lateral_boundary_conditions"] + offset = abs(lbcs["offset"]) + endhour = self._driver_config["length"] + offset + 1 + interval = lbcs["interval_hours"] + symlinks = {} + for n in [7] if self._driver_config["domain"] == "global" else range(1, 7): + for boundary_hour in range(offset, endhour, interval): + target = Path(lbcs["path"].format(tile=n, forecast_hour=boundary_hour)) + linkname = ( + self._rundir / "INPUT" / f"gfs_bndy.tile{n}.{(boundary_hour - offset):03d}.nc" + ) + symlinks[target] = linkname + yield [self._symlink(target=t, linkname=l) for t, l in symlinks.items()] + + @task + def diag_table(self): + """ + The FV3 diag_table file. + """ + fn = "diag_table" + yield self._taskname(fn) + path = self._rundir / fn + yield asset(path, path.is_file) + yield None + if src := self._driver_config.get(fn): + path.parent.mkdir(parents=True, exist_ok=True) + copy(src=src, dst=path) + else: + log.warning("No '%s' defined in config", fn) + + @task + def field_table(self): + """ + The FV3 field_table file. + """ + fn = "field_table" + yield self._taskname(fn) + path = self._rundir / fn + yield asset(path, path.is_file) + yield None + self._create_user_updated_config( + config_class=FieldTableConfig, + config_values=self._driver_config["field_table"], + path=path, + ) + + @tasks + def files_copied(self): + """ + Files copied for FV3 run. + """ + yield self._taskname("files copied") + yield [ + self._filecopy(src=Path(src), dst=self._rundir / dst) + for dst, src in self._driver_config.get("files_to_copy", {}).items() + ] + + @tasks + def files_linked(self): + """ + Files linked for FV3 run. + """ + yield self._taskname("files linked") + yield [ + self._symlink(target=Path(target), linkname=self._rundir / linkname) + for linkname, target in self._driver_config.get("files_to_link", {}).items() + ] + + @task + def model_configure(self): + """ + The FV3 model_configure file. + """ + fn = "model_configure" + yield self._taskname(fn) + path = self._rundir / fn + yield asset(path, path.is_file) + yield None + self._create_user_updated_config( + config_class=YAMLConfig, + config_values=self._driver_config["model_configure"], + path=path, + ) + + @task + def namelist_file(self): + """ + The FV3 namelist file. + """ + fn = "input.nml" + yield self._taskname(fn) + path = self._rundir / fn + yield asset(path, path.is_file) + yield None + self._create_user_updated_config( + config_class=NMLConfig, + config_values=self._driver_config.get("namelist", {}), + path=path, + ) + + @tasks + def provisioned_run_directory(self): + """ + The run directory provisioned with all required content. + """ + yield self._taskname("provisioned run directory") + yield [ + self.boundary_files(), + self.diag_table(), + self.field_table(), + self.files_copied(), + self.files_linked(), + self.model_configure(), + self.namelist_file(), + self.restart_directory(), + self.runscript(), + ] + + @task + def restart_directory(self): + """ + The FV3 RESTART directory. + """ + yield self._taskname("RESTART directory") + path = self._rundir / "RESTART" + yield asset(path, path.is_dir) + yield None + path.mkdir(parents=True) + + @tasks + def run(self): + """ + FV3 run execution. + """ + yield self._taskname("run") + yield (self._run_via_batch_submission() if self._batch else self._run_via_local_execution()) + + @task + def runscript(self): + """ + A runscript suitable for submission to the scheduler. + """ + yield self._taskname("runscript") + path = self._runscript_path + yield asset(path, path.is_file) + yield None + envvars = { + "ESMF_RUNTIME_COMPLIANCECHECK": "OFF:depth=4", + "KMP_AFFINITY": "scatter", + "MPI_TYPE_DEPTH": 20, + "OMP_NUM_THREADS": self._driver_config.get("execution", {}).get("threads", 1), + "OMP_STACKSIZE": "512m", + } + envcmds = self._driver_config.get("execution", {}).get("envcmds", []) + execution = [self._runcmd, "test $? -eq 0 && touch %s/done" % self._rundir] + scheduler = self._scheduler if self._batch else None + path.parent.mkdir(parents=True, exist_ok=True) + rs = self._runscript( + envcmds=envcmds, envvars=envvars, execution=execution, scheduler=scheduler + ) + with open(path, "w", encoding="utf-8") as f: + print(rs, file=f) + os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC) + + # Private workflow tasks + + @external + def _file(self, path: Path): + """ + An existing file. + + :param path: Path to the file. + """ + yield "File %s" % path + yield asset(path, path.is_file) + + @task + def _filecopy(self, src: Path, dst: Path): + """ + A copy of an existing file. + + :param src: Path to the source file. + :param dst: Path to the destination file to create. + """ + yield "Copy %s -> %s" % (src, dst) + yield asset(dst, dst.is_file) + yield self._file(src) + copy(src, dst) + + @task + def _run_via_batch_submission(self): + """ + FV3 run Execution via the batch system. + """ + yield self._taskname("run via batch submission") + path = Path("%s.submit" % self._runscript_path) + yield asset(path, path.is_file) + yield self.provisioned_run_directory() + self._scheduler.submit_job(runscript=self._runscript_path, submit_file=path) + + @task + def _run_via_local_execution(self): + """ + FV3 run execution directly on the local system. + """ + yield self._taskname("run via local execution") + path = self._rundir / "done" + yield asset(path, path.is_file) + yield self.provisioned_run_directory() + cmd = "{x} >{x}.out 2>&1".format(x=self._runscript_path) + execute(cmd=cmd, cwd=self._rundir, log_output=True) + + @task + def _symlink(self, target: Path, linkname: Path): + """ + A symbolic link. + + :param target: The existing file or directory. + :param linkname: The symlink to create. + """ + yield "Link %s -> %s" % (linkname, target) + yield asset(linkname, linkname.exists) + yield self._file(target) + linkname.parent.mkdir(parents=True, exist_ok=True) + os.symlink(src=target, dst=linkname) + + # Private helper methods + + @property + def _driver_config(self) -> Dict[str, Any]: + """ + Returns the config block specific to this driver. + """ + driver_config: Dict[str, Any] = self._config["fv3"] + return driver_config + + @property + def _resources(self) -> Dict[str, Any]: + """ + Returns configuration data for the FV3 runscript. + """ + return { + "account": self._config["platform"]["account"], + "rundir": self._rundir, + "scheduler": self._config["platform"]["scheduler"], + **self._driver_config.get("execution", {}).get("batchargs", {}), + } + + @property + def _runscript_path(self) -> Path: + """ + Returns the path to the runscript. + """ + return self._rundir / "runscript" + + def _taskname(self, suffix: str) -> str: + """ + Returns a common tag for graph-task log messages. + + :param suffix: Log-string suffix. + """ + return "%s FV3 %s" % (self._cycle.strftime("%Y%m%d %HZ"), suffix) + + def _validate(self) -> None: + """ + Perform all necessary schema validation. + """ + for schema_file in ("fv3.jsonschema", "platform.jsonschema"): + self._validate_one(resource_pathobj(schema_file)) diff --git a/src/uwtools/resources/FV3Forecast.jsonschema b/src/uwtools/resources/fv3.jsonschema similarity index 87% rename from src/uwtools/resources/FV3Forecast.jsonschema rename to src/uwtools/resources/fv3.jsonschema index 768b90117..51a88af04 100644 --- a/src/uwtools/resources/FV3Forecast.jsonschema +++ b/src/uwtools/resources/fv3.jsonschema @@ -11,7 +11,7 @@ } }, "properties": { - "forecast": { + "fv3": { "additionalProperties": false, "properties": { "diag_table": { @@ -24,8 +24,39 @@ ], "type": "string" }, - "executable": { - "type": "string" + "execution": { + "additionalProperties": false, + "properties": { + "batchargs": { + "type": "object" + }, + "envcmds": { + "items": { + "type": "string" + }, + "type": "array" + }, + "executable": { + "type": "string" + }, + "mpiargs": { + "items": { + "type": "string" + }, + "type": "array" + }, + "mpicmd": { + "type": "string" + }, + "threads": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "executable" + ], + "type": "object" }, "field_table": { "additionalProperties": false, @@ -115,6 +146,28 @@ "files_to_link": { "$ref": "#/$defs/filesToStage" }, + "lateral_boundary_conditions": { + "additionalProperties": false, + "properties": { + "interval_hours": { + "minimum": 1, + "type": "integer" + }, + "offset": { + "minimum": 0, + "type": "integer" + }, + "path": { + "type": "string" + } + }, + "required": [ + "interval_hours", + "offset", + "path" + ], + "type": "object" + }, "length": { "minimum": 1, "type": "integer" @@ -198,91 +251,17 @@ }, "run_dir": { "type": "string" - }, - "runtime_info": { - "additionalProperties": false, - "properties": { - "mpi_args": { - "items": { - "type": "string" - }, - "type": "array" - }, - "threads": { - "minimum": 0, - "type": "integer" - } - }, - "type": "object" } }, "required": [ "domain", - "executable", + "execution", + "lateral_boundary_conditions", "length", "run_dir" ], "type": "object" }, - "platform": { - "additionalProperties": false, - "dependentRequired": { - "scheduler": [ - "account" - ] - }, - "properties": { - "account": { - "type": "string" - }, - "mpicmd": { - "type": "string" - }, - "scheduler": { - "enum": [ - "lsf", - "pbs", - "slurm" - ], - "type": "string" - } - }, - "required": [ - "mpicmd" - ], - "type": "object" - }, - "preprocessing": { - "additionalProperties": false, - "properties": { - "lateral_boundary_conditions": { - "additionalProperties": false, - "properties": { - "interval_hours": { - "minimum": 1, - "type": "integer" - }, - "offset": { - "minimum": 0, - "type": "integer" - }, - "output_file_path": { - "type": "string" - } - }, - "required": [ - "interval_hours", - "offset", - "output_file_path" - ], - "type": "object" - } - }, - "required": [ - "lateral_boundary_conditions" - ], - "type": "object" - }, "user": { "type": "object" } diff --git a/src/uwtools/resources/platform.jsonschema b/src/uwtools/resources/platform.jsonschema new file mode 100644 index 000000000..64206225d --- /dev/null +++ b/src/uwtools/resources/platform.jsonschema @@ -0,0 +1,29 @@ +{ + "properties": { + "platform": { + "additionalProperties": false, + "dependentRequired": { + "account": [ + "scheduler" + ], + "scheduler": [ + "account" + ] + }, + "properties": { + "account": { + "type": "string" + }, + "scheduler": { + "enum": [ + "lsf", + "pbs", + "slurm" + ], + "type": "string" + } + }, + "type": "object" + } + } +} diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 822790a3c..62057c5ac 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -344,6 +344,7 @@ def _config_validate(self, config: Union[dict, YAMLConfig, OptionalPath]) -> Non Validate the given YAML config. :param config: YAMLConfig object or path to YAML file (None => read stdin). + :raises: UWConfigError if config fails validation. """ schema_file = resource_pathobj("rocoto.jsonschema") ok = validate_yaml(schema_file=schema_file, config=config) diff --git a/src/uwtools/scheduler.py b/src/uwtools/scheduler.py index 6649ab851..0b882e7c2 100644 --- a/src/uwtools/scheduler.py +++ b/src/uwtools/scheduler.py @@ -1,346 +1,364 @@ """ -Job Scheduling. +Support for HPC schedulers. """ from __future__ import annotations -import re -from collections import UserDict, UserList +from abc import ABC, abstractmethod from collections.abc import Mapping +from copy import deepcopy +from dataclasses import dataclass, fields from pathlib import Path from typing import Any, Dict, List +from uwtools.exceptions import UWConfigError from uwtools.logging import log from uwtools.types import OptionalPath -from uwtools.utils.file import writable -from uwtools.utils.memory import Memory from uwtools.utils.processing import execute -NONEISH = [None, "", " ", "None", "none", False] -IGNORED_ATTRIBS = ["scheduler"] - -class RequiredAttribs: +class JobScheduler(ABC): """ - Key for required attributes. + An abstract class for interacting with HPC schedulers. """ - ACCOUNT = "account" - QUEUE = "queue" - WALLTIME = "walltime" - + def __init__(self, props: Dict[str, Any]): + self._props = {k: v for k, v in props.items() if k != "scheduler"} + self._validate_props() -class OptionalAttribs: - """ - Key for optional attributes. - """ - - CORES = "cores" - DEBUG = "debug" - EXCLUSIVE = "exclusive" - EXPORT = "export" - JOB_NAME = "jobname" - JOIN = "join" - MEMORY = "memory" - NODES = "nodes" - PARTITION = "partition" - PLACEMENT = "placement" - SHELL = "shell" - STDERR = "stderr" - STDOUT = "stdout" - TASKS_PER_NODE = "tasks_per_node" - THREADS = "threads" - - -class BatchScript(UserList): - """ - Represents a batch script to submit to a scheduler. - """ + # Public methods - def __str__(self): + @property + def directives(self) -> List[str]: """ - Returns string representation. + Returns resource-request scheduler directives. """ - shebang = "#!/bin/bash\n" - return str(shebang + self.content()) + pre, sep = self._prefix, self._directive_separator + ds = [] + for key, value in self._processed_props.items(): + if key in self._managed_directives: + switch = self._managed_directives[key] + ds.append( + "%s %s" % (pre, switch(value)) + if callable(switch) + else "%s %s%s%s" % (pre, switch, sep, value) + ) + else: + ds.append("%s %s%s%s" % (pre, key, sep, value)) + return sorted(ds) - def content(self, line_separator: str = "\n") -> str: + @staticmethod + def get_scheduler(props: Mapping) -> JobScheduler: """ - Returns the formatted content of the batch script. + Returns a configured job scheduler. - Parameters - ---------- - line_separator - The character or characters to join the content lines with + :param props: Configuration settings for job scheduler. + :return: A configured job scheduler. + :raises: UWConfigError if 'scheduler' is un- or mis-defined. """ - return line_separator.join(self) - - def dump(self, output_file: OptionalPath) -> None: + schedulers = {"slurm": Slurm, "pbs": PBS, "lsf": LSF} + if name := props.get("scheduler"): + log.debug("Getting '%s' scheduler", name) + if scheduler_class := schedulers.get(name): + return scheduler_class(props) # type: ignore + raise UWConfigError( + "Scheduler '%s' should be one of: %s" % (name, ", ".join(schedulers.keys())) + ) + raise UWConfigError(f"No 'scheduler' defined in {props}") + + def submit_job(self, runscript: Path, submit_file: OptionalPath = None) -> bool: """ - Write a batch script to an output location. + Submits a job to the scheduler. - :param output_file: Path to the file to write the batch script to + :param runscript: Path to the runscript. + :param submit_file: Path to file to write output of submit command to. + :return: Did the run exit with a success status? """ - with writable(output_file) as f: - print(str(self).strip(), file=f) - - -class JobScheduler(UserDict): - """ - A class for interacting with HPC batch schedulers. - """ - - _map: dict = {} - prefix = "" - key_value_separator = "=" - - def __init__(self, props): - super().__init__(props) - self.validate_props(props) + cmd = f"{self._submit_cmd} {runscript}" + if submit_file: + cmd += " 2>&1 | tee %s" % submit_file + success, _ = execute(cmd=cmd, cwd=f"{runscript.parent}") + return success - def __getattr__(self, name) -> Any: - if name in self: - return self[name] - raise AttributeError(name) + # Private methods - @staticmethod - def validate_props(props) -> None: + @property + @abstractmethod + def _directive_separator(self) -> str: """ - Raises ValueError if invalid. + Returns the character used to separate directive keys and values. """ - members = [ - getattr(RequiredAttribs, attr) - for attr in dir(RequiredAttribs) - if not callable(getattr(RequiredAttribs, attr)) and not attr.startswith("__") - ] - if diff := [x for x in members if x not in props]: - raise ValueError(f"Missing required attributes: [{', '.join(diff)}]") - def pre_process(self) -> Dict[str, Any]: + @property + @abstractmethod + def _managed_directives(self) -> Dict[str, Any]: """ - Pre-process attributes before converting to batch script. + Returns a mapping from canonical names to scheduler-specific CLI switches. """ - return self.data - @staticmethod - def post_process(items: List[str]) -> List[str]: + @property + @abstractmethod + def _prefix(self) -> str: """ - Post process attributes before converting to batch script. + Returns the scheduler's resource-request prefix. """ - return [re.sub(r"\s{0,}\=\s{0,}", "=", x, count=0, flags=0) for x in items] @property - def batch_script(self) -> BatchScript: + def _processed_props(self) -> Dict[str, Any]: """ - Returns the batch script to be fed to external scheduler. + Pre-process directives before converting to runscript. """ + return self._props - sanitized_attribs = self.pre_process() - - known = [] - for key, value in sanitized_attribs.items(): - if key in self._map and key not in IGNORED_ATTRIBS: - scheduler_flag = ( - self._map[key](value) if callable(self._map[key]) else self._map[key] - ) - scheduler_value = "" if callable(self._map[key]) else value - key_value_separator = "" if callable(self._map[key]) else self.key_value_separator - directive = f"{self.prefix} {scheduler_flag}{key_value_separator}{scheduler_value}" - known.append(directive.strip()) - - unknown = [ - f"{self.prefix} {key}{self.key_value_separator}{value}".strip() - for (key, value) in sanitized_attribs.items() - if key not in self._map and value not in NONEISH and key not in IGNORED_ATTRIBS - ] + @property + @abstractmethod + def _submit_cmd(self) -> str: + """ + Returns the scheduler's job-submit executable name. + """ - flags = [ - f"{self.prefix} {key}".strip() - for (key, value) in sanitized_attribs.items() - if value in NONEISH - ] + def _validate_props(self) -> None: + """ + Validate scheduler-configuration properties. - processed = self.post_process(known + unknown + flags) + :raises: UWConfigError if required props are missing. + """ + if missing := [ + getattr(_DirectivesRequired, x.name) + for x in fields(_DirectivesRequired) + if getattr(_DirectivesRequired, x.name) not in self._props + ]: + raise UWConfigError("Missing required directives: %s" % ", ".join(missing)) - # Sort batch directives to normalize output w.r.t. potential differences - # in ordering of input dicts. - return BatchScript(sorted(processed)) +class LSF(JobScheduler): + """ + Represents a LSF based scheduler. + """ - @staticmethod - def get_scheduler(props: Mapping) -> JobScheduler: + @property + def _directive_separator(self) -> str: """ - Returns the appropriate scheduler. + Returns the character used to separate directive keys and values. + """ + return " " - Parameters - ---------- - props - Must contain a 'scheduler' key or a KeyError will be raised + @property + def _managed_directives(self) -> Dict[str, Any]: """ - if "scheduler" not in props: - raise KeyError(f"No scheduler defined in props: [{', '.join(props.keys())}]") - name = props["scheduler"] - log.debug("Getting '%s' scheduler", name) - schedulers = {"slurm": Slurm, "pbs": PBS, "lsf": LSF} - try: - scheduler = schedulers[name] - except KeyError as error: - raise KeyError( - f"{name} is not a supported scheduler" - + "Currently supported schedulers are:\n" - + f'{" | ".join(schedulers.keys())}"' - ) from error - return scheduler(props) - - def submit_job(self, script_path: Path) -> bool: + Returns a mapping from canonical names to scheduler-specific CLI switches. """ - Submits a job to the scheduler. + return { + _DirectivesOptional.JOB_NAME: "-J", + _DirectivesOptional.MEMORY: lambda x: f"-R rusage[mem={x}]", + _DirectivesOptional.NODES: lambda x: f"-n {x}", + _DirectivesOptional.QUEUE: "-q", + _DirectivesOptional.SHELL: "-L", + _DirectivesOptional.STDOUT: "-o", + _DirectivesOptional.TASKS_PER_NODE: lambda x: f"-R span[ptile={x}]", + _DirectivesOptional.THREADS: lambda x: f"-R affinity[core({x})]", + _DirectivesRequired.ACCOUNT: "-P", + _DirectivesRequired.WALLTIME: "-W", + } - :param script_path: Path to the batch script. - :return: Did the run exit with a success status? + @property + def _prefix(self) -> str: """ - success, _ = execute( - cmd=f"{self.submit_command} {script_path}", cwd=f"{script_path.parent}" - ) - return success - + Returns the scheduler's resource-request prefix. + """ + return "#BSUB" -class Slurm(JobScheduler): - """ - Represents a Slurm based scheduler. - """ + @property + def _processed_props(self) -> Dict[str, Any]: + props = deepcopy(self._props) + props[_DirectivesOptional.THREADS] = props.get(_DirectivesOptional.THREADS, 1) + return props - prefix = "#SBATCH" - submit_command = "sbatch" - - _map = { - RequiredAttribs.ACCOUNT: "--account", - RequiredAttribs.QUEUE: "--qos", - RequiredAttribs.WALLTIME: "--time", - OptionalAttribs.CORES: "--ntasks", - OptionalAttribs.EXCLUSIVE: lambda x: "--exclusive", - OptionalAttribs.EXPORT: "--export", - OptionalAttribs.JOB_NAME: "--job-name", - OptionalAttribs.MEMORY: "--mem", - OptionalAttribs.NODES: "--nodes", - OptionalAttribs.PARTITION: "--partition", - OptionalAttribs.STDERR: "--error", - OptionalAttribs.STDOUT: "--output", - OptionalAttribs.TASKS_PER_NODE: "--ntasks-per-node", - OptionalAttribs.THREADS: "--cpus-per-task", - } + @property + def _submit_cmd(self) -> str: + """ + Returns the scheduler's job-submit executable name. + """ + return "bsub" class PBS(JobScheduler): """ - Represents a PBS based scheduler. + Represents the PBS scheduler. """ - prefix = "#PBS" - key_value_separator = " " - submit_command = "qsub" - - _map = { - RequiredAttribs.ACCOUNT: "-A", - OptionalAttribs.NODES: lambda x: f"-l select={x}", - RequiredAttribs.QUEUE: "-q", - OptionalAttribs.TASKS_PER_NODE: "mpiprocs", - RequiredAttribs.WALLTIME: "-l walltime=", - OptionalAttribs.DEBUG: lambda x: f"-l debug={str(x).lower()}", - OptionalAttribs.JOB_NAME: "-N", - OptionalAttribs.MEMORY: "mem", - OptionalAttribs.SHELL: "-S", - OptionalAttribs.STDOUT: "-o", - OptionalAttribs.THREADS: "ompthreads", - } - - def pre_process(self) -> Dict[str, Any]: - output = self.data - output.update(self._select(output)) - output.update(self._placement(output)) - - output.pop(OptionalAttribs.TASKS_PER_NODE, None) - output.pop(OptionalAttribs.NODES, None) - output.pop(OptionalAttribs.THREADS, None) - output.pop(OptionalAttribs.MEMORY, None) - output.pop("exclusive", None) - output.pop("placement", None) - output.pop("select", None) - return dict(output) + @property + def _directive_separator(self) -> str: + """ + Returns the character used to separate directive keys and values. + """ + return " " - def _select(self, items: Dict[str, Any]) -> Dict[str, Any]: + @property + def _managed_directives(self) -> Dict[str, Any]: """ - Select logic. + Returns a mapping from canonical names to scheduler-specific CLI switches. """ - total_nodes = items.get(OptionalAttribs.NODES, "") - tasks_per_node = items.get(OptionalAttribs.TASKS_PER_NODE, "") - # Set default threads=1 to address job variability with PBS - threads = items.get(OptionalAttribs.THREADS, 1) - memory = items.get(OptionalAttribs.MEMORY, "") - select = [ - f"{total_nodes}", - f"{self._map[OptionalAttribs.TASKS_PER_NODE]}={tasks_per_node}", - f"{self._map[OptionalAttribs.THREADS]}={threads}", - f"ncpus={int(tasks_per_node) * int(threads)}", - ] - if memory not in NONEISH: - select.append(f"{self._map[OptionalAttribs.MEMORY]}={memory}") - items["-l select="] = ":".join(select) - return items + return { + _DirectivesOptional.DEBUG: lambda x: f"-l debug={str(x).lower()}", + _DirectivesOptional.JOB_NAME: "-N", + _DirectivesOptional.MEMORY: "mem", + _DirectivesOptional.NODES: lambda x: f"-l select={x}", + _DirectivesOptional.QUEUE: "-q", + _DirectivesOptional.SHELL: "-S", + _DirectivesOptional.STDOUT: "-o", + _DirectivesOptional.TASKS_PER_NODE: "mpiprocs", + _DirectivesOptional.THREADS: "ompthreads", + _DirectivesRequired.ACCOUNT: "-A", + _DirectivesRequired.WALLTIME: "-l walltime=", + } @staticmethod def _placement(items: Dict[str, Any]) -> Dict[str, Any]: """ Placement logic. """ - exclusive = items.get(OptionalAttribs.EXCLUSIVE, "") - placement = items.get(OptionalAttribs.PLACEMENT, "") - if all([exclusive in NONEISH, placement in NONEISH]): + exclusive = items.get(_DirectivesOptional.EXCLUSIVE) + placement = items.get(_DirectivesOptional.PLACEMENT) + if not exclusive and not placement: return items output = [] - if placement not in NONEISH: + if placement: output.append(str(placement)) - if exclusive not in NONEISH: + if exclusive: output.append("excl") if len(output) > 0: items["-l place="] = ":".join(output) return items + @property + def _prefix(self) -> str: + """ + Returns the scheduler's resource-request prefix. + """ + return "#PBS" + + @property + def _processed_props(self) -> Dict[str, Any]: + props = self._props + props.update(self._select(props)) + props.update(self._placement(props)) + props.pop(_DirectivesOptional.TASKS_PER_NODE, None) + props.pop(_DirectivesOptional.NODES, None) + props.pop(_DirectivesOptional.THREADS, None) + props.pop(_DirectivesOptional.MEMORY, None) + props.pop("exclusive", None) + props.pop("placement", None) + props.pop("select", None) + return dict(props) + + def _select(self, items: Dict[str, Any]) -> Dict[str, Any]: + """ + Select logic. + """ + select = [] + if nodes := items.get(_DirectivesOptional.NODES): + select.append(str(nodes)) + if tasks_per_node := items.get(_DirectivesOptional.TASKS_PER_NODE): + select.append( + f"{self._managed_directives[_DirectivesOptional.TASKS_PER_NODE]}={tasks_per_node}" + ) + threads = items.get(_DirectivesOptional.THREADS, 1) + select.append(f"{self._managed_directives[_DirectivesOptional.THREADS]}={threads}") + if tasks_per_node: + select.append(f"ncpus={int(tasks_per_node) * int(threads)}") + if memory := items.get(_DirectivesOptional.MEMORY): + select.append(f"{self._managed_directives[_DirectivesOptional.MEMORY]}={memory}") + items["-l select="] = ":".join(select) + return items + + @property + def _submit_cmd(self) -> str: + """ + Returns the scheduler's job-submit executable name. + """ + return "qsub" -class LSF(JobScheduler): + +class Slurm(JobScheduler): """ - Represents a LSF based scheduler. + Represents the Slurm scheduler. """ - prefix = "#BSUB" - key_value_separator = " " - submit_command = "bsub" - - _map = { - RequiredAttribs.ACCOUNT: "-P", - OptionalAttribs.NODES: lambda x: f"-n {x}", - RequiredAttribs.QUEUE: "-q", - OptionalAttribs.TASKS_PER_NODE: lambda x: f"-R span[ptile={x}]", - RequiredAttribs.WALLTIME: "-W", - OptionalAttribs.JOB_NAME: "-J", - OptionalAttribs.MEMORY: lambda x: f"-R rusage[mem={x}]", - OptionalAttribs.SHELL: "-L", - OptionalAttribs.STDOUT: "-o", - OptionalAttribs.THREADS: lambda x: f"-R affinity[core({x})]", - } - - def pre_process(self) -> Dict[str, Any]: - items = self.data - # LSF requires threads to be set (if None is provided, default to 1) - items[OptionalAttribs.THREADS] = items.get(OptionalAttribs.THREADS, 1) - nodes = items.get(OptionalAttribs.NODES, "") - tasks_per_node = items.get(OptionalAttribs.TASKS_PER_NODE, "") - - memory = items.get(OptionalAttribs.MEMORY, None) - if memory is not None: - mem_value = Memory(memory).convert("KB") - items[self._map[OptionalAttribs.MEMORY](mem_value)] = "" - - items[OptionalAttribs.NODES] = int(tasks_per_node) * int(nodes) - items.pop(OptionalAttribs.MEMORY, None) - return items + @property + def _managed_directives(self) -> Dict[str, Any]: + """ + Returns a mapping from canonical names to scheduler-specific CLI switches. + """ + return { + _DirectivesOptional.CORES: "--ntasks", + _DirectivesOptional.EXCLUSIVE: lambda _: "--exclusive", + _DirectivesOptional.EXPORT: "--export", + _DirectivesOptional.JOB_NAME: "--job-name", + _DirectivesOptional.MEMORY: "--mem", + _DirectivesOptional.NODES: "--nodes", + _DirectivesOptional.PARTITION: "--partition", + _DirectivesOptional.QUEUE: "--qos", + _DirectivesOptional.RUNDIR: "--chdir", + _DirectivesOptional.STDERR: "--error", + _DirectivesOptional.STDOUT: "--output", + _DirectivesOptional.TASKS_PER_NODE: "--ntasks-per-node", + _DirectivesOptional.THREADS: "--cpus-per-task", + _DirectivesRequired.ACCOUNT: "--account", + _DirectivesRequired.WALLTIME: "--time", + } + + @property + def _directive_separator(self) -> str: + """ + Returns the character used to separate directive keys and values. + """ + return "=" + + @property + def _prefix(self) -> str: + """ + Returns the scheduler's resource-request prefix. + """ + return "#SBATCH" + + @property + def _submit_cmd(self) -> str: + """ + Returns the scheduler's job-submit executable name. + """ + return "sbatch" + + +@dataclass(frozen=True) +class _DirectivesOptional: + """ + Keys for optional directives. + """ + + CORES: str = "cores" + DEBUG: str = "debug" + EXCLUSIVE: str = "exclusive" + EXPORT: str = "export" + JOB_NAME: str = "jobname" + MEMORY: str = "memory" + NODES: str = "nodes" + PARTITION: str = "partition" + PLACEMENT: str = "placement" + QUEUE: str = "queue" + RUNDIR: str = "rundir" + SHELL: str = "shell" + STDERR: str = "stderr" + STDOUT: str = "stdout" + TASKS_PER_NODE: str = "tasks_per_node" + THREADS: str = "threads" + + +@dataclass(frozen=True) +class _DirectivesRequired: + """ + Keys for required directives. + """ + + ACCOUNT: str = "account" + WALLTIME: str = "walltime" diff --git a/src/uwtools/tests/api/test_forecast.py b/src/uwtools/tests/api/test_forecast.py deleted file mode 100644 index 14f3057b3..000000000 --- a/src/uwtools/tests/api/test_forecast.py +++ /dev/null @@ -1,25 +0,0 @@ -# pylint: disable=missing-function-docstring,protected-access - -import datetime as dt -from unittest.mock import Mock, patch - -from uwtools.api import forecast - - -def test_run(): - SomeModel = Mock() - cycle = dt.datetime(2023, 11, 22, 12) - with patch.dict(forecast._CLASSES, {"foo": SomeModel}): - forecast.run( - model="foo", - cycle=cycle, - config_file="bar", - batch_script="baz", - dry_run=False, - ) - SomeModel.assert_called_once_with( - batch_script="baz", - config_file="bar", - dry_run=False, - ) - SomeModel().run.assert_called_once_with(cycle=cycle) diff --git a/src/uwtools/tests/api/test_fv3.py b/src/uwtools/tests/api/test_fv3.py new file mode 100644 index 000000000..d89c80919 --- /dev/null +++ b/src/uwtools/tests/api/test_fv3.py @@ -0,0 +1,44 @@ +# pylint: disable=missing-function-docstring,protected-access + +import datetime as dt +from unittest.mock import patch + +from iotaa import external, task, tasks + +from uwtools.api import fv3 + + +@external +def t1(): + "@external t1" + + +@task +def t2(): + "@task t2" + + +@tasks +def t3(): + "@tasks t3" + + +def test_execute(): + args: dict = { + "config_file": "config.yaml", + "cycle": dt.datetime.utcnow(), + "batch": False, + "dry_run": True, + } + with patch.object(fv3, "FV3") as FV3: + assert fv3.execute(**args, task="foo") is True + FV3.assert_called_once_with(**args) + FV3().foo.assert_called_once_with() + + +def test_tasks(): + with patch.object(fv3, "FV3") as FV3: + FV3.t1 = t1 + FV3.t2 = t2 + FV3.t3 = t3 + assert fv3.tasks() == {"t2": "@task t2", "t3": "@tasks t3", "t1": "@external t1"} diff --git a/src/uwtools/tests/config/formats/test_base.py b/src/uwtools/tests/config/formats/test_base.py index 3290a0faa..6a6971d63 100644 --- a/src/uwtools/tests/config/formats/test_base.py +++ b/src/uwtools/tests/config/formats/test_base.py @@ -2,7 +2,7 @@ """ Tests for the uwtools.config.base module. """ - +import datetime as dt import logging import os from unittest.mock import patch @@ -124,12 +124,19 @@ def test_dereference(tmp_path): # - Initially-unrenderable values do not cause errors. # - Initially-unrenderable values may be rendered via iteration. # - Finally-unrenderable values do not cause errors and are returned unmodified. + # - Tagged scalars in collections are handled correctly. log.setLevel(logging.DEBUG) yaml = """ a: !int '{{ b.c + 11 }}' b: c: !int '{{ N | int + 11 }}' d: '{{ X }}' +e: + - !int '88' + - !float '3.14' +f: + f1: !int '88' + f2: !float '3.14' """.strip() path = tmp_path / "config.yaml" with open(path, "w", encoding="utf-8") as f: @@ -137,7 +144,24 @@ def test_dereference(tmp_path): config = YAMLConfig(path) with patch.dict(os.environ, {"N": "55"}, clear=True): config.dereference() - assert config == {"a": 77, "b": {"c": 66}, "d": "{{ X }}"} + assert config == { + "a": 77, + "b": {"c": 66}, + "d": "{{ X }}", + "e": [88, 3.14], + "f": {"f1": 88, "f2": 3.14}, + } + + +def test_derefernce_context_override(tmp_path): + log.setLevel(logging.DEBUG) + yaml = "file: gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc" + path = tmp_path / "config.yaml" + with open(path, "w", encoding="utf-8") as f: + print(yaml, file=f) + config = YAMLConfig(path) + config.dereference(context={"cycle": dt.datetime(2024, 2, 12, 6)}) + assert config["file"] == "gfs.t06z.atmanl.nc" @pytest.mark.parametrize("fmt2", [FORMAT.ini, FORMAT.nml, FORMAT.sh]) diff --git a/src/uwtools/tests/config/formats/test_ini.py b/src/uwtools/tests/config/formats/test_ini.py index 4cc3bf32c..6566e4692 100644 --- a/src/uwtools/tests/config/formats/test_ini.py +++ b/src/uwtools/tests/config/formats/test_ini.py @@ -1,4 +1,4 @@ -# pylint: disable=duplicate-code,missing-function-docstring +# pylint: disable=missing-function-docstring """ Tests for uwtools.config.formats.ini module. """ diff --git a/src/uwtools/tests/config/formats/test_nml.py b/src/uwtools/tests/config/formats/test_nml.py index 0d2fa2e85..e15f6f7e4 100644 --- a/src/uwtools/tests/config/formats/test_nml.py +++ b/src/uwtools/tests/config/formats/test_nml.py @@ -1,20 +1,43 @@ -# pylint: disable=duplicate-code,missing-function-docstring +# pylint: disable=duplicate-code,missing-function-docstring,redefined-outer-name """ Tests for uwtools.config.formats.nml module. """ import filecmp -from pytest import raises +import f90nml # type: ignore +from pytest import fixture, raises from uwtools.config.formats.nml import NMLConfig from uwtools.exceptions import UWConfigError from uwtools.tests.support import fixture_path from uwtools.utils.file import FORMAT +# Fixtures + + +@fixture +def data(): + return {"nml": {"key": "val"}} + + # Tests +def test_dump_dict_dict(data, tmp_path): + path = tmp_path / "a.nml" + NMLConfig.dump_dict(cfg=data, path=path) + nml = f90nml.read(path) + assert nml == data + + +def test_dump_dict_Namelist(data, tmp_path): + path = tmp_path / "a.nml" + NMLConfig.dump_dict(cfg=f90nml.Namelist(data), path=path) + nml = f90nml.read(path) + assert nml == data + + def test_get_format(): assert NMLConfig.get_format() == FORMAT.nml diff --git a/src/uwtools/tests/config/formats/test_sh.py b/src/uwtools/tests/config/formats/test_sh.py index 57fae5440..f24a8816e 100644 --- a/src/uwtools/tests/config/formats/test_sh.py +++ b/src/uwtools/tests/config/formats/test_sh.py @@ -1,4 +1,4 @@ -# pylint: disable=duplicate-code,missing-function-docstring +# pylint: disable=missing-function-docstring """ Tests for uwtools.config.formats.sh module. """ diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index ae04b59d4..9192b773e 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -71,6 +71,12 @@ def validate(template): # Tests +def test_dereference_key_expression(): + assert jinja2.dereference(val={"{{ fruit }}": "red"}, context={"fruit": "apple"}) == { + "apple": "red" + } + + def test_dereference_local_values(): # Rendering can use values from the local contents of the enclosing dict, but are shadowed by # values from the top-level context object. diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index 2cd41a54d..637595edd 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -1,102 +1,164 @@ -# pylint: disable=missing-function-docstring,redefined-outer-name +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name """ Tests for uwtools.drivers.driver module. """ - -import datetime -import logging -from collections.abc import Mapping +import json from pathlib import Path -from unittest.mock import patch +from textwrap import dedent +from typing import Any, Dict +from unittest.mock import Mock, patch import pytest -from pytest import fixture +import yaml +from pytest import fixture, raises + +from uwtools.config.formats.yaml import YAMLConfig +from uwtools.drivers import driver +from uwtools.exceptions import UWConfigError -from uwtools.drivers.driver import Driver -from uwtools.logging import log -from uwtools.tests.support import logged +# Helpers -class ConcreteDriver(Driver): +class ConcreteDriver(driver.Driver): """ Driver subclass for testing purposes. """ - def batch_script(self): - pass + @property + def _driver_config(self) -> Dict[str, Any]: + return self._config.data - def output(self): - pass + @property + def _resources(self) -> Dict[str, Any]: + return {"some": "resource"} - def requirements(self): + def _validate(self) -> None: pass - def resources(self) -> Mapping: - return {} - def run(self, cycle: datetime.date) -> bool: - return True +def write(path, s): + with open(path, "w", encoding="utf-8") as f: + json.dump(s, f) + return path - def run_cmd(self, *args): - pass - @property - def schema_file(self) -> Path: - return Path() +# Fixtures @fixture -def configs(): - config_good = """ -platform: - WORKFLOW_MANAGER: rocoto -""" - config_bad = """ -platform: - WORKFLOW_MANAGER: 20 -""" - return config_good, config_bad +def driver_bad(tmp_path): + cf = write(tmp_path / "bad.yaml", {"base_file": 88}) + return ConcreteDriver(config_file=cf, dry_run=True, batch=True) @fixture -def schema(): - return """ -{ - "title": "workflow config", - "description": "This document is to validate user-defined FV3 forecast config files", - "type": "object", - "properties": { - "platform": { - "description": "attributes of the platform", - "type": "object", - "properties": { - "WORKFLOW_MANAGER": { - "type": "string", - "enum": [ - "rocoto", - "none" - ] - } - } - } - } -} -""".strip() - - -@pytest.mark.parametrize("valid", [True, False]) -def test_validation(caplog, configs, schema, tmp_path, valid): - config_good, config_bad = configs - config_file = str(tmp_path / "config.yaml") - with open(config_file, "w", encoding="utf-8") as f: - print(config_good if valid else config_bad, file=f) - schema_file = str(tmp_path / "test.jsonschema") - with open(schema_file, "w", encoding="utf-8") as f: - print(schema, file=f) - with patch.object(ConcreteDriver, "schema_file", new=schema_file): - log.setLevel(logging.INFO) - ConcreteDriver(config_file=config_file) - if valid: - assert logged(caplog, "0 UW schema-validation errors found") - else: - assert logged(caplog, "2 UW schema-validation errors found") +def driver_good(tmp_path): + cf = write( + tmp_path / "good.yaml", + { + "base_file": str(write(tmp_path / "base.yaml", {"a": 11, "b": 22})), + "execution": {"executable": "qux", "mpiargs": ["bar", "baz"], "mpicmd": "foo"}, + "update_values": {"a": 33}, + }, + ) + return ConcreteDriver(config_file=cf, dry_run=True, batch=True) + + +@fixture +def schema(tmp_path): + return write( + tmp_path / "a.jsonschema", + { + "properties": {"base_file": {"type": "string"}, "update_values": {"type": "object"}}, + "type": "object", + }, + ) + + +def test_Driver(driver_good): + assert Path(driver_good._config["base_file"]).name == "base.yaml" + assert driver_good._dry_run is True + assert driver_good._batch is True + + +@pytest.mark.parametrize( + "base_file,update_values,expected", + [ + (False, False, {}), + (False, True, {"a": 33}), + (True, False, {"a": 11, "b": 22}), + (True, True, {"a": 33, "b": 22}), + ], +) +def test_Driver__create_user_updated_config_base_file( + base_file, driver_good, expected, tmp_path, update_values +): + path = tmp_path / "updated.yaml" + cv = driver_good._config + if not base_file: + del cv["base_file"] + if not update_values: + del cv["update_values"] + ConcreteDriver._create_user_updated_config(config_class=YAMLConfig, config_values=cv, path=path) + with open(path, "r", encoding="utf-8") as f: + updated = yaml.safe_load(f) + assert updated == expected + + +def test_Driver__runcmd(driver_good): + assert driver_good._runcmd == "foo bar baz qux" + + +def test_Driver__runscript(driver_good): + expected = """ + #!/bin/bash + + #DIR --d1 + #DIR --d2 + + cmd1 + cmd2 + + export VAR1=1 + export VAR2=2 + + foo + bar + """ + scheduler = Mock(directives=["#DIR --d1", "#DIR --d2"]) + assert ( + driver_good._runscript( + execution=["foo", "bar"], + envcmds=["cmd1", "cmd2"], + envvars={"VAR1": 1, "VAR2": 2}, + scheduler=scheduler, + ) + == dedent(expected).strip() + ) + + +def test_Driver__runscript_execution_only(driver_good): + expected = """ + #!/bin/bash + + foo + bar + """ + assert driver_good._runscript(execution=["foo", "bar"]) == dedent(expected).strip() + + +def test_Driver__scheduler(driver_good): + with patch.object(driver, "JobScheduler") as JobScheduler: + scheduler = JobScheduler.get_scheduler() + assert driver_good._scheduler == scheduler + JobScheduler.get_scheduler.assert_called_with(driver_good._resources) + + +def test_driver__validate_one_no(driver_bad, schema): + with raises(UWConfigError) as e: + driver_bad._validate_one(schema) + assert str(e.value) == "YAML validation errors" + + +def test_Driver__validate_one_ok(driver_good, schema): + driver_good._validate_one(schema) diff --git a/src/uwtools/tests/drivers/test_forecast.py b/src/uwtools/tests/drivers/test_forecast.py deleted file mode 100644 index 41241f2d4..000000000 --- a/src/uwtools/tests/drivers/test_forecast.py +++ /dev/null @@ -1,719 +0,0 @@ -# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name -""" -Tests for forecast driver. -""" -import datetime as dt -import logging -import os -from functools import partial -from pathlib import Path -from unittest.mock import ANY, patch - -import pytest -from pytest import fixture, raises - -from uwtools import scheduler -from uwtools.config.formats.yaml import YAMLConfig -from uwtools.drivers import forecast -from uwtools.drivers.driver import Driver -from uwtools.drivers.forecast import FV3Forecast -from uwtools.logging import log -from uwtools.tests.support import compare_files, fixture_path, logged, validator, with_del, with_set -from uwtools.types import ExistAct - - -def test_batch_script(): - expected = """ -#SBATCH --account=user_account -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --qos=batch -#SBATCH --time=00:01:00 -KMP_AFFINITY=scatter -OMP_NUM_THREADS=1 -OMP_STACKSIZE=512m -MPI_TYPE_DEPTH=20 -ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4 -srun --export=NONE test_exec.py -""".strip() - config_file = fixture_path("forecast.yaml") - with patch.object(Driver, "_validate", return_value=True): - forecast = FV3Forecast(config_file=config_file) - assert forecast.batch_script().content() == expected - - -def test_schema_file(): - """ - Tests that the schema is properly defined with a file value. - """ - - config_file = fixture_path("forecast.yaml") - with patch.object(Driver, "_validate", return_value=True): - forecast = FV3Forecast(config_file=config_file) - - path = Path(forecast.schema_file) - assert path.is_file() - - -def test_create_model_configure(tmp_path): - """ - Test that providing a YAML base input file and a config file will create and update YAML config - file. - """ - - config_file = fixture_path("fruit_config_similar_for_fcst.yaml") - base_file = fixture_path("fruit_config.yaml") - fcst_config_file = tmp_path / "fcst.yml" - - fcst_config = YAMLConfig(config_file) - fcst_config["forecast"]["model_configure"]["base_file"] = base_file - fcst_config.dump(fcst_config_file) - - output_file = (tmp_path / "test_config_from_yaml.yaml").as_posix() - with patch.object(FV3Forecast, "_validate", return_value=True): - forecast_obj = FV3Forecast(config_file=fcst_config_file) - forecast_obj.create_model_configure(output_file) - expected = YAMLConfig(base_file) - expected.update_values(YAMLConfig(config_file)["forecast"]["model_configure"]["update_values"]) - expected_file = tmp_path / "expected_yaml.yaml" - expected.dump(expected_file) - assert compare_files(expected_file, output_file) - - -def test_create_directory_structure(tmp_path): - """ - Tests create_directory_structure method given a directory. - """ - - rundir = tmp_path / "rundir" - - # Test delete behavior when run directory does not exist. - FV3Forecast.create_directory_structure(rundir, ExistAct.delete) - assert (rundir / "RESTART").is_dir() - - # Create a file in the run directory. - test_file = rundir / "test.txt" - test_file.touch() - assert test_file.is_file() - - # Test delete behavior when run directory exists. Test file should be gone - # since old run directory was deleted. - FV3Forecast.create_directory_structure(rundir, ExistAct.delete) - assert (rundir / "RESTART").is_dir() - assert not test_file.is_file() - - # Test rename behavior when run directory exists. - FV3Forecast.create_directory_structure(rundir, ExistAct.rename) - copy_directory = next(tmp_path.glob("%s_*" % rundir.name)) - assert (copy_directory / "RESTART").is_dir() - - # Test quit behavior when run directory exists. - with raises(SystemExit) as pytest_wrapped_e: - FV3Forecast.create_directory_structure(rundir, ExistAct.quit) - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 - - -@fixture -def create_field_table_update_obj(): - return YAMLConfig(fixture_path("FV3_GFS_v16_update.yaml")) - - -def test_create_field_table_with_base_file(create_field_table_update_obj, tmp_path): - """ - Tests create_field_table method with optional base file. - """ - base_file = fixture_path("FV3_GFS_v16.yaml") - outfldtbl_file = tmp_path / "field_table_two.FV3_GFS" - expected = fixture_path("field_table_from_base.FV3_GFS") - config_file = tmp_path / "fcst.yaml" - forecast_config = create_field_table_update_obj - forecast_config["forecast"]["field_table"]["base_file"] = base_file - forecast_config.dump(config_file) - FV3Forecast(config_file).create_field_table(outfldtbl_file) - assert compare_files(expected, outfldtbl_file) - - -def test_create_field_table_without_base_file(tmp_path): - """ - Tests create_field_table without optional base file. - """ - outfldtbl_file = tmp_path / "field_table_one.FV3_GFS" - expected = fixture_path("field_table_from_input.FV3_GFS") - config_file = fixture_path("FV3_GFS_v16_update.yaml") - FV3Forecast(config_file).create_field_table(outfldtbl_file) - assert compare_files(expected, outfldtbl_file) - - -def test_create_directory_structure_bad_existing_act(): - with raises(ValueError): - FV3Forecast.create_directory_structure(run_directory="/some/path", exist_act="foo") - - -def test_create_model_configure_call_private(tmp_path): - basefile = str(tmp_path / "base.yaml") - infile = fixture_path("forecast.yaml") - outfile = str(tmp_path / "out.yaml") - for path in infile, basefile: - Path(path).touch() - with patch.object(Driver, "_create_user_updated_config") as _create_user_updated_config: - with patch.object(FV3Forecast, "_validate", return_value=True): - FV3Forecast(config_file=infile).create_model_configure(outfile) - _create_user_updated_config.assert_called_with( - config_class=YAMLConfig, config_values={}, output_path=outfile - ) - - -@fixture -def create_namelist_assets(tmp_path): - update_values = { - "salad": { - "base": "kale", - "fruit": "banana", - "vegetable": "tomato", - "how_many": 12, - "dressing": "balsamic", - } - } - return update_values, tmp_path / "create_out.nml" - - -def test_create_namelist_with_base_file(create_namelist_assets, tmp_path): - """ - Tests create_namelist method with optional base file. - """ - update_values, outnml_file = create_namelist_assets - base_file = fixture_path("simple3.nml") - fcst_config = { - "forecast": { - "namelist": { - "base_file": base_file, - "update_values": update_values, - }, - }, - } - fcst_config_file = tmp_path / "fcst.yml" - YAMLConfig.dump_dict(cfg=fcst_config, path=fcst_config_file) - FV3Forecast(fcst_config_file).create_namelist(outnml_file) - expected = """ -&salad - base = 'kale' - fruit = 'banana' - vegetable = 'tomato' - how_many = 12 - dressing = 'balsamic' - toppings = , - extras = 0 - dessert = .false. - appetizer = , -/ -""".lstrip() - with open(outnml_file, "r", encoding="utf-8") as out_file: - assert out_file.read() == expected - - -def test_create_namelist_without_base_file(create_namelist_assets, tmp_path): - """ - Tests create_namelist method without optional base file. - """ - update_values, outnml_file = create_namelist_assets - fcst_config = { - "forecast": { - "namelist": { - "update_values": update_values, - }, - }, - } - fcst_config_file = tmp_path / "fcst.yml" - YAMLConfig.dump_dict(cfg=fcst_config, path=fcst_config_file) - FV3Forecast(fcst_config_file).create_namelist(outnml_file) - expected = """ -&salad - base = 'kale' - fruit = 'banana' - vegetable = 'tomato' - how_many = 12 - dressing = 'balsamic' -/ -""".lstrip() - with open(outnml_file, "r", encoding="utf-8") as out_file: - assert out_file.read() == expected - - -def test_forecast_run_cmd(): - """ - Tests that the command to be used to run the forecast executable was built successfully. - """ - config_file = fixture_path("forecast.yaml") - with patch.object(FV3Forecast, "_validate", return_value=True): - fcstobj = FV3Forecast(config_file=config_file) - srun_expected = "srun --export=NONE test_exec.py" - fcstobj._config["runtime_info"]["mpi_args"] = ["--export=NONE"] - assert srun_expected == fcstobj.run_cmd() - mpirun_expected = "mpirun -np 4 test_exec.py" - fcstobj._experiment_config["platform"]["mpicmd"] = "mpirun" - fcstobj._config["runtime_info"]["mpi_args"] = ["-np", 4] - assert mpirun_expected == fcstobj.run_cmd() - fcstobj._experiment_config["platform"]["mpicmd"] = "mpiexec" - fcstobj._config["runtime_info"]["mpi_args"] = [ - "-n", - 4, - "-ppn", - 8, - "--cpu-bind", - "core", - "-depth", - 2, - ] - mpiexec_expected = "mpiexec -n 4 -ppn 8 --cpu-bind core -depth 2 test_exec.py" - assert mpiexec_expected == fcstobj.run_cmd() - - -@pytest.mark.parametrize("section", ["static", "cycle_dependent"]) -@pytest.mark.parametrize("link_files", [True, False]) -def test_stage_files(tmp_path, section, link_files): - """ - Tests that files from static or cycle_dependent sections of the config obj are being staged - (copied or linked) to the run directory. - """ - - run_directory = tmp_path / "run" - src_directory = tmp_path / "src" - files_to_stage = YAMLConfig(fixture_path("expt_dir.yaml"))[section] - # Fix source paths so that they are relative to our test temp directory and - # create the test files. - src_directory.mkdir() - for dst_fn, src_path in files_to_stage.items(): - if isinstance(src_path, list): - files_to_stage[dst_fn] = [str(src_directory / Path(sp).name) for sp in src_path] - else: - fixed_src_path = src_directory / Path(src_path).name - files_to_stage[dst_fn] = str(fixed_src_path) - fixed_src_path.touch() - # Test that none of the destination files exist yet: - for dst_fn in files_to_stage.keys(): - assert not (run_directory / dst_fn).is_file() - # Ask a forecast object to stage the files to the run directory: - FV3Forecast.create_directory_structure(run_directory) - FV3Forecast.stage_files(run_directory, files_to_stage, link_files=link_files) - # Test that all of the destination files now exist: - link_or_file = Path.is_symlink if link_files else Path.is_file - for dst_rel_path, src_paths in files_to_stage.items(): - if isinstance(src_paths, list): - dst_paths = [run_directory / dst_rel_path / os.path.basename(sp) for sp in src_paths] - assert all(link_or_file(d_fn) for d_fn in dst_paths) - else: - assert link_or_file(run_directory / dst_rel_path) - if section == "cycle_dependent": - assert link_or_file(run_directory / "INPUT" / "gfs_bndy.tile7.006.nc") - - -@fixture -def fv3_run_assets(tmp_path): - batch_script = tmp_path / "batch.sh" - config_file = fixture_path("forecast.yaml") - config = YAMLConfig(config_file) - config["forecast"]["run_dir"] = tmp_path.as_posix() - config["forecast"]["cycle_dependent"] = {"foo-file": str(tmp_path / "foo")} - config["forecast"]["static"] = {"static-foo-file": str(tmp_path / "foo")} - return batch_script, config_file, config.data["forecast"] - - -@fixture -def fv3_mpi_assets(): - return [ - "KMP_AFFINITY=scatter", - "OMP_NUM_THREADS=1", - "OMP_STACKSIZE=512m", - "MPI_TYPE_DEPTH=20", - "ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4", - "srun --export=NONE test_exec.py", - ] - - -def test_run_direct(fv3_mpi_assets, fv3_run_assets): - _, config_file, config = fv3_run_assets - expected_command = " ".join(fv3_mpi_assets) - with patch.object(FV3Forecast, "_validate", return_value=True): - with patch.object(forecast, "execute") as execute: - execute.return_value = (True, "") - fcstobj = FV3Forecast(config_file=config_file) - with patch.object(fcstobj, "_config", config): - fcstobj.run(cycle=dt.datetime.now()) - execute.assert_called_once_with(cmd=expected_command, cwd=ANY, log_output=True) - - -@pytest.mark.parametrize("with_batch_script", [True, False]) -def test_FV3Forecast_run_dry_run(caplog, fv3_mpi_assets, fv3_run_assets, with_batch_script): - log.setLevel(logging.INFO) - batch_script, config_file, config = fv3_run_assets - if with_batch_script: - batch_components = [ - "#!/bin/bash", - "#SBATCH --account=user_account", - "#SBATCH --nodes=1", - "#SBATCH --ntasks-per-node=1", - "#SBATCH --qos=batch", - "#SBATCH --time=00:01:00", - ] + fv3_mpi_assets - expected_lines = batch_components - else: - batch_script = None - expected_lines = [" ".join(fv3_mpi_assets)] - - with patch.object(FV3Forecast, "_validate", return_value=True): - fcstobj = FV3Forecast(config_file=config_file, dry_run=True, batch_script=batch_script) - with patch.object(fcstobj, "_config", config): - fcstobj.run(cycle=dt.datetime.now()) - for line in expected_lines: - assert logged(caplog, line) - - -@pytest.mark.parametrize( - "vals", [(True, "_run_via_batch_submission"), (False, "_run_via_local_execution")] -) -def test_FV3Forecast_run(fv3_run_assets, vals): - batch_script, config_file, _ = fv3_run_assets - use_batch, helper_method = vals - fcstobj = FV3Forecast(config_file=config_file, batch_script=batch_script if use_batch else None) - with patch.object(fcstobj, helper_method) as helper: - helper.return_value = (True, None) - assert fcstobj.run(cycle=dt.datetime.now()) is True - helper.assert_called_once_with() - - -def test_FV3Forecast__run_via_batch_submission(fv3_run_assets): - batch_script, config_file, config = fv3_run_assets - fcstobj = FV3Forecast(config_file=config_file, batch_script=batch_script) - with patch.object(fcstobj, "_config", config): - with patch.object(scheduler, "execute") as execute: - with patch.object(Driver, "_create_user_updated_config"): - execute.return_value = (True, "") - success, lines = fcstobj._run_via_batch_submission() - assert success is True - assert lines[0] == "Batch script:" - execute.assert_called_once_with(cmd=ANY, cwd=ANY) - - -def test_FV3Forecast__run_via_local_execution(fv3_run_assets): - _, config_file, config = fv3_run_assets - fcstobj = FV3Forecast(config_file=config_file) - with patch.object(fcstobj, "_config", config): - with patch.object(forecast, "execute") as execute: - execute.return_value = (True, "") - success, lines = fcstobj._run_via_local_execution() - assert success is True - assert lines[0] == "Command:" - execute.assert_called_once_with(cmd=ANY, cwd=ANY, log_output=True) - - -# Schema tests - - -@fixture -def field_table_vals(): - return ( - { - "foo": { - "longname": "foofoo", - "profile_type": {"name": "fixed", "surface_value": 1}, - "units": "cubits", - } - }, - { - "bar": { - "longname": "barbar", - "profile_type": {"name": "profile", "surface_value": 2, "top_value": 3}, - "units": "rods", - } - }, - ) - - -@fixture -def fcstprop(): - return partial(validator, "FV3Forecast.jsonschema", "properties", "forecast", "properties") - - -def test_FV3Forecast_schema_filesToStage(): - errors = validator("FV3Forecast.jsonschema", "$defs", "filesToStage") - # The input must be an dict: - assert "is not of type 'object'" in errors([]) - # A str -> str dict is ok: - assert not errors({"file1": "/path/to/file1", "file2": "/path/to/file2"}) - # An empty dict is not allowed: - assert "does not have enough properties" in errors({}) - # Non-string values are not allowed: - assert "True is not of type 'string'" in errors({"file1": True}) - - -def test_FV3Forecast_schema_forecast(): - d = {"domain": "regional", "executable": "fv3", "length": 3, "run_dir": "/tmp"} - errors = validator("FV3Forecast.jsonschema", "properties", "forecast") - # Basic correctness: - assert not errors(d) - # Some top-level keys are required: - assert "'domain' is a required property" in errors(with_del(d, "domain")) - assert "'executable' is a required property" in errors(with_del(d, "executable")) - assert "'length' is a required property" in errors(with_del(d, "length")) - assert "'run_dir' is a required property" in errors(with_del(d, "run_dir")) - # Some top-level keys are optional: - assert not errors( - { - **d, - "diag_table": "/path", - "field_table": {"base_file": "/path"}, - "files_to_copy": {"fn": "/path"}, - "files_to_link": {"fn": "/path"}, - "model_configure": {"base_file": "/path"}, - "namelist": {"base_file": "/path"}, - "runtime_info": {}, - } - ) - # Additional top-level keys are not allowed: - assert "Additional properties are not allowed" in errors({**d, "foo": "bar"}) - - -def test_FV3Forecast_schema_forecast_diag_table(fcstprop): - errors = fcstprop("diag_table") - # String value is ok: - assert not errors("/path/to/file") - # Anything else is not: - assert "88 is not of type 'string'" in errors(88) - - -def test_FV3Forecast_schema_forecast_domain(fcstprop): - errors = fcstprop("domain") - # There is a fixed set of domain values: - assert "'foo' is not one of ['global', 'regional']" in errors("foo") - - -def test_FV3Forecast_schema_forecast_executable(fcstprop): - errors = fcstprop("executable") - # String value is ok: - assert not errors("fv3.exe") - # Anything else is not: - assert "88 is not of type 'string'" in errors(88) - - -def test_FV3Forecast_schema_forecast_field_table(fcstprop, field_table_vals): - val, _ = field_table_vals - base_file = {"base_file": "/some/path"} - update_values = {"update_values": val} - errors = fcstprop("field_table") - # Just base_file is ok: - assert not errors(base_file) - # Just update_values is ok: - assert not errors(update_values) - # A combination of base_file and update_values is ok: - assert not errors({**base_file, **update_values}) - # At least one is required: - assert "is not valid" in errors({}) - - -def test_FV3Forecast_schema_forecast_field_table_update_values(fcstprop, field_table_vals): - val1, val2 = field_table_vals - errors = fcstprop("field_table", "properties", "update_values") - # A "fixed" profile-type entry is ok: - assert not errors(val1) - # A "profile" profile-type entry is ok: - assert not errors(val2) - # A combination of two valid entries is ok: - assert not errors({**val1, **val2}) - # At least one entry is required: - assert "does not have enough properties" in errors({}) - # longname is required: - assert "'longname' is a required property" in errors(with_del(val1, "foo", "longname")) - # longname must be a string: - assert "88 is not of type 'string'" in errors(with_set(val1, 88, "foo", "longname")) - # units is required: - assert "'units' is a required property" in errors(with_del(val1, "foo", "units")) - # units must be a string: - assert "88 is not of type 'string'" in errors(with_set(val1, 88, "foo", "units")) - # profile_type is required: - assert "'profile_type' is a required property" in errors(with_del(val1, "foo", "profile_type")) - # profile_type name has to be "fixed" or "profile": - assert "'bogus' is not one of ['fixed', 'profile']" in errors( - with_set(val1, "bogus", "foo", "profile_type", "name") - ) - # surface_value is required: - assert "'surface_value' is a required property" in errors( - with_del(val1, "foo", "profile_type", "surface_value") - ) - # surface_value is numeric: - assert "'a string' is not of type 'number'" in errors( - with_set(val1, "a string", "foo", "profile_type", "surface_value") - ) - # top_value is required if name is "profile": - assert "'top_value' is a required property" in errors( - with_del(val2, "bar", "profile_type", "top_value") - ) - # top_value is numeric: - assert "'a string' is not of type 'number'" in errors( - with_set(val2, "a string", "bar", "profile_type", "top_value") - ) - - -def test_FV3Forecast_schema_forecast_files_to_copy(): - test_FV3Forecast_schema_filesToStage() - - -def test_FV3Forecast_schema_forecast_files_to_link(): - test_FV3Forecast_schema_filesToStage() - - -def test_FV3Forecast_schema_forecast_length(fcstprop): - errors = fcstprop("length") - # Positive int is ok: - assert not errors(6) - # Zero is not ok: - assert "0 is less than the minimum of 1" in errors(0) - # A negative number is not ok: - assert "-1 is less than the minimum of 1" in errors(-1) - # Something other than an int is not ok: - assert "'a string' is not of type 'integer'" in errors("a string") - - -def test_FV3Forecast_schema_forecast_model_configure(fcstprop): - base_file = {"base_file": "/some/path"} - update_values = {"update_values": {"foo": 88}} - errors = fcstprop("model_configure") - # Just base_file is ok: - assert not errors(base_file) - # But base_file must be a string: - assert "88 is not of type 'string'" in errors({"base_file": 88}) - # Just update_values is ok: - assert not errors(update_values) - # A combination of base_file and update_values is ok: - assert not errors({**base_file, **update_values}) - # At least one is required: - assert "is not valid" in errors({}) - - -def test_FV3Forecast_schema_forecast_model_configure_update_values(fcstprop): - errors = fcstprop("model_configure", "properties", "update_values") - # boolean, number, and string values are ok: - assert not errors({"bool": True, "int": 88, "float": 3.14, "string": "foo"}) - # Other types are not, e.g.: - assert "None is not of type 'boolean', 'number', 'string'" in errors({"null": None}) - # At least one entry is required: - assert "does not have enough properties" in errors({}) - - -def test_FV3Forecast_schema_forecast_namelist(fcstprop): - base_file = {"base_file": "/some/path"} - update_values = {"update_values": {"nml": {"var": "val"}}} - errors = fcstprop("namelist") - # Just base_file is ok: - assert not errors(base_file) - # base_file must be a string: - assert "88 is not of type 'string'" in errors({"base_file": 88}) - # Just update_values is ok: - assert not errors(update_values) - # A combination of base_file and update_values is ok: - assert not errors({**base_file, **update_values}) - # At least one is required: - assert "is not valid" in errors({}) - - -def test_FV3Forecast_schema_forecast_namelist_update_values(fcstprop): - errors = fcstprop("namelist", "properties", "update_values") - # array, boolean, number, and string values are ok: - assert not errors( - {"nml": {"array": [1, 2, 3], "bool": True, "int": 88, "float": 3.14, "string": "foo"}} - ) - # Other types are not, e.g.: - assert "None is not of type 'array', 'boolean', 'number', 'string'" in errors( - {"nml": {"null": None}} - ) - # At least one namelist entry is required: - assert "does not have enough properties" in errors({}) - # At least one val/var pair ir required: - assert "does not have enough properties" in errors({"nml": {}}) - - -def test_FV3Forecast_schema_forecast_run_dir(fcstprop): - errors = fcstprop("run_dir") - # Must be a string: - assert not errors("/some/path") - assert "88 is not of type 'string'" in errors(88) - - -def test_FV3Forecast_schema_forecast_runtime_info(fcstprop): - mpi_args = {"mpi_args": ["--flag1", "--flag2"]} - threads = {"threads": 32} - errors = fcstprop("runtime_info") - # mpi_args is a list of strings: - assert not errors(mpi_args) - # mpi_args may be empty: - assert not errors({"mpi_args": []}) - # String values are expected: - assert "88 is not of type 'string'" in errors({"mpi_args": [88]}) - # threads must be non-negative, and an integer: - assert not errors(threads) - assert not errors({"threads": 0}) - assert "-1 is less than the minimum of 0" in errors({"threads": -1}) - assert "3.14 is not of type 'integer'" in errors({"threads": 3.14}) - # Both properties are ok: - assert not errors({**mpi_args, **threads}) - # Additional properties are not allowed: - assert "Additional properties are not allowed" in errors({**mpi_args, **threads, "foo": "bar"}) - - -def test_FV3Forecast_schema_platform(): - d = {"account": "me", "mpicmd": "cmd", "scheduler": "slurm"} - errors = validator("FV3Forecast.jsonschema", "properties", "platform") - # Basic correctness: - assert not errors(d) - # At least mpicmd is required: - assert "'mpicmd' is a required property" in errors({}) - # Extra top-level keys are forbidden: - assert "Additional properties are not allowed" in errors(with_set(d, "bar", "foo")) - # There is a fixed set of supported schedulers: - assert "'foo' is not one of ['lsf', 'pbs', 'slurm']" in errors(with_set(d, "foo", "scheduler")) - # account and scheduler are optional: - assert not errors({"mpicmd": "cmd"}) - # account is required if scheduler is specified: - assert "'account' is a dependency of 'scheduler'" in errors(with_del(d, "account")) - - -def test_FV3Forecast_schema_preprocessing(): - d = { - "lateral_boundary_conditions": { - "interval_hours": 1, - "offset": 0, - "output_file_path": "/some/path", - } - } - errors = validator("FV3Forecast.jsonschema", "properties", "preprocessing") - # Basic correctness: - assert not errors(d) - assert "'lateral_boundary_conditions' is a required property" in errors({}) - # All lateral_boundary_conditions items are required: - assert "'interval_hours' is a required property" in errors( - with_del(d, "lateral_boundary_conditions", "interval_hours") - ) - assert "'offset' is a required property" in errors( - with_del(d, "lateral_boundary_conditions", "offset") - ) - assert "'output_file_path' is a required property" in errors( - with_del(d, "lateral_boundary_conditions", "output_file_path") - ) - # interval_hours must be an integer of at least 1: - assert "0 is less than the minimum of 1" in errors( - with_set(d, 0, "lateral_boundary_conditions", "interval_hours") - ) - assert "'a string' is not of type 'integer'" in errors( - with_set(d, "a string", "lateral_boundary_conditions", "interval_hours") - ) - # offset must be an integer of at least 0: - assert "-1 is less than the minimum of 0" in errors( - with_set(d, -1, "lateral_boundary_conditions", "offset") - ) - assert "'a string' is not of type 'integer'" in errors( - with_set(d, "a string", "lateral_boundary_conditions", "offset") - ) - # output_file_path must be a string: - assert "88 is not of type 'string'" in errors( - with_set(d, 88, "lateral_boundary_conditions", "output_file_path") - ) diff --git a/src/uwtools/tests/drivers/test_fv3.py b/src/uwtools/tests/drivers/test_fv3.py new file mode 100644 index 000000000..eefc90b4d --- /dev/null +++ b/src/uwtools/tests/drivers/test_fv3.py @@ -0,0 +1,587 @@ +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +""" +FV3 driver tests. +""" +import datetime as dt +from functools import partial +from pathlib import Path +from unittest.mock import DEFAULT as D +from unittest.mock import PropertyMock, patch + +import pytest +import yaml +from pytest import fixture + +from uwtools.drivers import fv3 +from uwtools.tests.support import logged, validator, with_del, with_set + +# Fixtures + + +@fixture +def cycle(): + return dt.datetime(2024, 2, 1, 18) + + +# Driver fixtures + + +@fixture +def config(tmp_path): + return { + "fv3": { + "domain": "global", + "execution": {"executable": "fv3"}, + "lateral_boundary_conditions": { + "interval_hours": 1, + "offset": 0, + "path": str(tmp_path / "f{forecast_hour}"), + }, + "length": 1, + "run_dir": str(tmp_path), + } + } + + +@fixture +def config_file(config, tmp_path): + path = tmp_path / "fv3.yaml" + with open(path, "w", encoding="utf-8") as f: + yaml.dump(config, f) + return path + + +@fixture +def fv3obj(config_file, cycle): + return fv3.FV3(config_file=config_file, cycle=cycle, batch=True) + + +# Driver tests + + +def test_FV3(fv3obj): + assert isinstance(fv3obj, fv3.FV3) + + +def test_FV3_dry_run(config_file, cycle): + with patch.object(fv3, "dryrun") as dryrun: + fv3obj = fv3.FV3(config_file=config_file, cycle=cycle, batch=True, dry_run=True) + assert fv3obj._dry_run is True + dryrun.assert_called_once_with() + + +def test_FV3_boundary_files(fv3obj): + ns = (0, 1) + links = [fv3obj._rundir / "INPUT" / f"gfs_bndy.tile7.{n:03d}.nc" for n in ns] + assert not any(link.is_file() for link in links) + for n in ns: + (fv3obj._rundir / f"f{n}").touch() + fv3obj.boundary_files() + assert all(link.is_symlink() for link in links) + + +def test_FV3_diag_table(fv3obj): + src = fv3obj._rundir / "diag_table.in" + src.touch() + fv3obj._driver_config["diag_table"] = src + dst = fv3obj._rundir / "diag_table" + assert not dst.is_file() + fv3obj.diag_table() + assert dst.is_file() + + +def test_FV3_diag_table_warn(caplog, fv3obj): + fv3obj.diag_table() + assert logged(caplog, "No 'diag_table' defined in config") + + +def test_FV3_field_table(fv3obj): + src = fv3obj._rundir / "field_table.in" + with open(src, "w", encoding="utf-8") as f: + yaml.dump({}, f) + dst = fv3obj._rundir / "field_table" + assert not dst.is_file() + fv3obj._driver_config["field_table"] = {"base_file": src} + fv3obj.field_table() + assert dst.is_file() + + +@pytest.mark.parametrize( + "key,task,test", + [("files_to_copy", "files_copied", "is_file"), ("files_to_link", "files_linked", "is_symlink")], +) +def test_FV3_files_copied(config, cycle, key, task, test, tmp_path): + atm, sfc = "gfs.t%sz.atmanl.nc", "gfs.t%sz.sfcanl.nc" + atm_cfg_dst, sfc_cfg_dst = [x % "{{ cycle.strftime('%H') }}" for x in [atm, sfc]] + atm_cfg_src, sfc_cfg_src = [str(tmp_path / (x + ".in")) for x in [atm_cfg_dst, sfc_cfg_dst]] + config["fv3"].update({key: {atm_cfg_dst: atm_cfg_src, sfc_cfg_dst: sfc_cfg_src}}) + path = tmp_path / "fv3.yaml" + with open(path, "w", encoding="utf-8") as f: + yaml.dump(config, f) + fv3obj = fv3.FV3(config_file=path, cycle=cycle, batch=True) + atm_dst, sfc_dst = [tmp_path / (x % cycle.strftime("%H")) for x in [atm, sfc]] + assert not any(dst.is_file() for dst in [atm_dst, sfc_dst]) + atm_src, sfc_src = [Path(str(x) + ".in") for x in [atm_dst, sfc_dst]] + for src in (atm_src, sfc_src): + src.touch() + getattr(fv3obj, task)() + assert all(getattr(dst, test)() for dst in [atm_dst, sfc_dst]) + + +def test_FV3_model_configure(fv3obj): + src = fv3obj._rundir / "model_configure.in" + with open(src, "w", encoding="utf-8") as f: + yaml.dump({}, f) + dst = fv3obj._rundir / "model_configure" + assert not dst.is_file() + fv3obj._driver_config["model_configure"] = {"base_file": src} + fv3obj.model_configure() + assert dst.is_file() + + +def test_FV3_namelist_file(fv3obj): + src = fv3obj._rundir / "input.nml.in" + with open(src, "w", encoding="utf-8") as f: + yaml.dump({}, f) + dst = fv3obj._rundir / "input.nml" + assert not dst.is_file() + fv3obj._driver_config["namelist_file"] = {"base_file": src} + fv3obj.namelist_file() + assert dst.is_file() + + +def test_FV3_provisioned_run_directory(fv3obj): + with patch.multiple( + fv3obj, + boundary_files=D, + diag_table=D, + field_table=D, + files_copied=D, + files_linked=D, + model_configure=D, + namelist_file=D, + restart_directory=D, + runscript=D, + ) as mocks: + fv3obj.provisioned_run_directory() + for m in mocks: + mocks[m].assert_called_once_with() + + +def test_FV3_restart_directory(fv3obj): + path = fv3obj._rundir / "RESTART" + assert not path.is_dir() + fv3obj.restart_directory() + assert path.is_dir() + + +def test_FV3_run_batch(fv3obj): + with patch.object(fv3obj, "_run_via_batch_submission") as func: + fv3obj.run() + func.assert_called_once_with() + + +def test_FV3_run_local(fv3obj): + fv3obj._batch = False + with patch.object(fv3obj, "_run_via_local_execution") as func: + fv3obj.run() + func.assert_called_once_with() + + +def test_FV3_runscript(fv3obj): + dst = fv3obj._rundir / "runscript" + assert not dst.is_file() + fv3obj._driver_config["execution"].update( + { + "batchargs": {"walltime": "01:10:00"}, + "envcmds": ["cmd1", "cmd2"], + "mpicmd": "runit", + "threads": 8, + } + ) + fv3obj._config["platform"] = {"account": "me", "scheduler": "slurm"} + fv3obj.runscript() + with open(dst, "r", encoding="utf-8") as f: + lines = f.read().split("\n") + # Check directives: + assert "#SBATCH --account=me" in lines + assert "#SBATCH --time=01:10:00" in lines + # Check environment variables: + assert "export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4" in lines + assert "export KMP_AFFINITY=scatter" in lines + assert "export MPI_TYPE_DEPTH=20" in lines + assert "export OMP_NUM_THREADS=8" in lines + assert "export OMP_STACKSIZE=512m" in lines + # Check environment commands: + assert "cmd1" in lines + assert "cmd2" in lines + # Check execution: + assert "runit fv3" in lines + assert "test $? -eq 0 && touch %s/done" % fv3obj._rundir + + +def test_FV3__run_via_batch_submission(fv3obj): + runscript = fv3obj._runscript_path + with patch.object(fv3obj, "provisioned_run_directory") as prd: + with patch.object(fv3.FV3, "_scheduler", new_callable=PropertyMock) as scheduler: + fv3obj._run_via_batch_submission() + scheduler().submit_job.assert_called_once_with( + runscript=runscript, submit_file=Path(f"{runscript}.submit") + ) + prd.assert_called_once_with() + + +def test_FV3__run_via_local_execution(fv3obj): + with patch.object(fv3obj, "provisioned_run_directory") as prd: + with patch.object(fv3, "execute") as execute: + fv3obj._run_via_local_execution() + execute.assert_called_once_with( + cmd="{x} >{x}.out 2>&1".format(x=fv3obj._runscript_path), + cwd=fv3obj._rundir, + log_output=True, + ) + prd.assert_called_once_with() + + +def test_FV3__driver_config(fv3obj): + assert fv3obj._driver_config == fv3obj._config["fv3"] + + +def test_FV3__resources(fv3obj): + account = "me" + scheduler = "slurm" + walltime = "01:10:00" + fv3obj._driver_config["execution"].update({"batchargs": {"walltime": walltime}}) + fv3obj._config["platform"] = {"account": account, "scheduler": scheduler} + assert fv3obj._resources == { + "account": account, + "rundir": fv3obj._rundir, + "scheduler": scheduler, + "walltime": walltime, + } + + +def test_FV3__runscript_path(fv3obj): + assert fv3obj._runscript_path == fv3obj._rundir / "runscript" + + +def test_FV3__taskanme(fv3obj): + assert fv3obj._taskname("foo") == "20240201 18Z FV3 foo" + + +def test_FV3__validate(fv3obj): + fv3obj._validate() + + +# Schema fixtures + + +@fixture +def field_table_vals(): + return ( + { + "foo": { + "longname": "foofoo", + "profile_type": {"name": "fixed", "surface_value": 1}, + "units": "cubits", + } + }, + { + "bar": { + "longname": "barbar", + "profile_type": {"name": "profile", "surface_value": 2, "top_value": 3}, + "units": "rods", + } + }, + ) + + +@fixture +def fcstprop(): + return partial(validator, "fv3.jsonschema", "properties", "fv3", "properties") + + +# Schema tests + + +def test_fv3_schema_filesToStage(): + errors = validator("fv3.jsonschema", "$defs", "filesToStage") + # The input must be an dict: + assert "is not of type 'object'" in errors([]) + # A str -> str dict is ok: + assert not errors({"file1": "/path/to/file1", "file2": "/path/to/file2"}) + # An empty dict is not allowed: + assert "does not have enough properties" in errors({}) + # Non-string values are not allowed: + assert "True is not of type 'string'" in errors({"file1": True}) + + +def test_fv3_schema_forecast(): + d = { + "domain": "regional", + "execution": {"executable": "fv3"}, + "lateral_boundary_conditions": {"interval_hours": 1, "offset": 0, "path": "/tmp/file"}, + "length": 3, + "run_dir": "/tmp", + } + errors = validator("fv3.jsonschema", "properties", "fv3") + # Basic correctness: + assert not errors(d) + # Some top-level keys are required: + for key in ("domain", "execution", "lateral_boundary_conditions", "length", "run_dir"): + assert f"'{key}' is a required property" in errors(with_del(d, key)) + # Some top-level keys are optional: + assert not errors( + { + **d, + "diag_table": "/path", + "field_table": {"base_file": "/path"}, + "files_to_copy": {"fn": "/path"}, + "files_to_link": {"fn": "/path"}, + "model_configure": {"base_file": "/path"}, + "namelist": {"base_file": "/path"}, + } + ) + # Additional top-level keys are not allowed: + assert "Additional properties are not allowed" in errors({**d, "foo": "bar"}) + + +def test_fv3_schema_forecast_diag_table(fcstprop): + errors = fcstprop("diag_table") + # String value is ok: + assert not errors("/path/to/file") + # Anything else is not: + assert "88 is not of type 'string'" in errors(88) + + +def test_fv3_schema_forecast_domain(fcstprop): + errors = fcstprop("domain") + # There is a fixed set of domain values: + assert "'foo' is not one of ['global', 'regional']" in errors("foo") + + +def test_fv3_schema_forecast_execution(fcstprop): + d = {"executable": "fv3"} + batchargs = {"batchargs": {"queue": "string", "walltime": "string"}} + mpiargs = {"mpiargs": ["--flag1", "--flag2"]} + threads = {"threads": 32} + errors = fcstprop("execution") + # Basic correctness: + assert not errors(d) + # batchargs may optionally be specified: + assert not errors({**d, **batchargs}) + # mpiargs may be optionally specified: + assert not errors({**d, **mpiargs}) + # threads may optionally be specified: + assert not errors({**d, **threads}) + # All properties are ok: + assert not errors({**d, **batchargs, **mpiargs, **threads}) + # Additional properties are not allowed: + assert "Additional properties are not allowed" in errors( + {**d, **mpiargs, **threads, "foo": "bar"} + ) + + +def test_fv3_schema_forecast_execution_batchargs(fcstprop): + errors = fcstprop("execution", "properties", "batchargs") + # Basic correctness, empty map is ok: + assert not errors({}) + # Managed properties are fine: + assert not errors({"queue": "string", "walltime": "string"}) + # But so are unknown ones: + assert not errors({"--foo": 88}) + # It just has to be a map: + assert "[] is not of type 'object'" in errors([]) + + +def test_fv3_schema_forecast_execution_executable(fcstprop): + errors = fcstprop("execution", "properties", "executable") + # String value is ok: + assert not errors("fv3.exe") + # Anything else is not: + assert "88 is not of type 'string'" in errors(88) + + +def test_fv3_schema_forecast_execution_mpiargs(fcstprop): + errors = fcstprop("execution", "properties", "mpiargs") + # Basic correctness: + assert not errors(["string1", "string2"]) + # mpiargs may be empty: + assert not errors([]) + # String values are expected: + assert "88 is not of type 'string'" in errors(["string1", 88]) + + +def test_fv3_schema_forecast_execution_threads(fcstprop): + errors = fcstprop("execution", "properties", "threads") + # threads must be non-negative, and an integer: + assert not errors(0) + assert not errors(4) + assert "-1 is less than the minimum of 0" in errors(-1) + assert "3.14 is not of type 'integer'" in errors(3.14) + + +def test_fv3_schema_forecast_field_table(fcstprop, field_table_vals): + val, _ = field_table_vals + base_file = {"base_file": "/some/path"} + update_values = {"update_values": val} + errors = fcstprop("field_table") + # Just base_file is ok: + assert not errors(base_file) + # Just update_values is ok: + assert not errors(update_values) + # A combination of base_file and update_values is ok: + assert not errors({**base_file, **update_values}) + # At least one is required: + assert "is not valid" in errors({}) + + +def test_fv3_schema_forecast_field_table_update_values(fcstprop, field_table_vals): + val1, val2 = field_table_vals + errors = fcstprop("field_table", "properties", "update_values") + # A "fixed" profile-type entry is ok: + assert not errors(val1) + # A "profile" profile-type entry is ok: + assert not errors(val2) + # A combination of two valid entries is ok: + assert not errors({**val1, **val2}) + # At least one entry is required: + assert "does not have enough properties" in errors({}) + # longname is required: + assert "'longname' is a required property" in errors(with_del(val1, "foo", "longname")) + # longname must be a string: + assert "88 is not of type 'string'" in errors(with_set(val1, 88, "foo", "longname")) + # units is required: + assert "'units' is a required property" in errors(with_del(val1, "foo", "units")) + # units must be a string: + assert "88 is not of type 'string'" in errors(with_set(val1, 88, "foo", "units")) + # profile_type is required: + assert "'profile_type' is a required property" in errors(with_del(val1, "foo", "profile_type")) + # profile_type name has to be "fixed" or "profile": + assert "'bogus' is not one of ['fixed', 'profile']" in errors( + with_set(val1, "bogus", "foo", "profile_type", "name") + ) + # surface_value is required: + assert "'surface_value' is a required property" in errors( + with_del(val1, "foo", "profile_type", "surface_value") + ) + # surface_value is numeric: + assert "'a string' is not of type 'number'" in errors( + with_set(val1, "a string", "foo", "profile_type", "surface_value") + ) + # top_value is required if name is "profile": + assert "'top_value' is a required property" in errors( + with_del(val2, "bar", "profile_type", "top_value") + ) + # top_value is numeric: + assert "'a string' is not of type 'number'" in errors( + with_set(val2, "a string", "bar", "profile_type", "top_value") + ) + + +def test_fv3_schema_forecast_files_to_copy(): + test_fv3_schema_filesToStage() + + +def test_fv3_schema_forecast_files_to_link(): + test_fv3_schema_filesToStage() + + +def test_fv3_schema_forecast_lateral_boundary_conditions(fcstprop): + d = { + "interval_hours": 1, + "offset": 0, + "path": "/some/path", + } + errors = fcstprop("lateral_boundary_conditions") + # Basic correctness: + assert not errors(d) + # All lateral_boundary_conditions items are required: + assert "'interval_hours' is a required property" in errors(with_del(d, "interval_hours")) + assert "'offset' is a required property" in errors(with_del(d, "offset")) + assert "'path' is a required property" in errors(with_del(d, "path")) + # interval_hours must be an integer of at least 1: + assert "0 is less than the minimum of 1" in errors(with_set(d, 0, "interval_hours")) + assert "'s' is not of type 'integer'" in errors(with_set(d, "s", "interval_hours")) + # offset must be an integer of at least 0: + assert "-1 is less than the minimum of 0" in errors(with_set(d, -1, "offset")) + assert "'s' is not of type 'integer'" in errors(with_set(d, "s", "offset")) + # path must be a string: + assert "88 is not of type 'string'" in errors(with_set(d, 88, "path")) + + +def test_fv3_schema_forecast_length(fcstprop): + errors = fcstprop("length") + # Positive int is ok: + assert not errors(6) + # Zero is not ok: + assert "0 is less than the minimum of 1" in errors(0) + # A negative number is not ok: + assert "-1 is less than the minimum of 1" in errors(-1) + # Something other than an int is not ok: + assert "'a string' is not of type 'integer'" in errors("a string") + + +def test_fv3_schema_forecast_model_configure(fcstprop): + base_file = {"base_file": "/some/path"} + update_values = {"update_values": {"foo": 88}} + errors = fcstprop("model_configure") + # Just base_file is ok: + assert not errors(base_file) + # But base_file must be a string: + assert "88 is not of type 'string'" in errors({"base_file": 88}) + # Just update_values is ok: + assert not errors(update_values) + # A combination of base_file and update_values is ok: + assert not errors({**base_file, **update_values}) + # At least one is required: + assert "is not valid" in errors({}) + + +def test_fv3_schema_forecast_model_configure_update_values(fcstprop): + errors = fcstprop("model_configure", "properties", "update_values") + # boolean, number, and string values are ok: + assert not errors({"bool": True, "int": 88, "float": 3.14, "string": "foo"}) + # Other types are not, e.g.: + assert "None is not of type 'boolean', 'number', 'string'" in errors({"null": None}) + # At least one entry is required: + assert "does not have enough properties" in errors({}) + + +def test_fv3_schema_forecast_namelist(fcstprop): + base_file = {"base_file": "/some/path"} + update_values = {"update_values": {"nml": {"var": "val"}}} + errors = fcstprop("namelist") + # Just base_file is ok: + assert not errors(base_file) + # base_file must be a string: + assert "88 is not of type 'string'" in errors({"base_file": 88}) + # Just update_values is ok: + assert not errors(update_values) + # A combination of base_file and update_values is ok: + assert not errors({**base_file, **update_values}) + # At least one is required: + assert "is not valid" in errors({}) + + +def test_fv3_schema_forecast_namelist_update_values(fcstprop): + errors = fcstprop("namelist", "properties", "update_values") + # array, boolean, number, and string values are ok: + assert not errors( + {"nml": {"array": [1, 2, 3], "bool": True, "int": 88, "float": 3.14, "string": "foo"}} + ) + # Other types are not, e.g.: + assert "None is not of type 'array', 'boolean', 'number', 'string'" in errors( + {"nml": {"null": None}} + ) + # At least one namelist entry is required: + assert "does not have enough properties" in errors({}) + # At least one val/var pair ir required: + assert "does not have enough properties" in errors({"nml": {}}) + + +def test_fv3_schema_forecast_run_dir(fcstprop): + errors = fcstprop("run_dir") + # Must be a string: + assert not errors("/some/path") + assert "88 is not of type 'string'" in errors(88) diff --git a/src/uwtools/tests/drivers/test_schema_platform.py b/src/uwtools/tests/drivers/test_schema_platform.py new file mode 100644 index 000000000..8360885fc --- /dev/null +++ b/src/uwtools/tests/drivers/test_schema_platform.py @@ -0,0 +1,23 @@ +# pylint: disable=missing-function-docstring +""" +Tests for the "platform" schema. +""" + +from uwtools.tests.support import validator, with_del, with_set + + +def test_fv3_schema_platform(): + d = {"account": "me", "scheduler": "slurm"} + errors = validator("platform.jsonschema", "properties", "platform") + # Basic correctness: + assert not errors(d) + # Extra top-level keys are forbidden: + assert "Additional properties are not allowed" in errors(with_set(d, "bar", "foo")) + # There is a fixed set of supported schedulers: + assert "'foo' is not one of ['lsf', 'pbs', 'slurm']" in errors(with_set(d, "foo", "scheduler")) + # account and scheduler are optional: + assert not errors({}) + # account is required if scheduler is specified: + assert "'account' is a dependency of 'scheduler'" in errors(with_del(d, "account")) + # scheduler is required if account is specified: + assert "'scheduler' is a dependency of 'account'" in errors(with_del(d, "scheduler")) diff --git a/src/uwtools/tests/fixtures/forecast.yaml b/src/uwtools/tests/fixtures/fv3.yaml similarity index 79% rename from src/uwtools/tests/fixtures/forecast.yaml rename to src/uwtools/tests/fixtures/fv3.yaml index ffe759337..8a1a8743b 100644 --- a/src/uwtools/tests/fixtures/forecast.yaml +++ b/src/uwtools/tests/fixtures/fv3.yaml @@ -1,6 +1,17 @@ -forecast: +fv3: + diag_table: /some/path domain: regional - executable: test_exec.py + execution: + batchargs: + queue: some-queue + walltime: "00:10:00" + executable: ufs_model + mpiargs: + - "--export=NONE" + mpicmd: srun + threads: 1 + field_table: + base_file: /some/path files_to_copy: INPUT/gfs_bndy.tile7.000.nc: path/to/gfs_bndy.tile7.000.nc INPUT/gfs_bndy.tile7.006.nc: path/to/gfs_bndy.tile7.006.nc @@ -17,25 +28,16 @@ forecast: co2historicaldata_2016.txt: src/uwtools/drivers/global_co2historicaldata_2016.txt co2historicaldata_2017.txt: src/uwtools/drivers/global_co2historicaldata_2017.txt co2historicaldata_2018.txt: src/uwtools/drivers/global_co2historicaldata_2018.txt - jobinfo: - nodes: 1 - queue: batch - tasks_per_node: 1 - walltime: "00:01:00" + lateral_boundary_conditions: + interval_hours: 3 + offset: 0 + path: gfs_bndy.tile{tile}.f{forecast_hour}.nc length: 12 - model: FV3 + model_configure: + base_file: /some/path + namelist: + base_file: /some/path run_dir: some/path - runtime_info: - mpi_args: - - "--export=NONE" - stacksize: 512m - threads: 1 platform: account: user_account - mpicmd: srun scheduler: slurm -preprocessing: - lateral_boundary_conditions: - interval_hours: 3 - offset: 0 - output_file_path: gfs_bndy.tile{tile}.f{forecast_hour}.nc diff --git a/src/uwtools/tests/support.py b/src/uwtools/tests/support.py index dd2e41026..e53065aa3 100644 --- a/src/uwtools/tests/support.py +++ b/src/uwtools/tests/support.py @@ -105,8 +105,10 @@ def validator(schema_fn: str, *args: Any) -> Callable: """ with open(resource_pathobj(schema_fn), "r", encoding="utf-8") as f: schema = yaml.safe_load(f) + defs = schema.get("$defs", {}) for arg in args: - schema = {"$defs": schema["$defs"], **schema[arg]} + schema = schema[arg] + schema.update({"$defs": defs}) return lambda config: "\n".join(str(x) for x in _validation_errors(config, schema)) diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index eb3909a2e..2ae1f8bc3 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -1,5 +1,5 @@ # pylint: disable=missing-function-docstring,protected-access,redefined-outer-name - +import datetime as dt import logging import sys from argparse import ArgumentParser as Parser @@ -11,10 +11,10 @@ from pytest import fixture, raises import uwtools.api.config -import uwtools.api.forecast +import uwtools.api.fv3 import uwtools.api.rocoto import uwtools.api.template -import uwtools.drivers.forecast +import uwtools.drivers.fv3 from uwtools import cli from uwtools.cli import STR from uwtools.exceptions import UWError @@ -52,14 +52,21 @@ def test__add_subparser_config_validate(subparsers): assert subparsers.choices[STR.validate] -def test__add_subparser_forecast(subparsers): - cli._add_subparser_forecast(subparsers) - assert actions(subparsers.choices[STR.forecast]) == [STR.run] - - -def test__add_subparser_forecast_run(subparsers): - cli._add_subparser_forecast_run(subparsers) - assert subparsers.choices[STR.run] +def test__add_subparser_fv3(subparsers): + cli._add_subparser_fv3(subparsers) + assert actions(subparsers.choices[STR.fv3]) == [ + "boundary_files", + "diag_table", + "field_table", + "files_copied", + "files_linked", + "model_configure", + "namelist_file", + "provisioned_run_directory", + "restart_directory", + "run", + "runscript", + ] def test__add_subparser_template(subparsers): @@ -256,29 +263,16 @@ def test__dispatch_config_validate_config_obj(): _validate_yaml.assert_called_once_with(**_validate_yaml_args) -@pytest.mark.parametrize("params", [(STR.run, "_dispatch_forecast_run")]) -def test__dispatch_forecast(params): - action, funcname = params - args = {STR.action: action} - with patch.object(cli, funcname) as module: - cli._dispatch_forecast(args) - module.assert_called_once_with(args) - - -def test__dispatch_forecast_run(): - args = { - STR.batch_script: None, - STR.cfgfile: 1, - STR.cycle: "2023-01-01T00:00:00", - STR.dryrun: True, - STR.model: "foo", +def test__dispatch_fv3(): + args: dict = { + "batch": True, + "config_file": "config.yaml", + "cycle": dt.datetime.now(), + "dry_run": False, } - with patch.object(uwtools.drivers.forecast, "FooForecast", create=True) as FooForecast: - CLASSES = {"foo": getattr(uwtools.drivers.forecast, "FooForecast")} - with patch.object(uwtools.api.forecast, "_CLASSES", new=CLASSES): - cli._dispatch_forecast_run(args) - FooForecast.assert_called_once_with(batch_script=None, config_file=1, dry_run=True) - FooForecast().run.assert_called_once_with(cycle="2023-01-01T00:00:00") + with patch.object(uwtools.api.fv3, "execute") as execute: + cli._dispatch_fv3({**args, "action": "foo"}) + execute.assert_called_once_with(**{**args, "task": "foo"}) @pytest.mark.parametrize( @@ -291,9 +285,9 @@ def test__dispatch_forecast_run(): def test__dispatch_rocoto(params): action, funcname = params args = {STR.action: action} - with patch.object(cli, funcname) as module: + with patch.object(cli, funcname) as func: cli._dispatch_rocoto(args) - module.assert_called_once_with(args) + func.assert_called_once_with(args) def test__dispatch_rocoto_realize(): @@ -305,9 +299,9 @@ def test__dispatch_rocoto_realize(): def test__dispatch_rocoto_realize_no_optional(): args = {STR.infile: None, STR.outfile: None} - with patch.object(uwtools.api.rocoto, "_realize") as module: + with patch.object(uwtools.api.rocoto, "_realize") as func: cli._dispatch_rocoto_realize(args) - module.assert_called_once_with(config=None, output_file=None) + func.assert_called_once_with(config=None, output_file=None) def test__dispatch_rocoto_validate_xml(): diff --git a/src/uwtools/tests/test_scheduler.py b/src/uwtools/tests/test_scheduler.py index 49e7a0052..a8da62c79 100644 --- a/src/uwtools/tests/test_scheduler.py +++ b/src/uwtools/tests/test_scheduler.py @@ -1,317 +1,255 @@ -# pylint: disable=missing-function-docstring,redefined-outer-name +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name """ Tests for uwtools.scheduler module. """ -import os -from pathlib import Path +from typing import Any, Dict from unittest.mock import patch from pytest import fixture, raises from uwtools import scheduler +from uwtools.exceptions import UWConfigError from uwtools.scheduler import JobScheduler -from uwtools.tests.support import compare_files -# LSF tests +# Fixtures + +directive_separator = "=" +managed_directives = {"account": lambda x: f"--a={x}", "walltime": lambda x: f"--t={x}"} +prefix = "#DIR" +submit_cmd = "sub" + + +@fixture +def props(): + return {"scheduler": "slurm", "walltime": "01:10:00", "account": "foo", "--pi": 3.14} + + +@fixture +def schedulerobj(props): + return ConcreteScheduler(props=props) @fixture -def lsf_props(): - config = { - "account": "account_name", - "nodes": 1, - "queue": "batch", - "scheduler": "lsf", - "tasks_per_node": 1, - "threads": 1, - "walltime": "00:01:00", - } - expected = [ - "#BSUB -P account_name", - "#BSUB -R affinity[core(1)]", - "#BSUB -R span[ptile=1]", - "#BSUB -W 00:01:00", - "#BSUB -n 1", - "#BSUB -q batch", - ] - return config, expected - - -def test_lsf_1(lsf_props): - lsf_config, expected_items = lsf_props - expected = "\n".join(expected_items) - assert JobScheduler.get_scheduler(lsf_config).batch_script.content() == expected - - -def test_lsf_2(lsf_props): - lsf_config, expected_items = lsf_props - lsf_config.update({"tasks_per_node": 12}) - expected_items[2] = "#BSUB -R span[ptile=12]" - expected_items[4] = "#BSUB -n 12" - expected = "\n".join(expected_items) - assert JobScheduler.get_scheduler(lsf_config).batch_script.content() == expected - - -def test_lsf_3(lsf_props): - lsf_config, expected_items = lsf_props - lsf_config.update({"nodes": 2, "tasks_per_node": 6}) - expected_items[2] = "#BSUB -R span[ptile=6]" - expected_items[4] = "#BSUB -n 12" - expected = "\n".join(expected_items) - assert JobScheduler.get_scheduler(lsf_config).batch_script.content() == expected - - -def test_lsf_4(lsf_props): - lsf_config, expected_items = lsf_props - lsf_config.update({"memory": "1MB", "nodes": 2, "tasks_per_node": 3, "threads": 2}) - expected_items[1] = "#BSUB -R affinity[core(2)]" - expected_items[2] = "#BSUB -R span[ptile=3]" - expected_items[4] = "#BSUB -n 6" - new_items = [ - "#BSUB -R rusage[mem=1000KB]", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(lsf_config).batch_script.content() == expected - - -def test_lsf_5(lsf_props): - lsf_config, _ = lsf_props - expected = "bsub" - assert JobScheduler.get_scheduler(lsf_config).submit_command == expected - - -# PBS tests +def lsf(props): + return scheduler.LSF(props=props) @fixture -def pbs_props(): - config = { - "account": "account_name", - "nodes": 1, - "queue": "batch", - "scheduler": "pbs", - "tasks_per_node": 1, - "walltime": "00:01:00", - } - expected = [ - "#PBS -A account_name", - "#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1", - "#PBS -l walltime=00:01:00", - "#PBS -q batch", - ] - return config, expected - - -def test_pbs_1(pbs_props): - pbs_config, expected_items = pbs_props - expected = "\n".join(expected_items) - assert JobScheduler.get_scheduler(pbs_config).batch_script.content() == expected - - -def test_pbs_2(pbs_props): - pbs_config, expected_items = pbs_props - pbs_config.update({"memory": "512M", "tasks_per_node": 4}) - expected_items[1] = "#PBS -l select=1:mpiprocs=4:ompthreads=1:ncpus=4:mem=512M" - expected = "\n".join(expected_items) - assert JobScheduler.get_scheduler(pbs_config).batch_script.content() == expected - - -def test_pbs_3(pbs_props): - pbs_config, expected_items = pbs_props - pbs_config.update({"nodes": 3, "tasks_per_node": 4, "threads": 2}) - expected_items[1] = "#PBS -l select=3:mpiprocs=4:ompthreads=2:ncpus=8" - expected = "\n".join(expected_items) - assert JobScheduler.get_scheduler(pbs_config).batch_script.content() == expected - - -def test_pbs_4(pbs_props): - pbs_config, expected_items = pbs_props - pbs_config.update({"memory": "512M", "nodes": 3, "tasks_per_node": 4, "threads": 2}) - expected_items[1] = "#PBS -l select=3:mpiprocs=4:ompthreads=2:ncpus=8:mem=512M" - expected = "\n".join(expected_items) - assert JobScheduler.get_scheduler(pbs_config).batch_script.content() == expected - - -def test_pbs_5(pbs_props): - pbs_config, expected_items = pbs_props - pbs_config.update({"exclusive": "True"}) - new_items = [ - "#PBS -l place=excl", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(pbs_config).batch_script.content() == expected - - -def test_pbs_6(pbs_props): - pbs_config, expected_items = pbs_props - pbs_config.update({"exclusive": False, "placement": "vscatter"}) - new_items = [ - "#PBS -l place=vscatter", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(pbs_config).batch_script.content() == expected - - -def test_pbs_7(pbs_props): - pbs_config, expected_items = pbs_props - pbs_config.update({"exclusive": True, "placement": "vscatter"}) - new_items = [ - "#PBS -l place=vscatter:excl", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(pbs_config).batch_script.content() == expected - - -def test_pbs_8(pbs_props): - pbs_config, expected_items = pbs_props - pbs_config.update({"debug": "True"}) - new_items = [ - "#PBS -l debug=true", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(pbs_config).batch_script.content() == expected - - -def test_pbs_9(pbs_props): - pbs_config, _ = pbs_props - expected = "qsub" - assert JobScheduler.get_scheduler(pbs_config).submit_command == expected - - -# Slurm tests +def pbs(props): + return scheduler.PBS(props=props) @fixture -def slurm_props(): - config = { - "account": "account_name", - "nodes": 1, - "queue": "batch", - "scheduler": "slurm", - "tasks_per_node": 1, - "walltime": "00:01:00", - } - expected = [ - "#SBATCH --account=account_name", - "#SBATCH --nodes=1", - "#SBATCH --ntasks-per-node=1", - "#SBATCH --qos=batch", - "#SBATCH --time=00:01:00", - ] - return config, expected - - -def test_slurm_1(slurm_props): - slurm_config, expected_items = slurm_props - expected = "\n".join(expected_items) - assert JobScheduler.get_scheduler(slurm_config).batch_script.content() == expected - - -def test_slurm_2(slurm_props): - slurm_config, expected_items = slurm_props - slurm_config.update({"partition": "debug"}) - new_items = [ - "#SBATCH --partition=debug", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(slurm_config).batch_script.content() == expected - - -def test_slurm_3(slurm_props): - slurm_config, expected_items = slurm_props - slurm_config.update({"tasks_per_node": 2, "threads": 4}) - expected_items[2] = "#SBATCH --ntasks-per-node=2" - new_items = [ - "#SBATCH --cpus-per-task=4", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(slurm_config).batch_script.content() == expected - - -def test_slurm_4(slurm_props): - slurm_config, expected_items = slurm_props - slurm_config.update({"memory": "4MB", "tasks_per_node": 2}) - expected_items[2] = "#SBATCH --ntasks-per-node=2" - new_items = [ - "#SBATCH --mem=4MB", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(slurm_config).batch_script.content() == expected - - -def test_slurm_5(slurm_props): - slurm_config, expected_items = slurm_props - slurm_config.update({"exclusive": "True"}) - new_items = [ - "#SBATCH --exclusive", - ] - expected = "\n".join(sorted(expected_items + new_items)) - assert JobScheduler.get_scheduler(slurm_config).batch_script.content() == expected - - -def test_slurm_6(slurm_props): - slurm_config, _ = slurm_props - expected = "sbatch" - assert JobScheduler.get_scheduler(slurm_config).submit_command == expected - - -# Generic tests using PBS support. - - -def test_batchscript_dump(pbs_props, tmpdir): - outfile = tmpdir / "outfile.sh" - pbs_config, expected_items = pbs_props - bs = JobScheduler.get_scheduler(pbs_config).batch_script - bs.dump(outfile) - reference = tmpdir / "reference.sh" - with open(reference, "w", encoding="utf-8") as f: - f.write("\n".join(["#!/bin/bash"] + expected_items)) - assert compare_files(reference, outfile) - - -def test_scheduler_bad_attr(pbs_props): - pbs_config, _ = pbs_props - js = JobScheduler.get_scheduler(pbs_config) - with raises(AttributeError): - assert js.bad_attr - - -def test_scheduler_bad_scheduler(): - with raises(KeyError) as e: - JobScheduler.get_scheduler({"scheduler": "FOO"}) - assert "FOO is not a supported scheduler" in str(e.value) - - -def test_scheduler_dot_notation(pbs_props): - pbs_config, _ = pbs_props - js = JobScheduler.get_scheduler(pbs_config) - assert js.account == "account_name" - - -def test_scheduler_prop_not_defined_raises_key_error(pbs_props): - pbs_config, _ = pbs_props - del pbs_config["scheduler"] - with raises(KeyError) as e: - JobScheduler.get_scheduler(pbs_config) - assert "No scheduler defined in props" in str(e.value) - - -def test_scheduler_raises_exception_when_missing_required_attrs(pbs_props): - pbs_config, _ = pbs_props - del pbs_config["account"] - with raises(ValueError) as e: - JobScheduler.get_scheduler(pbs_config) - assert "Missing required attributes: [account]" in str(e.value) +def slurm(props): + return scheduler.Slurm(props=props) + + +class ConcreteScheduler(scheduler.JobScheduler): + @property + def _directive_separator(self) -> str: + return directive_separator + + @property + def _managed_directives(self) -> Dict[str, Any]: + return managed_directives + + @property + def _prefix(self) -> str: + return prefix + + @property + def _submit_cmd(self) -> str: + return submit_cmd + + +def test_JobScheduler(schedulerobj): + assert isinstance(schedulerobj, JobScheduler) + + +def test_JobScheduler_directives(schedulerobj): + assert schedulerobj.directives == ["#DIR --a=foo", "#DIR --pi=3.14", "#DIR --t=01:10:00"] + + +def test_JobScheduler_get_scheduler_bad_scheduler_specified(): + with raises(UWConfigError) as e: + ConcreteScheduler.get_scheduler(props={"scheduler": "foo"}) + assert str(e.value).startswith("Scheduler 'foo' should be one of:") + + +def test_JobScheduler_get_scheduler_no_scheduler_specified(): + with raises(UWConfigError) as e: + ConcreteScheduler.get_scheduler(props={}) + assert str(e.value).startswith("No 'scheduler' defined in") -def test_scheduler_submit_job(pbs_props): - pbs_config, _ = pbs_props - js = JobScheduler.get_scheduler(pbs_config) - submit_command = js.submit_command - outpath = Path("/path/to/batch/script") - expected_command = f"{submit_command} {outpath}" +def test_JobScheduler_get_scheduler_ok(props): + assert isinstance(ConcreteScheduler.get_scheduler(props=props), scheduler.Slurm) + + +def test_JobScheduler_submit_job(schedulerobj, tmp_path): + runscript = tmp_path / "runscript" + submit_file = tmp_path / "runscript.submit" with patch.object(scheduler, "execute") as execute: - execute.return_value = (True, "") - js.submit_job(outpath) - execute.assert_called_once_with(cmd=expected_command, cwd=os.path.dirname(outpath)) + execute.return_value = (True, None) + assert schedulerobj.submit_job(runscript=runscript, submit_file=submit_file) is True + execute.assert_called_once_with( + cmd=f"sub {runscript} 2>&1 | tee {submit_file}", cwd=str(tmp_path) + ) + + +def test_JobScheduler__directive_separator(schedulerobj): + assert schedulerobj._directive_separator == directive_separator + + +def test_JobScheduler__managed_directives(schedulerobj): + assert schedulerobj._managed_directives == managed_directives + + +def test_JobScheduler__prefix(schedulerobj): + assert schedulerobj._prefix == prefix + + +def test_JobScheduler__processed_props(props, schedulerobj): + del props["scheduler"] + assert schedulerobj._processed_props == props + + +def test_JobScheduler__submit_cmd(schedulerobj): + assert schedulerobj._submit_cmd == submit_cmd + + +def test_JobScheduler__validate_props_no(schedulerobj): + schedulerobj._props = {} + with raises(UWConfigError) as e: + schedulerobj._validate_props() + assert str(e.value) == "Missing required directives: account, walltime" + + +def test_JobScheduler__validate_props_ok(schedulerobj): + assert schedulerobj._validate_props() is None + + +def test_LSF(lsf): + assert isinstance(lsf, JobScheduler) + + +def test_LSF__directive_separator(lsf): + assert lsf._directive_separator == " " + + +def test_LSF__managed_directives(lsf): + mds = lsf._managed_directives + assert mds["account"] == "-P" + assert mds["jobname"] == "-J" + assert mds["memory"]("1GB") == "-R rusage[mem=1GB]" + assert mds["nodes"](2) == "-n 2" + assert mds["queue"] == "-q" + assert mds["shell"] == "-L" + assert mds["stdout"] == "-o" + assert mds["tasks_per_node"](4) == "-R span[ptile=4]" + assert mds["threads"](8) == "-R affinity[core(8)]" + assert mds["walltime"] == "-W" + + +def test_LSF__prefix(lsf): + assert lsf._prefix == "#BSUB" + + +def test_LSF__processed_props(lsf): + assert lsf._processed_props == {**lsf._props, "threads": 1} + + +def test_LSF__submit_cmd(lsf): + assert lsf._submit_cmd == "bsub" + + +def test_PBS(pbs): + assert isinstance(pbs, JobScheduler) + + +def test_PBS__directive_separator(pbs): + assert pbs._directive_separator == " " + + +def test_PBS__managed_directives(pbs): + mds = pbs._managed_directives + assert mds["account"] == "-A" + assert mds["debug"](True) == "-l debug=true" + assert mds["jobname"] == "-N" + assert mds["memory"] == "mem" + assert mds["nodes"](2) == "-l select=2" + assert mds["queue"] == "-q" + assert mds["shell"] == "-S" + assert mds["stdout"] == "-o" + assert mds["tasks_per_node"] == "mpiprocs" + assert mds["threads"] == "ompthreads" + assert mds["walltime"] == "-l walltime=" + + +def test_PBS__placement(pbs): + extras = {"placement": "foo", "exclusive": True} + assert pbs._placement(items={**pbs._props, **extras})["-l place="] == "foo:excl" + + +def test_PBS__placement_no_op(pbs): + assert pbs._placement(items=pbs._props) == pbs._props + + +def test_PBS__prefix(pbs): + assert pbs._prefix == "#PBS" + + +def test_PBS__processed_props(pbs): + assert pbs._processed_props == pbs._props + + +def test_PBS__select(pbs): + expected = "2:mpiprocs=4:ompthreads=1:ncpus=4:mem=1GB" + extras = {"nodes": 2, "tasks_per_node": 4, "memory": "1GB"} + assert pbs._select(items={**pbs._props, **extras})["-l select="] == expected + + +def test_PBS__submit_cmd(pbs): + assert pbs._submit_cmd == "qsub" + + +def test_Slurm(slurm): + assert isinstance(slurm, JobScheduler) + + +def test_Slurm__directive_separator(slurm): + assert slurm._directive_separator == "=" + + +def test_Slurm__managed_directives(slurm): + mds = slurm._managed_directives + assert mds["cores"] == "--ntasks" + assert mds["exclusive"](None) == "--exclusive" + assert mds["export"] == "--export" + assert mds["jobname"] == "--job-name" + assert mds["memory"] == "--mem" + assert mds["nodes"] == "--nodes" + assert mds["partition"] == "--partition" + assert mds["queue"] == "--qos" + assert mds["stderr"] == "--error" + assert mds["stdout"] == "--output" + assert mds["tasks_per_node"] == "--ntasks-per-node" + assert mds["threads"] == "--cpus-per-task" + assert mds["account"] == "--account" + assert mds["walltime"] == "--time" + + +def test_Slurm__prefix(slurm): + assert slurm._prefix == "#SBATCH" + + +def test_Slurm__processed_props(slurm): + assert slurm._processed_props == slurm._props + + +def test_Slurm__submit_cmd(slurm): + assert slurm._submit_cmd == "sbatch"