BASH wrapper to create_experiment.py

.github/workflows/globalworkflow-ci.yaml

@@ -0,0 +1,86 @@
+name: gw-ci-orion
+
+on: [workflow_dispatch]
+
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+#     ├── COMROT
+#     │   └── ${pslot}
+#     └── EXPDIR
+#         └── ${pslot}
+env:
+  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
+  MACHINE_ID: orion
+
+jobs:
+  checkout-build-link:
+    runs-on: [self-hosted, orion-ready]
+    timeout-minutes: 600
+
+    steps:
+    - name: Checkout global-workflow
+      uses: actions/checkout@v3
+      with:
+        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
+
+    - name: Checkout components
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./checkout.sh -c -g  # Options e.g. -u can be added late
+
+    - name: Build components
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./build_all.sh
+
+    - name: Link artifacts
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./link_workflow.sh
+
+  create-experiments:
+    needs: checkout-build-link
+    runs-on: [self-hosted, orion-ready]
+    strategy:
+      matrix:
+        case: ["C48_S2S", "C96_atm3DVar"]
+
+    steps:
+      - name: Create Experiments ${{ matrix.case }}
+        env:
+          HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs
+          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
+          pslot: ${{ matrix.case }}.${{ github.run_id }}
+        run: |
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          source workflow/gw_setup.sh
+          source ci/platforms/orion.sh
+          ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
+
+  run-experiments:
+    needs: create-experiments
+    runs-on: [self-hosted, orion-ready]
+    strategy:
+      max-parallel: 2
+      matrix:
+        case: ["C48_S2S", "C96_atm3DVar"]
+    steps:
+      - name: Run Experiment ${{ matrix.case }}
+        run: |
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }}
+
+  clean-up:
+    needs: run-experiments
+    runs-on: [self-hosted, orion-ready]
+    steps:
+      - name: Clean-up
+        run: |
+          cd ${{ github.workspace }}
+          rm -rf ${{ github.run_id }}
+

ci/cases/C48_S2S.yaml

@@ -3,10 +3,11 @@ experiment:
   mode: forecast-only
 
 arguments:
+  pslot: ${pslot}
   app: S2S
   resdet: 48
-  comrot: ${RUNTESTS}/${pslot}/COMROT
-  expdir: ${RUNTESTS}/${pslot}/EXPDIR
+  comrot: ${RUNTESTS}/COMROT
+  expdir: ${RUNTESTS}/EXPDIR
   idate: 2021032312
   edate: 2021032312
   yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml

ci/cases/C96_atm3DVar.yaml

@@ -3,10 +3,11 @@ experiment:
   mode: cycled
 
 arguments:
+  pslot: ${pslot}
   app: ATM
   resdet: 96
-  comrot: ${RUNTESTS}/${pslot}/COMROT
-  expdir: ${RUNTESTS}/${pslot}/EXPDIR
+  comrot: ${RUNTESTS}/COMROT
+  expdir: ${RUNTESTS}/EXPDIR
   icsdir: ${ICSDIR_ROOT}/C96C48
   idate: 2021122018
   edate: 2021122106

ci/platforms/orion.sh

@@ -5,3 +5,4 @@ export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
 export SLURM_ACCOUNT=nems
 export max_concurrent_cases=2
 export max_concurrent_pr=2
+export GH=/home/mterry/bin/gh

ci/scripts/check_ci.sh

@@ -6,7 +6,6 @@ set -eux
 #                     running rocotostat on each to determine if the experiment has
 #                     succeeded or faild.  This script is intended
 #                     to run from within a cron job in the CI Managers account
-# Abstract TODO
 #####################################################################################
 
 HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
@@ -78,11 +77,10 @@ for pr in ${pr_list}; do
   if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then
      continue
   fi
-  num_cases=$(find "${pr_dir}/RUNTESTS" -mindepth 1 -maxdepth 1 -type d | wc -l) || true
 
-  #Check for PR success  when ${pr_dir}/RUNTESTS is void of subfolders
+  #Check for PR success  when ${pr_dir}/RUNTESTS/EXPDIR is void of subfolders
   # since all successfull ones where previously removed
-  if [[ "${num_cases}" -eq 0 ]] && [[ -d "${pr_dir}/RUNTESTS" ]]; then
+  if [[ -z "$(ls -A ${pr_dir}/RUNTESTS/EXPDIR)" ]] ; then
     "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed"
     sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
@@ -92,14 +90,14 @@ for pr in ${pr_list}; do
     continue 
   fi
 
-  for cases in "${pr_dir}/RUNTESTS/"*; do
+  for cases in "${pr_dir}/RUNTESTS/EXPDIR/"*; do
     pslot=$(basename "${cases}") || true
     if [[ -z "${pslot+x}" ]]; then
-      echo "No cases found in ${pr_dir}/RUNTESTS .. exiting"
+      echo "No cases found in ${pr_dir}/RUNTESTS/EXPDIR .. exiting"
       exit 0
     fi
-    xml="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.xml"
-    db="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.db"
+    xml="${pr_dir}/RUNTESTS/EXPDIR/${pslot}/${pslot}.xml"
+    db="${pr_dir}/RUNTESTS/EXPDIR/${pslot}/${pslot}.db"
     if [[ ! -f "${db}" ]]; then
        continue
     fi
@@ -138,7 +136,8 @@ for pr in ${pr_list}; do
       sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
       #Remove Experment cases that completed successfully
-      rm -Rf "${pr_dir}/RUNTESTS/${pslot}"
+      rm -Rf "${pr_dir}/RUNTESTS/EXPDIR/${pslot}"
+      rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}"
     fi
   done
 done

ci/scripts/clone-build_ci.sh

@@ -39,6 +39,8 @@ while getopts "p:d:o:h" opt; do
   esac
 done
 
+mkdir -p "${repodir}"
+
 cd "${repodir}" || exit 1
 # clone copy of repo
 if [[ -d global-workflow ]]; then
@@ -78,7 +80,7 @@ echo "${commit}" > "../commit"
 # run checkout script
 cd sorc || exit 1
 set +e
-./checkout.sh -c -g -u >> log.checkout 2>&1
+./checkout.sh -c -g >> log.checkout 2>&1
 checkout_status=$?
 if [[ ${checkout_status} != 0 ]]; then
   {

ci/scripts/create_experiment.py

@@ -2,111 +2,87 @@
 
 """
 Basic python script to create an experiment directory on the fly from a given
-
 yaml file for the arguments to the two scripts below in ${HOMEgfs}/workflow
-
 where ${HOMEgfs} is specified within the input yaml file.
 
  ${HOMEgfs}/workflow/setup_expt.py
  ${HOMEgfs}/workflow/setup_xml.py
 
 The yaml file are simply the arguments for these two scripts.
-After this scripts runs these two the use will have an experiment ready for launching
+After this scripts runs the experiment is ready for launch.
 
 Output
 ------
-
 Functionally an experiment is setup as a result running the two scripts described above
 with an error code of 0 upon success.
 """
 
+import os
+import sys
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from pathlib import Path
 
-from wxflow import YAMLFile, Logger, logit, Executable
+from wxflow import YAMLFile, Logger, logit
 
 
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../..'))
 logger = Logger(level='DEBUG', colored_log=True)
 
 
+# TODO: move create_experiment.py to workflow/ and remove this sys.path.insert business
+sys.path.insert(0, os.path.join(_top, 'workflow'))
+import setup_expt
+import setup_xml
+
+
 @logit(logger)
 def input_args():
     """
-    Method to collect user arguments for `create_experiment.py`
-
-    Input
-    -----
-
-    A single key valued argument: --yaml <full path to YAML file>
-
     Description
     -----------
 
-    A full path to a YAML file with the following format with required sections: experiment, arguments
+    Method to collect user arguments for `create_experiment.py`
 
-    experiment:
-        mode: <cycled> <forecast-only>
-            used to hold the only required positional argument to setup_expt.py
+    Parameters
+    ----------
 
-    arguments:
-        holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py
-        Note: the argument pslot is derived from the basename of the yamlfile itself
+    None
 
     Returns
     -------
 
-    args: Namespace
-
-        Namespace with the value of the file path to a yaml file from the key yaml
+    argparse.Namespace:
+        argparse.Namespace with the value of the file path to a yaml file from the key yaml
     """
 
-    description = """Single argument as a yaml file containing the
-    key value pairs as arguments to setup_expt.py
-    """
+    description = """Create a global-workflow experiment"""
 
     parser = ArgumentParser(description=description,
                             formatter_class=ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('--yaml', help='yaml configuration file per experiment', type=str, required=True)
-    parser.add_argument('--dir', help='full path to top level of repo of global-workflow', type=str, required=True)
+    parser.add_argument('--yaml', help='full path to yaml file describing the experiment configuration', type=str, required=True)
 
-    args = parser.parse_args()
-    return args
+    return parser.parse_args()
 
 
 if __name__ == '__main__':
 
     user_inputs = input_args()
-    setup_expt_args = YAMLFile(path=user_inputs.yaml)
-
-    HOMEgfs = Path.absolute(Path(user_inputs.dir))
-    pslot = Path(user_inputs.yaml).stem
-    type = setup_expt_args.experiment.type
-    mode = setup_expt_args.experiment.mode
-
-    setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py'))
-
-    setup_expt_cmd.add_default_arg(type)
-    setup_expt_cmd.add_default_arg(mode)
-
-    for conf, value in setup_expt_args.arguments.items():
-        setup_expt_cmd.add_default_arg(f'--{conf}')
-        setup_expt_cmd.add_default_arg(str(value))
+    testconf = YAMLFile(path=user_inputs.yaml)
+    experiment_dir = Path.absolute(Path.joinpath(Path(testconf.arguments.expdir), Path(testconf.arguments.pslot)))
 
-    setup_expt_cmd.add_default_arg('--pslot')
-    setup_expt_cmd.add_default_arg(pslot)
+    # Create a list of arguments to setup_expt.py
+    setup_expt_args = [testconf.experiment.type, testconf.experiment.mode]  # TODO: rename 'type' as 'system' in case.yaml
+    for kk, vv in testconf.arguments.items():
+        setup_expt_args.append(f"--{kk}")
+        setup_expt_args.append(str(vv))
 
-    logger.info(f'Run command: {setup_expt_cmd.command}')
-    setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr'))
-    setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout'))
-    print(setup_expt_stderr)
-    setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr)
+    logger.info(f'Call: setup_expt.main()')
+    setup_expt.main(setup_expt_args)
 
-    setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py'))
-    expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(pslot)))
-    setup_xml_cmd.add_default_arg(str(expdir))
+    # Create a list of arguments to setup_xml.py
+    setup_xml_args = [str(experiment_dir)]
 
-    logger.info(f'Run command: {setup_xml_cmd.command}')
-    setup_xml_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stderr'))
-    setup_xml_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stdout'))
-    setup_xml_cmd(output=setup_xml_stdout, error=setup_xml_stderr)
+    logger.info(f"Call: setup_xml.main()")
+    setup_xml.main(setup_xml_args)