added BASH wrapper to create_experiment.py for user access from comma…

…nd line, removed redundent seceond argument to it as well

.github/workflows/globalworkflow-ci.yaml

@@ -1,13 +1,20 @@
 name: gw-ci-orion
-# A GitHub Action that creates a deployment for a PR, clones, builds and runs a test suite
-# This workflow is triggered by a label being added to a PR name Orion-Ready
 
 on: [workflow_dispatch]
 
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+#     ├── COMROT
+#     │   └── ${pslot}
+#     └── EXPDIR
+#         └── ${pslot}
 env:
-  pr: ${{ github.run_id }}
-  HOMEgfs_PR: ${{ github.workspace }}/${{ github.run_id }}
-  RUNTESTS: ${{ github.workspace }}/RUNTESTS/${{ github.run_id }}
+  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
   MACHINE_ID: orion
 
 jobs:
@@ -16,65 +23,64 @@ jobs:
     timeout-minutes: 600
 
     steps:
-    - name: Checkout
+    - name: Checkout global-workflow
       uses: actions/checkout@v3
       with:
-        path: ${{ github.run_id }}
+        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
 
     - name: Checkout components
       run: |
-        cd ${{ env.HOMEgfs_PR }}/sorc
-        ./checkout.sh  # Options e.g. -g -u can be added late
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./checkout.sh -c -g  # Options e.g. -u can be added late
 
     - name: Build components
       run: |
-        cd ${{ env.HOMEgfs_PR }}/sorc
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
         ./build_all.sh
 
     - name: Link artifacts
       run: |
-        cd ${{ env.HOMEgfs_PR }}/sorc
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
         ./link_workflow.sh
 
   create-experiments:
     needs: checkout-build-link
     runs-on: [self-hosted, orion-ready]
     strategy:
       matrix:
-       case: ["C48_S2S","C96_atm3DVar"]
+        case: ["C48_S2S", "C96_atm3DVar"]
 
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
+          HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs
+          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
           pslot: ${{ matrix.case }}.${{ github.run_id }}
         run: |
-         cd ${{ env.HOMEgfs_PR }}
-         source workflow/gw_setup.sh
-         source ci/platforms/orion.sh
-         ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }}
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          source workflow/gw_setup.sh
+          source ci/platforms/orion.sh
+          ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
 
   run-experiments:
     needs: create-experiments
     runs-on: [self-hosted, orion-ready]
     strategy:
       max-parallel: 2
       matrix:
-       case: ["C48_S2S", "C96_atm3DVar"]
+        case: ["C48_S2S", "C96_atm3DVar"]
     steps:
       - name: Run Experiment ${{ matrix.case }}
-        env:
-          pr: ${{ env.pr }}
         run: |
-         cd ${{ env.HOMEgfs_PR }}
-         ./ci/scripts/run-check_ci.sh ${{ env.HOMEgfs_PR }} ${{ env.RUNTESTS }}/EXPDIR ${{ matrix.case }}.${{ github.run_id }}
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }}
 
-  release-lock:
+  clean-up:
     needs: run-experiments
     runs-on: [self-hosted, orion-ready]
     steps:
-      - uses: actions-ecosystem/action-remove-labels@v1
-        with:
-          labels: "CI-Orion-Running"
-      - uses: actions-ecosystem/action-add-labels@v1
-        with:
-          labels: "CI-Orion-Passed"
+      - name: Clean-up
+        run: |
+          cd ${{ github.workspace }}
+          rm -rf ${{ github.run_id }}
+

ci/scripts/create_experiment.py

@@ -2,107 +2,87 @@
 
 """
 Basic python script to create an experiment directory on the fly from a given
-
 yaml file for the arguments to the two scripts below in ${HOMEgfs}/workflow
-
 where ${HOMEgfs} is specified within the input yaml file.
 
  ${HOMEgfs}/workflow/setup_expt.py
  ${HOMEgfs}/workflow/setup_xml.py
 
 The yaml file are simply the arguments for these two scripts.
-After this scripts runs these two the use will have an experiment ready for launching
+After this scripts runs the experiment is ready for launch.
 
 Output
 ------
-
 Functionally an experiment is setup as a result running the two scripts described above
 with an error code of 0 upon success.
 """
 
 import os
+import sys
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from pathlib import Path
 
-from wxflow import YAMLFile, Logger, logit, Executable
+from wxflow import YAMLFile, Logger, logit
 
 
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../..'))
 logger = Logger(level='DEBUG', colored_log=True)
 
 
+# TODO: move create_experiment.py to workflow/ and remove this sys.path.insert business
+sys.path.insert(0, os.path.join(_top, 'workflow'))
+import setup_expt
+import setup_xml
+
+
 @logit(logger)
 def input_args():
     """
-    Method to collect user arguments for `create_experiment.py`
-
-    Input
-    -----
-
-    A single key valued argument: --yaml <full path to YAML file>
-
     Description
     -----------
 
-    A full path to a YAML file with the following format with required sections: experiment, arguments
+    Method to collect user arguments for `create_experiment.py`
 
-    experiment:
-        mode: <cycled> <forecast-only>
-            used to hold the only required positional argument to setup_expt.py
+    Parameters
+    ----------
 
-    arguments:
-        holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py
-        Note: the argument pslot is derived from the basename of the yamlfile itself
+    None
 
     Returns
     -------
 
-    args: Namespace
-
-        Namespace with the value of the file path to a yaml file from the key yaml
+    argparse.Namespace:
+        argparse.Namespace with the value of the file path to a yaml file from the key yaml
     """
 
-    description = """Single argument as a yaml file containing the
-    key value pairs as arguments to setup_expt.py
-    """
+    description = """Create a global-workflow experiment"""
 
     parser = ArgumentParser(description=description,
                             formatter_class=ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('--yaml', help='yaml configuration file per experiment', type=str, required=True)
-    parser.add_argument('--dir', help='full path to top level of repo of global-workflow', type=str, required=True)
+    parser.add_argument('--yaml', help='full path to yaml file describing the experiment configuration', type=str, required=True)
 
-    args = parser.parse_args()
-    return args
+    return parser.parse_args()
 
 
 if __name__ == '__main__':
 
     user_inputs = input_args()
-    setup_expt_args = YAMLFile(path=user_inputs.yaml)
-
-    HOMEgfs = Path.absolute(Path(user_inputs.dir))
-    type = setup_expt_args.experiment.type
-    mode = setup_expt_args.experiment.mode
-
-    setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py'))
-
-    setup_expt_cmd.add_default_arg(type)
-    setup_expt_cmd.add_default_arg(mode)
+    testconf = YAMLFile(path=user_inputs.yaml)
+    experiment_dir = Path.absolute(Path.joinpath(Path(testconf.arguments.expdir), Path(testconf.arguments.pslot)))
 
-    for conf, value in setup_expt_args.arguments.items():
-        setup_expt_cmd.add_default_arg(f'--{conf}')
-        setup_expt_cmd.add_default_arg(str(value))
+    # Create a list of arguments to setup_expt.py
+    setup_expt_args = [testconf.experiment.type, testconf.experiment.mode]  # TODO: rename 'type' as 'system' in case.yaml
+    for kk, vv in testconf.arguments.items():
+        setup_expt_args.append(f"--{kk}")
+        setup_expt_args.append(str(vv))
 
-    logger.info(f'Run command: {setup_expt_cmd.command}')
-    setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr'))
-    setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout'))
-    setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr)
+    logger.info(f'Call: setup_expt.main()')
+    setup_expt.main(setup_expt_args)
 
-    setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py'))
-    expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir),Path(setup_expt_args.arguments.pslot)))
-    setup_xml_cmd.add_default_arg(str(expdir))
+    # Create a list of arguments to setup_xml.py
+    setup_xml_args = [str(experiment_dir)]
 
-    logger.info(f'Run command: {setup_xml_cmd.command}')
-    setup_xml_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stderr'))
-    setup_xml_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stdout'))
-    setup_xml_cmd(output=setup_xml_stdout, error=setup_xml_stderr)
+    logger.info(f"Call: setup_xml.main()")
+    setup_xml.main(setup_xml_args)

ci/scripts/create_experiment.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -eux
+
+#########################################################################
+#
+# Script description: BASH script for creating an experiment
+#                     by only taking a single YAML file
+########################################################################
+
+usage() {
+  set +x
+  echo
+  echo "Usage: $0 -y case/yaml_conf_file.yaml"
+  echo
+  echo "  -c  global workflow configuration yaml defining an experiment (\$HOMEgfs/ci/cases)"
+  echo "  -d  full path to directory to place COMROT with EXPDIR for createing experiments"
+  echo "  -h  display this message and quit"
+  echo
+  exit 1
+}
+
+while getopts "c:d:h" opt; do
+  case ${opt} in
+    c)
+      YAML_CASE=${OPTARG}
+      ;;
+    d)
+      RUNTESTS=${OPTARG}
+      ;;
+    h|\?|:)
+      usage
+      ;;
+    *)
+      echo "Unrecognized option"
+      usage
+     ;;
+  esac
+done
+
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+scriptname=$(basename "${BASH_SOURCE[0]}")
+echo "Begin ${scriptname} at $(date -u)" || true
+export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
+
+#########################################################################
+#  Set up runtime environment varibles for accounts on supproted machines
+#########################################################################
+
+source "${HOMEgfs}/ush/detect_machine.sh"
+case ${MACHINE_ID} in
+  hera | orion)
+   echo "Running Automated Testing on ${MACHINE_ID}"
+   source "${HOMEgfs}/ci/platforms/${MACHINE_ID}.sh"
+   ;;
+ *)
+   echo "Unsupported platform. Exiting with error."
+   exit 1
+   ;;
+esac
+
+source "${HOMEgfs}/ci/platforms/${MACHINE_ID}.sh"
+
+filename=$(basename -- ${YAML_CASE})
+export pslot="${filename%.*}"
+export RUNTESTS=${RUNTESTS:-"${PWD}"}
+# TODO env HOMEgfs being set will cause runtime error
+# with HOMEgfs being blank in Rocoto XLM file
+export HOMEgfs_PR="${HOMEgfs}"
+unset HOMEgfs;
+
+${HOMEgfs_PR}/ci/scripts/create_experiment.py --yaml $YAML_CASE

ci/scripts/driver.sh

@@ -13,7 +13,6 @@ set -eux
 # PR number and calls clone-build_ci.sh to perform a clone and full build from $(HOMEgfs)/sorc
 # of the PR. It then is ready to run a suite of regression tests with various
 # configurations with run_tests.py.
-# No-op for test
 #######################################################################################
 
 #################################################################
@@ -149,14 +148,14 @@ for pr in ${pr_list}; do
     HOMEgfs_PR="${pr_dir}/global-workflow"
     export HOMEgfs_PR
     cd "${HOMEgfs_PR}"
-    discribe=$(git rev-parse --short HEAD)
+    pr_sha=$(git rev-parse --short HEAD)
 
     for yaml_config in "${HOMEgfs_PR}/ci/cases/"*.yaml; do
       case=$(basename "${yaml_config}" .yaml) || true
-      pslot="${pslot}_${discribe}"
+      pslot="${case}_${pr_sha}"
       export pslot
       set +e
-      "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml" --dir "${HOMEgfs_PR}"
+      "${HOMEgfs_PR}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml"
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then