Added two scripts and replced run_bufr2ioda.py by run_bufr2ioda.sh

rrfs-test/IODA/python/gen_bufr2ioda_json.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# gen_bufr2ioda_json.py
+# generate JSON from a template
+# as input to the various
+# python BUFR2IODA scripts
+import argparse
+import json
+import os
+from wxflow import Logger, parse_j2yaml, cast_strdict_as_dtypedict
+from wxflow import add_to_datetime, to_timedelta
+
+# Initialize root logger
+logger = Logger('gen_bufr2ioda_json.py', level='INFO', colored_log=True)
+
+
+def gen_bufr_json(config, template, output):
+    # read in templated JSON and do substitution
+    logger.info(f"Using {template} as input {config}")
+    bufr_config = parse_j2yaml(template, config)
+    # write out JSON
+    json_object = json.dumps(bufr_config, indent=4)
+    with open(output, "w") as outfile:
+        outfile.write(json_object)
+    logger.info(f"Wrote to {output}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-t', '--template', type=str, help='Input JSON template', required=True)
+    parser.add_argument('-o', '--output', type=str, help='Output JSON file', required=True)
+    args = parser.parse_args()
+    # get the config from your environment
+    config = cast_strdict_as_dtypedict(os.environ)
+    logger.info(f"Config: {config}")
+    # we need to add in current cycle from PDYcyc
+    config['current_cycle'] = add_to_datetime(config['PDY'], to_timedelta(f"{config['cyc']}H"))
+    # call the parsing function
+    gen_bufr_json(config, args.template, args.output)

rrfs-test/IODA/python/gen_bufr2ioda_yaml.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# gen_bufr2ioda_yaml.py
+# generate YAML for bufr2ioda.x
+# given a template
+# and certain configuration parameters
+import argparse
+import os
+from wxflow import Logger, parse_j2yaml, cast_strdict_as_dtypedict, save_as_yaml
+from wxflow import Template, TemplateConstants
+
+# initialize root logger
+logger = Logger('gen_bufr2ioda_yaml.py', level='INFO', colored_log=True)
+
+
+def gen_bufr_yaml(config, template, output):
+    # read in templated YAML and do substitution
+    logger.info(f"Using {template} as input")
+    bufr_config = parse_j2yaml(template, config)
+    # need to do some special manipulation for the splits
+    substitutions = {'splitvar': '{splits/satId}'}
+    bufr_config = Template.substitute_structure(bufr_config, TemplateConstants.DOLLAR_PARENTHESES, substitutions.get)
+    save_as_yaml(bufr_config, output)
+    logger.info(f"Wrote to {output}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-t', '--template', type=str, help='Input YAML template', required=True)
+    parser.add_argument('-o', '--output', type=str, help='Output YAML file', required=True)
+    args = parser.parse_args()
+    # get the config from your environment
+    config = cast_strdict_as_dtypedict(os.environ)
+    # call the parsing function
+    gen_bufr_yaml(config, args.template, args.output)

rrfs-test/IODA/python/run_bufr2ioda.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+#--------------------------------------------------------------------------------------
+# run_bufr2ioda.sh
+# This driver script will:
+# - generate config files from templates for each BUFR file
+# - use bufr2ioda python scripts or executable and produce output IODA files
+# usage:
+#       run_bufr2ioda.sh YYYYMMDDHH $DMPDIR /path/to/templates $COM_OBS
+#
+#--------------------------------------------------------------------------------------
+if [[ $# -ne 5 ]] ; then
+    echo "usage:"
+    echo "      $0 YYYYMMDDHH gdas|gfs /path/to/files.bufr_d/ /path/to/templates /path/to/output.ioda/"
+    exit 1
+fi
+
+# some of these need exported to be picked up by the python script below
+# input parameters
+CDATE=${CDATE:-$1}
+export DUMP=${RUN:-$2}
+export DMPDIR=${DMPDIR:-$3}
+config_template_dir=${config_template_dir:-$4}
+export COM_OBS=${COM_OBS:-$5}
+
+# derived parameters
+export PDY=$(echo $CDATE | cut -c1-8)
+export cyc=$(echo $CDATE | cut -c9-10)
+
+# get gdasapp root directory
+readonly DIR_ROOT=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/../../.." && pwd -P)
+BUFR2IODA=$DIR_ROOT/build/bin/bufr2ioda.x
+USH_IODA=$DIR_ROOT/rrfs-test/IODA/python
+BUFRYAMLGEN=$USH_IODA/gen_bufr2ioda_yaml.py
+BUFRJSONGEN=$USH_IODA/gen_bufr2ioda_json.py
+
+# create output directory if it doesn't exist
+mkdir -p $COM_OBS
+if [ $? -ne 0 ]; then
+    echo "cannot make $COM_OBS"
+    exit 1
+fi
+
+# add to pythonpath the necessary libraries
+PYIODALIB=`echo $DIR_ROOT/build/lib/python3.?`
+export PYTHONPATH=${PYIODALIB}:${PYTHONPATH}
+
+#----- python and json -----
+# first specify what observation types will be processed by a script
+#BUFR_py="msonet"
+BUFR_py="msonet_prepbufr"
+
+for obtype in $BUFR_py; do
+  # this loop assumes that there is a python script and template with the same name
+  echo "Processing ${obtype}..."
+
+  # first generate a JSON from the template
+  ${BUFRJSONGEN} -t ${config_template_dir}/bufr2ioda_${obtype}.json -o ${COM_OBS}/${obtype}_${PDY}${cyc}.json
+
+  # now use the converter script for the ob type
+  python $USH_IODA/bufr2ioda_${obtype}.py -c ${COM_OBS}/${obtype}_${PDY}${cyc}.json
+
+  # check if converter was successful
+  if [ $? == 0 ]; then
+    # remove JSON file
+    rm -rf ${COM_OBS}/${obtype}_${PDY}${cyc}.json
+  else
+    # warn and keep the JSON file
+    echo "Problem running converter script for ${obtype}"
+  fi
+done
+
+#----------------------------
+#---- bufr2ioda and yaml ----
+BUFR_yaml=""