diff --git a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.json b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.json index e3fdc151f..2598f08ef 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.json @@ -1,7 +1,7 @@ { "data_format" : "bufr_d", "data_type" : "satwnd", - "cycle_type" : "{{ DUMP }}", + "cycle_type" : "{{ RUN }}", "cycle_datetime" : "{{ current_cycle | to_YMDH }}", "dump_directory" : "{{ DMPDIR }}", "ioda_directory" : "{{ COM_OBS }}", diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py old mode 100644 new mode 100755 index 2a0f0e214..709dfa878 --- a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse import numpy as np import numpy.ma as ma diff --git a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py index 0748e20f6..617a8b024 100755 --- a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py +++ b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py @@ -6,8 +6,7 @@ import argparse import json import os -from wxflow import Logger, parse_j2yaml, cast_strdict_as_dtypedict -from wxflow import add_to_datetime, to_timedelta +from wxflow import Logger, parse_j2yaml # Initialize root logger logger = Logger('gen_bufr2ioda_json.py', level='INFO', colored_log=True) @@ -29,9 +28,4 @@ def gen_bufr_json(config, template, output): parser.add_argument('-t', '--template', type=str, help='Input JSON template', required=True) parser.add_argument('-o', '--output', type=str, help='Output JSON file', required=True) args = parser.parse_args() - # get the config from your environment - config = cast_strdict_as_dtypedict(os.environ) - # we need to add in current cycle from PDYcyc - config['current_cycle'] = add_to_datetime(config['PDY'], to_timedelta(f"{config['cyc']}H")) - # call the parsing function gen_bufr_json(config, args.template, args.output) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py new file mode 100755 index 000000000..caf38f16f --- /dev/null +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +import argparse +import os +from pathlib import Path +from gen_bufr2ioda_json import gen_bufr_json +from wxflow import (Logger, Executable, cast_as_dtype, logit, + to_datetime, datetime_to_YMDH, Task, rm_p) + +# Initialize root logger +logger = Logger('run_bufr2ioda.py', level='INFO', colored_log=True) + + +@logit(logger) +def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): + logger.info(f"Process {current_cycle} {RUN} from {DMPDIR} to {COM_OBS} using {config_template_dir}") + + # Get gdasapp root directory + DIR_ROOT = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) + USH_IODA = os.path.join(DIR_ROOT, "ush", "ioda", "bufr2ioda") + + # Create output directory if it doesn't exist + os.makedirs(COM_OBS, exist_ok=True) + + # Load configuration + config = { + 'RUN': RUN, + 'current_cycle': current_cycle, + 'DMPDIR': DMPDIR, + 'COM_OBS': COM_OBS + } + + # Specify observation types to be processed by a script + BUFR_py = ["satwind_amv_goes"] + + for obtype in BUFR_py: + logger.info(f"Convert {obtype}...") + json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") + filename = 'bufr2ioda_' + obtype + '.json' + template = os.path.join(config_template_dir, filename) + gen_bufr_json(config, template, json_output_file) + + # Use the converter script for the ob type + bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" + cmd = Executable(bufr2iodapy) + cmd.add_default_arg('-c') + cmd.add_default_arg(json_output_file) + logger.info(f"Executing {cmd}") + cmd() + + # Check if the converter was successful + if os.path.exists(json_output_file): + rm_p(json_output_file) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Convert bufr dump file to ioda format') + parser.add_argument('current_cycle', help='current cycle to process', type=lambda dd: to_datetime(dd)) + parser.add_argument('RUN', type=str, help='dump to process, either gdas or gdas') + parser.add_argument('DMPDIR', type=Path, help='path to bufr dump files') + parser.add_argument('config_template_dir', type=Path, help='path to templates') + parser.add_argument('COM_OBS', type=Path, help='path to output ioda format dump files') + args = parser.parse_args() + bufr2ioda(args.current_cycle, args.RUN, args.DMPDIR, args.config_template_dir, args.COM_OBS) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.sh b/ush/ioda/bufr2ioda/run_bufr2ioda.sh deleted file mode 100755 index de53767cf..000000000 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -#-------------------------------------------------------------------------------------- -# run_bufr2ioda.sh -# This driver script will: -# - generate config files from templates for each BUFR file -# - use bufr2ioda python scripts or executable and produce output IODA files -# usage: -# run_bufr2ioda.sh YYYYMMDDHH $DMPDIR /path/to/templates $COM_OBS -# -#-------------------------------------------------------------------------------------- -if [[ $# -ne 5 ]] ; then - echo "usage:" - echo " $0 YYYYMMDDHH gdas|gfs /path/to/files.bufr_d/ /path/to/templates /path/to/output.ioda/" - exit 1 -fi - -# some of these need exported to be picked up by the python script below -# input parameters -CDATE=${CDATE:-$1} -export DUMP=${RUN:-$2} -export DMPDIR=${DMPDIR:-$3} -config_template_dir=${config_template_dir:-$4} -export COM_OBS=${COM_OBS:-$5} - -# derived parameters -export PDY=$(echo $CDATE | cut -c1-8) -export cyc=$(echo $CDATE | cut -c9-10) - -# get gdasapp root directory -readonly DIR_ROOT=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/../../.." && pwd -P) -BUFR2IODA=$DIR_ROOT/build/bin/bufr2ioda.x -USH_IODA=$DIR_ROOT/ush/ioda/bufr2ioda -BUFRYAMLGEN=$USH_IODA/gen_bufr2ioda_yaml.py -BUFRJSONGEN=$USH_IODA/gen_bufr2ioda_json.py - -# create output directory if it doesn't exist -mkdir -p $COM_OBS -if [ $? -ne 0 ]; then - echo "cannot make $COM_OBS" - exit 1 -fi - -# add to pythonpath the necessary libraries -PYIODALIB=`echo $DIR_ROOT/build/lib/python3.?` -export PYTHONPATH=${PYIODALIB}:${PYTHONPATH} - -#--------------------------------------------------------- -# for some BUFR files, we will use python scripts -# and for others, bufr2ioda.x - -#--------------------------- -#----- python and json ----- -# first specify what observation types will be processed by a script -BUFR_py="satwind_amv_goes" - -for obtype in $BUFR_py; do - # this loop assumes that there is a python script and template with the same name - echo "Processing ${obtype}..." - - # first generate a JSON from the template - ${BUFRJSONGEN} -t ${config_template_dir}/bufr2ioda_${obtype}.json -o ${COM_OBS}/${obtype}_${PDY}${cyc}.json - - # now use the converter script for the ob type - python3 $USH_IODA/bufr2ioda_${obtype}.py -c ${COM_OBS}/${obtype}_${PDY}${cyc}.json - - # check if converter was successful - if [ $? == 0 ]; then - # remove JSON file - rm -rf ${COM_OBS}/${obtype}_${PDY}${cyc}.json - else - # warn and keep the JSON file - echo "Problem running converter script for ${obtype}" - fi -done - -#---------------------------- -#---- bufr2ioda and yaml ---- -# need to ask Emily which will be YAML based -BUFR_yaml=""