From 8f58a20cd39f820424d9b80dc126e949f5eafcfb Mon Sep 17 00:00:00 2001 From: Ed Safford <62339196+EdwardSafford-NOAA@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:50:43 -0500 Subject: [PATCH] Put RadMon regional html in repository and add scripting for regional web site generation (#154) * Ref #153 Add regional html files and scripting changes to support regional web site generation. * Ref #153 File cleanup. --- .../image_gen/html/Install_html.sh | 39 +- .../image_gen/html/bcoef.html.glb | 275 --- .../image_gen/html/bcor.html.glb | 283 ---- .../image_gen/html/index.html.rgn | 316 ++++ .../image_gen/html/install_glb.sh | 34 +- .../image_gen/html/install_rgn.sh | 406 +++++ .../image_gen/html/mk_intro.sh | 13 +- .../{plot_angle.html => plot_angle.html.glb} | 4 - .../image_gen/html/plot_angle.html.rgn | 1486 ++++++++++++++++ .../{plot_bcoef.html => plot_bcoef.html.glb} | 4 - .../image_gen/html/plot_bcoef.html.rgn | 1066 ++++++++++++ ...lot_summary.html => plot_summary.html.glb} | 4 - .../image_gen/html/plot_summary.html.rgn | 1149 +++++++++++++ .../{plot_time.html => plot_time.html.glb} | 4 - .../image_gen/html/plot_time.html.rgn | 1502 +++++++++++++++++ .../image_gen/html/summary.html.glb | 100 -- .../image_gen/html/time.html.glb | 314 ---- 17 files changed, 5971 insertions(+), 1028 deletions(-) delete mode 100755 src/Radiance_Monitor/image_gen/html/bcoef.html.glb delete mode 100755 src/Radiance_Monitor/image_gen/html/bcor.html.glb create mode 100644 src/Radiance_Monitor/image_gen/html/index.html.rgn create mode 100755 src/Radiance_Monitor/image_gen/html/install_rgn.sh rename src/Radiance_Monitor/image_gen/html/{plot_angle.html => plot_angle.html.glb} (99%) create mode 100644 src/Radiance_Monitor/image_gen/html/plot_angle.html.rgn rename src/Radiance_Monitor/image_gen/html/{plot_bcoef.html => plot_bcoef.html.glb} (97%) create mode 100644 src/Radiance_Monitor/image_gen/html/plot_bcoef.html.rgn rename src/Radiance_Monitor/image_gen/html/{plot_summary.html => plot_summary.html.glb} (97%) create mode 100644 src/Radiance_Monitor/image_gen/html/plot_summary.html.rgn rename src/Radiance_Monitor/image_gen/html/{plot_time.html => plot_time.html.glb} (97%) create mode 100644 src/Radiance_Monitor/image_gen/html/plot_time.html.rgn delete mode 100755 src/Radiance_Monitor/image_gen/html/summary.html.glb delete mode 100755 src/Radiance_Monitor/image_gen/html/time.html.glb diff --git a/src/Radiance_Monitor/image_gen/html/Install_html.sh b/src/Radiance_Monitor/image_gen/html/Install_html.sh index 07237b2a..e7525611 100755 --- a/src/Radiance_Monitor/image_gen/html/Install_html.sh +++ b/src/Radiance_Monitor/image_gen/html/Install_html.sh @@ -14,12 +14,16 @@ # usage #-------------------------------------------------------------------- function usage { - echo "Usage: Install_html.sh suffix [-t|--tank]" + echo "Usage: Install_html.sh suffix -t|--tank [-r|--run -a|--area]" echo " Suffix is data source identifier that matches data in " echo " the $TANKDIR/stats directory." - echo " -t | --tank parent directory to the radmon data file location. This" - echo " will be extended by \$RADMON_SUFFIX, \$RUN, and \$PDATE to locate the" - echo " extracted radmon data." + echo " -t | --tank parent directory to the radmon data file location. This" + echo " will be extended by /$RADMON_SUFFIX, /$RUN, and /$PDATE to locate the" + echo " extracted radmon data. Note if using internal RadMon format for data" + echo " then stop at '/nbns' and do not include '/stats' in tank." + echo " -r | --run Run value for data source. Default is 'gdas'." + echo " -a | --area Area value for data source. Valid values are 'glb' or 'rgn'" + echo " indicating global or regional data source. The default is 'glb'." echo "" } @@ -28,7 +32,7 @@ echo "" nargs=$# -if [[ $nargs -lt 1 || $nargs -gt 3 ]]; then +if [[ $nargs -lt 1 || $nargs -gt 7 ]]; then usage exit 2 fi @@ -36,9 +40,9 @@ fi #----------------------------------------------------------- # Set default values and process command line arguments. # -#run=gdas +run=gdas tank="" -area="" +area=glb while [[ $# -ge 1 ]]; do key="$1" @@ -48,6 +52,14 @@ while [[ $# -ge 1 ]]; do tank="$2" shift # past argument ;; + -r|--run) + run="$2" + shift # past argument + ;; + -a|--area) + area="$2" + shift # past argument + ;; *) #any unspecified key is RADMON_SUFFIX export RADMON_SUFFIX=$key @@ -59,6 +71,7 @@ done this_file=`basename $0` this_dir=`dirname $0` +export RAD_AREA=${area} top_parm=${this_dir}/../../parm radmon_config=${radmon_config:-${top_parm}/RadMon_config} @@ -90,13 +103,21 @@ fi if [[ ${#tank} -le 0 ]]; then tank=${TANKDIR} fi + export R_TANKDIR=${tank} -echo R_TANKDIR = $R_TANKDIR +export RUN=${run} +if [[ ${RAD_AREA} == "glb" ]]; then + ${RADMON_IMAGE_GEN}/html/install_glb.sh -${RADMON_IMAGE_GEN}/html/install_glb.sh +elif [[ ${RAD_AREA} == "rgn" ]]; then + ${RADMON_IMAGE_GEN}/html/install_rgn.sh +else + echo "area value ${RAD_AREA} is not recognized. Only valid values are 'glb' and 'rgn'." +fi +echo echo "END Install_html.sh" exit diff --git a/src/Radiance_Monitor/image_gen/html/bcoef.html.glb b/src/Radiance_Monitor/image_gen/html/bcoef.html.glb deleted file mode 100755 index b0f920d8..00000000 --- a/src/Radiance_Monitor/image_gen/html/bcoef.html.glb +++ /dev/null @@ -1,275 +0,0 @@ - - - - - - Radiance Monitoring - - - - - - - - - - -

Bias correction coefficients

-
- - - - - - - - - - - - - - -
Select Platform:
-

-

Bias correction term:
-

-

Channel groupings:
- -
- Click button to...
-
-
- - -
- - - diff --git a/src/Radiance_Monitor/image_gen/html/bcor.html.glb b/src/Radiance_Monitor/image_gen/html/bcor.html.glb deleted file mode 100755 index 97996236..00000000 --- a/src/Radiance_Monitor/image_gen/html/bcor.html.glb +++ /dev/null @@ -1,283 +0,0 @@ - - - - - Radiance Monitoring - - - - - - - - - - -

Bias correction terms

- -
- - - - - - - - - - - - - - - - - - -
Select Platform:
- -
Region:
- -
Term:
- -
Channel groupings:
- -
- Click button to...
-
-
- -
- - diff --git a/src/Radiance_Monitor/image_gen/html/index.html.rgn b/src/Radiance_Monitor/image_gen/html/index.html.rgn new file mode 100644 index 00000000..ad985cec --- /dev/null +++ b/src/Radiance_Monitor/image_gen/html/index.html.rgn @@ -0,0 +1,316 @@ + + + + + + INSERT_SUFFIX -- Experimental Radiance Monitoring plots + + + + + + + + + + +
+ + +
+ + +
+ +
+ +
+ + + +
+ + + diff --git a/src/Radiance_Monitor/image_gen/html/install_glb.sh b/src/Radiance_Monitor/image_gen/html/install_glb.sh index 77942825..cdd20f9a 100755 --- a/src/Radiance_Monitor/image_gen/html/install_glb.sh +++ b/src/Radiance_Monitor/image_gen/html/install_glb.sh @@ -74,7 +74,9 @@ cd $workdir # backwards. If not found stop after 5 days and exit. # -RUN=gdas +if [[ $RUN == "" ]]; then + RUN=gdas +fi PDATE=`${MON_USH}/find_last_cycle.sh --net ${RADMON_SUFFIX} \ --run ${RUN} --mon radmon --tank ${R_TANKDIR}` @@ -88,16 +90,12 @@ limit=`$NDATE -120 $PDATE` # 5 days data_found=0 while [[ data_found -eq 0 && $PDATE -ge $limit ]]; do - PDY=`echo $PDATE|cut -c1-8` - CYC=`echo $PDATE|cut -c9-10` ieee_src=`${MON_USH}/get_stats_path.sh --run $RUN --pdate ${PDATE} \ --net ${RADMON_SUFFIX} --tank ${R_TANKDIR} --mon radmon` - echo "ieee_src = $ieee_src" if [[ -d ${ieee_src} ]]; then using_tar=0 - echo " ieee_src is GO " if [[ -e ${ieee_src}/radmon_angle.tar || -e ${ieee_src}/radmon_angle.tar.gz ]]; then if [[ -e ${ieee_src}/radmon_angle.tar.gz ]]; then @@ -116,7 +114,7 @@ while [[ data_found -eq 0 && $PDATE -ge $limit ]]; do fi fi else - echo "ieee_src is NOGO" + echo "ieee_src ${ieee_src} is NOGO" fi if [[ data_found -eq 0 ]]; then @@ -150,15 +148,14 @@ for test in ${test_list}; do tmp=`echo "$this_file" | cut -d. -f2` fi - SATYPE_LIST="$SATYPE_LIST $tmp" + satypes="${satypes} ${tmp}" done -export SATYPE=$SATYPE_LIST - -if [[ ${#SATYPE} -le 0 ]]; then +if [[ ${#satypes} -le 0 ]]; then echo "SATYPE list is zero length, unable to complete html installation" exit fi +export SATYPE=${satypes} #-------------------------------------------------------------- @@ -169,8 +166,6 @@ UNSORTED_LIST=./unsorted.txt export SORTED_LIST=./sorted.txt >$SORTED_LIST -echo "SATYPE = $SATYPE" - for satype in $SATYPE; do ins=${satype%_*} tmp="${ins}_" @@ -221,7 +216,6 @@ done # Sort the list by Satellite # `sort -d -u $UNSORTED_LIST > $SORTED_LIST` -echo SORTED_LIST = $SORTED_LIST #-------------------------------------------------------------- # Read the sorted list and create the platform table @@ -250,7 +244,6 @@ while read line; do echo $hline >> $PLATFORM_TBL done < "$SORTED_LIST" - #-------------------------------------------------------------- # Edit the html files to add the platform table to each. # @@ -258,8 +251,8 @@ mod_html_files="plot_summary.html plot_time.html plot_angle.html plot_bcoef.html for html_file in $mod_html_files; do echo "processing ${html_file}" - $NCP ${RADMON_IMAGE_GEN}/html/${html_file} . - + ${NCP} ${RADMON_IMAGE_GEN}/html/${html_file}.glb ${html_file} + tmp_html=./tmp_${html_file} rm -f ${tmp_html} @@ -272,7 +265,7 @@ for html_file in $mod_html_files; do # copy the $file from 'END_TABLE_INSERT' comment to end sed -n '/END_TABLE_INSERT/,$p' ${html_file} >> ${tmp_html} - rm $html_file + rm ${html_file} # switch all 'INSERT_SUFFIX' tags to the actual suffix # and route output to $html_file and we're done. @@ -295,18 +288,19 @@ if [[ $do_cmp == 1 ]]; then # for html_file in $comp_html_files; do - echo "processing ${html_file}" tmp_html=./tmp_${html_file} rm -f ${tmp_html} #---------------------------------------------------------------------------- # remove the OPTIONAL_COMPARE lines which uncomments the comparison check box + # sed '/OPTIONAL_COMPARE/d' ./${html_file} > ${tmp_html} mv -f ${tmp_html} ${html_file} #--------------------------------------------------------------- # if we're using a source other than GDAS make that change here + # if [[ $cmp_src != "GDAS" ]]; then cmp_sc_line=" var compSrc = \"${cmp_src}\";" cmp_nm_line=" var compName = \"${cmp_src}\";" @@ -331,7 +325,7 @@ rm mk_intro.sh #-------------------------------------------------------------- # Copy the index.html file and change INSERT_SUFFIX to actual suffix. -index_file="index.html.$RAD_AREA" +index_file="index.html.glb" tmp_index="tmp.index.html" new_index="index.html" @@ -351,8 +345,6 @@ fi rm ./${index_file} -echo workdir = $workdir - #-------------------------------------------------------------- # Make starting directory in $imgndir and copy over html, # misc, and thumb images. diff --git a/src/Radiance_Monitor/image_gen/html/install_rgn.sh b/src/Radiance_Monitor/image_gen/html/install_rgn.sh new file mode 100755 index 00000000..385af735 --- /dev/null +++ b/src/Radiance_Monitor/image_gen/html/install_rgn.sh @@ -0,0 +1,406 @@ +#!/bin/bash + +#-------------------------------------------------------------------- +#-------------------------------------------------------------------- +# install_rgn.sh +# +# Build the html files necessary for a radiance monitor web site +# and store it in the plot location (~/nbns/imgn/$RADMON_SUFFIX). +#-------------------------------------------------------------------- +#-------------------------------------------------------------------- + +echo "BEGIN install_rgn.sh" +echo "" + +do_cmp=0 +cmp_src="" + + +# This section intentionally left commented out. The comparison +# option is not yet implemented for regional sources. +#-------------------------------------------------------------- +# Allow user to enable comparison plots +# +#echo "Do you wish to enable data plots to include comparison to" +#echo " operational GDAS data, or another data source?" +#echo "" +#echo -n " Enter YES to enable comparison plots, any other input to disable. > " +#read text +#short=`echo $text | cut -c1` +# +#if [[ $short = "Y" || $short = "y" ]]; then +# do_cmp=1 +# cmp_src="GDAS" +# +# echo "Please specify the suffix of your comparison data source," +# echo " or just hit the return key to use the operational GDAS as " +# echo " the comparison source" +# echo "" +# echo -n " > " +# read text +# +# if [[ ${#text} -gt 0 ]]; then +# cmp_src=${text} +# fi +# +# echo "${cmp_src} will be used as the comparison source." +#fi + +#-------------------------------------------------------------- +# Create a temporary working directory. +# +workdir=$MON_STMP/${RADMON_SUFFIX}_html +if [[ -e $workdir ]]; then + rm -rf $workdir +fi +mkdir $workdir +cd $workdir + + +#------------------------------------------------------------- +# Assemble the SATYPE list from available data files in +# $TANKverf using angle.* files. +# +# Find the first date with data. Start at today and work +# backwards. If not found stop after 5 days and exit. +# + +#if [[ $RUN == "" ]]; then +# RUN=gdas +#fi + +PDATE=`${MON_USH}/rgn_find_cycle.pl --dir ${TANKverf} --mon radmon` +echo PDATE=$PDATE + +limit=`$NDATE -6 $PDATE` # 6 cycles + +#----------------------------------------------------------- +# Build test_list which will contain all data files for +# one cycle in $PDATE. We start with $PDATE and back up +# for 6 hours. This intentionally introduces duplicates +# but catches any sat/instrument sources that didn't have +# any data for a given cycle. +# +tankdir="${R_TANKDIR}/${RADMON_SUFFIX}" +data_found=0 +test_list="" + +while [[ $PDATE -ge $limit ]]; do + pdy=`echo $PDATE|cut -c1-8` + + pdy=`echo ${PDATE}|cut -c1-8` + ieee_src=${tankdir}/radmon.${pdy} + + if [[ -d ${ieee_src} ]]; then + using_tar=0 + + if [[ -e ${ieee_src}/radmon_angle.tar || -e ${ieee_src}/radmon_angle.tar.gz ]]; then + if [[ -e ${ieee_src}/radmon_angle.tar.gz ]]; then + found=`tar -tf ${ieee_src}/radmon_angle.tar.gz` + else + found=`tar -tf ${ieee_src}/radmon_angle.tar` + fi + + data_found=1 + + else + test=`ls ${ieee_src}/angle.*${PDATE}*.ieee_d* | wc -l` + if [[ $test -gt 0 ]]; then + found=`ls ${ieee_src}/angle.*${PDATE}*.ieee_d*` + data_found=1 + fi + fi + test_list="${test_list} ${found}" + else + echo "ieee_src is NOGO, skipping $PDATE" + fi + + PDATE=`$NDATE -1 $PDATE` + +done + +if [[ $data_found -eq 0 ]]; then + echo Unable to locate any data files in the past 5 days for ${RADMON_SUFFIX} + echo in ${TANKverf}/angle. + exit +fi + +#----------------------------------------------------------- +# Screen out all non-angle, analysis (*_anl*) and control +# (*.ctl) files from $test_list. Reduce file names to just +# 'instrument_sat'. +# +for test in ${test_list}; do + this_file=`basename $test` + + test_anl=`echo $this_file | grep "_anl"` + test_ctl=`echo $this_file | grep "ctl"` + if [[ $test_anl != "" || $test_ctl != "" ]]; then + continue + fi + + tmp=`echo "$this_file" | cut -d. -f1` + if [[ $tmp == "angle" ]]; then + tmp=`echo "$this_file" | cut -d. -f2` + fi + + satype_list="${satype_list} ${tmp}" +done + +# Eliminate duplicates and sort $satype_list +array=(${satype_list}) +export SATYPE=$(printf "%s\n" "${array[@]}" | sort | uniq) + +echo SATYPE: $SATYPE + +if [[ ${#SATYPE} -le 0 ]]; then + echo "SATYPE list is zero length, unable to complete html installation" + exit +fi + +#-------------------------------------------------------------- +# Use the SATYPE list to construct the platform table. +# +UNSORTED_LIST=./unsorted.txt +>${UNSORTED_LIST} +export SORTED_LIST=./sorted.txt +>${SORTED_LIST} + +for satype in ${SATYPE}; do + ins=${satype%_*} + tmp="${ins}_" + sat=${satype#${tmp}} + + sat_num=`echo ${sat} | tr -d '[[:alpha:]]'` + + #----------------------------------------------------------------- + # If sat_num has a length > 0 then we have a goes or noaa series + # satellite. Otherwise, convert sat to upper case and stop there. + # + if [[ ${#sat_num} -gt 0 ]]; then + char=`expr substr ${sat} 1 1` + if [[ $char == "g" ]]; then + sat="GOES-${sat_num}" + elif [[ $char == "n" ]]; then + sat="NOAA-${sat_num}" + + else + sat=`echo ${sat} | tr 'a-z' 'A-Z'` + fi + else + sat=`echo ${sat} | tr 'a-z' 'A-Z'` + fi + + #----------------------------------------------------------------- + # Certain instruments require specific formatting. + # + amsu_test=`expr match ${ins} "amsu"` + hirs_test=`expr match ${ins} "hirs"` + ins_num=`echo ${ins} | tr -d '[[:alpha:]]'` + + if [[ $amsu_test -gt 0 ]]; then + char=`expr substr ${ins} 5 5` + char=`echo ${char} | tr -s 'a-z' 'A-Z'` + ins="AMSU-${char}" + elif [[ $hirs_test -gt 0 ]]; then + ins="HIRS/${ins_num}" + else + ins=`echo ${ins} | tr -s 'a-z' 'A-Z'` + fi + + echo ${sat} ${ins} ${satype} >> ${UNSORTED_LIST} + +done + +echo UNSORTED_LIST: ${UNSORTED_LIST} + +#-------------------------------------------------------------- +# Sort the list by Satellite +# +`sort -d -u ${UNSORTED_LIST} > ${SORTED_LIST}` +export SORTED_LIST=${SORTED_LIST} + +#-------------------------------------------------------------- +# Read the sorted list and create the platform table +# +PLATFORM_TBL=./platform.txt +> ${PLATFORM_TBL} + +quote='"' +id=' id="' +extra='">' +end_option='' + +while read line; do + sat=`echo ${line} | gawk '{print $1}'` + ins=`echo ${line} | gawk '{print $2}'` + satype=`echo ${line} | gawk '{print $3}'` + + hline='