Add I/O and obsop for geostationary SST (resolved #133) (#136)

* Add config files for ESSIC server

1. Add config files for ESSIC server
2. Add utility `gen_config_template.sh` for generate
   templates of config files.

* Fix bugs about allocated arrays and loop index

1. Allocate missing vars ifdef DYNAMIC
2. Correct loop index for area_t calculation
3. Fix calculation of lat/lon of T-cells from supergrids x,y
4. Increase character length of file and vars names for flexibility

* Add module to read ACSPO SST retrievals

1. Add module to process ACSPO SST retrievals
2. Add support module for I/O of NetCDF files.

* Add downloading support to all geo sats.

1. Rename get_l2p_ghrsst_star.py -> get_l2p_ghrsst_geostationary.py
   now it supports all geostaionary satelites
2. Upate the CI get_data test

* Update read_* module for reading SST from geo sats

Add support to read SST from geo sats other than AHI/ABI.

* update CMake

* Add SST/SSS in read_diag

1. Include SST/SSS into read_diag
2. let sst read from read_geostationary,f90 has units in degC,
   consistent with model sst from read_diag.
3. Revise kmt calculation

.github/workflows/build_gfortran.yml

@@ -20,6 +20,7 @@ jobs:
 
     - name: "install MPI & NetCDF"
       run: |
+           sudo apt-get update
            sudo apt install mpich   
            sudo apt install libnetcdf-dev libnetcdff-dev netcdf-bin
 
@@ -54,6 +55,7 @@ jobs:
 
     - name: "install MPI & NetCDF"
       run: |
+           sudo apt-get update
            sudo apt install mpich   
            sudo apt install libnetcdf-dev libnetcdff-dev netcdf-bin
 

.github/workflows/get_data.yml

@@ -32,11 +32,11 @@ jobs:
            rm -rf test_l2_sss
            ls -al
 
-    - name: "check: get_l2p_ghrsst_star.py"
+    - name: "check: get_l2p_ghrsst_geostationary.py"
       run: |
            cd ${{github.workspace}}/utils/obs_proc/level-2
            pwd
-           PYTHONPATH=../../pycommon ./get_l2p_ghrsst_star.py --start_date 20220110 --end_date 20220111 --sis ABI_G17 --outdir ./ --topdir_name test_l2p_sst --skip_remote_check  --get_nav_file --verbose --log
+           PYTHONPATH=../../pycommon ./get_l2p_ghrsst_geostationary.py --start_date 20220110 --end_date 20220111 --sis ABI_G17 --outdir ./ --topdir_name test_l2p_sst --skip_remote_check  --get_nav_file --verbose --log
            ls -al 
            ls -al ./test_l2p_sst/ABI_G17/2022/202201/20220111/
            ls -al ./test_l2p_sst/ABI_G17/nav

CMakeLists.txt

@@ -28,8 +28,8 @@ if (SOLO_BUILD) # build ocean-letkf with local libs. Most users should use this
 
     # compiler-dependent flags   
     if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
-        set(CMAKE_Fortran_FLAGS "-O2 -ffree-line-length-none -frecord-marker=4 -finit-local-zero -fbacktrace -g")
-        set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -ffree-line-length-none -frecord-marker=4 -finit-local-zero -fbacktrace -fcheck=bounds")
+        set(CMAKE_Fortran_FLAGS "-O2 -ffree-line-length-none -frecord-marker=4 -finit-local-zero -fbacktrace")
+        set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -ffree-line-length-none -frecord-marker=4 -finit-local-zero -fbacktrace -fcheck=mem,pointer,bounds -fno-realloc-lhs")
         set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ffree-line-length-none -frecord-marker=4 -finit-local-zero")
         ## additional flags for gfortran version >= 10.0
         if (${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) 

build/lnkcommon_obsop.sh

@@ -60,6 +60,16 @@ cp $OBSDIR/gsw_pot_to_insitu.f90 ./
 cp $OBSDIR/read_ice_txt.f90 ./
 cp $OBSDIR/obsop_icefrac.f90 ./
 
+IODIR=$root/support/io
+if [ "${model}" = "mom6" ]; then
+   cp $OBSDIR/read_geostationary.f90 ./
+   cp $OBSDIR/obsop_sst_geostationary.f90 ./
+   cp $OBSDIR/w3movdat_full.f ./
+   cp $IODIR/m_ncio.f90 ./
+   cp $IODIR/*.f90.inc ./
+fi
+
+
 if [ "${model}" = "hycom" ]; then
    cp $OBSDIR/read_bufr_hycom.f90 ./
    cp $OBSDIR/obsop_bufr_tprof_hycom.f90 ./

build/make_obsop.mom6_dynamic.sh

@@ -60,6 +60,15 @@ rm -f $BDIR/*.dat
 
 sh $CDIR/lnkcommon_obsop.sh $model $CDIR/../
 
+#--
+# F90GIO lib
+#--
+$F90 $OMP $F90_OPT $F90_DEBUG $F90_FPP $F90_OBJECT_FLAG $NETCDF_INC m_ncio.f90
+$F90 $OMP $F90_OPT $F90_DEBUG $F90_FPP $F90_OBJECT_FLAG $NETCDF_INC w3movdat_full.f
+
+
+
+
 $F90 $OMP $F90_OPT $INLINE $F90_OBJECT_FLAG SFMT.f90
 $F90 $OMP $F90_OPT $INLINE $F90_OBJECT_FLAG common.f90
 $F90 $OMP $F90_OPT $F90_DEBUG $F90_FPP $F90_OBJECT_FLAG params_model.f90
@@ -74,6 +83,9 @@ $F90 $OMP $F90_OPT $F90_OBJECT_FLAG compute_profile_error.f90
 $F90 $OMP $F90_OPT $F90_OBJECT_FLAG $NETCDF_INC read_argo.f90
 $F90 $OMP $F90_OPT $F90_OBJECT_FLAG $NETCDF_INC read_avhrr_pathfinder.f90
 $F90 $OMP $F90_OPT $F90_OBJECT_FLAG $NETCDF_INC read_aviso_adt.f90
+
+$F90 $OMP $F90_OPT $F90_OBJECT_FLAG $NETCDF_INC read_geostationary.f90
+
 $F90 $OMP $F90_OPT $F90_DEBUG $F90_FPP $F90_OBJECT_FLAG input_nml_${model}.f90
 
 #--
@@ -86,6 +98,8 @@ $F90 $OMP $F90_OPT obsop_sprof.f90 -o ${PGM}.sprof.x *.o $NETCDF_LIB
 $F90 $OMP $F90_OPT obsop_adt.f90   -o ${PGM}.adt.x   *.o $NETCDF_LIB
 $F90 $OMP $F90_OPT obsop_sst.f90   -o ${PGM}.sst.x   *.o $NETCDF_LIB
 
+$F90 $OMP $F90_OPT $F90_FPP obsop_sst_geostationary.f90   -o ${PGM}.sst_geostationary.x   *.o $NETCDF_LIB
+
 rm -f *.mod
 rm -f *.o
 #--

config/ESSIC.bufr.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# BUFR lib is only used for model=hycom
+
+BUILD_BUFRLIB="yes"  # "yes" or "no"
+
+# variables to set if BUILD_BUFRLIB = "no"
+BUFR_DIR=/Users/cda14/Desktop/bufr_util/src/bufr_10.2.3
+BUFR_LIB="-L$BUFR_DIR/ -lbufr"
+
+# variables to set if BUILD_BUFRLIB = "yes"
+CC=gcc
+
+#--------------------------------------------------------------------
+# build BUFRLIB under support:
+# uses F90s and F90_OPT in MACHINE.fortran.sh
+# (generally you should not need to modify below if you use 
+#  Intel or GNU compiler. Otherwise you need to modify below )
+#
+if [ "$BUILD_BUFRLIB" = "yes" ]; then
+   BUILD_DIR=$PWD
+   SUPPORT_DIR="${BUILD_DIR}/../support"
+   BUFR_DIR_NAME="bufr_10.2.3_LE"
+   BUFR_PATH="$SUPPORT_DIR/$BUFR_DIR_NAME"
+   BUFR_URL="https://github.com/cd10kfsu/${BUFR_DIR_NAME}.git"
+   if [ -e $BUFR_PATH/libbufr.a ]; then
+       echo "bufrlib found at: ${SUPPORT_DIR}/libbufr.a"
+   else
+       echo "bufrlib not found at: $SUPPORT_DIR"
+       if [ -d $BUFR_PATH ]; then
+          echo "bufrlib directory found at ($BUFR_PATH) but no libbufs.a: removing this directory"
+          rm -rf $BUFR_PATH
+       fi
+       # clone bufrlibs under support/
+       cd $SUPPORT_DIR && git clone $BUFR_URL
+       cd $BUFR_DIR_NAME
+
+       # setting fortran compiler flags
+       FC=${F90s}
+       if [ "$($FC --version|grep -i 'intel\|ifort'|wc -l)0" -gt 0 ]; then
+          # Intel compiler
+          FC_BUFR_OPT="$F90_OPT -DUNDERSCORE"
+       elif [ "$($FC --version|grep -i 'gnu\|gcc\|gfortran'|wc -l)0" -gt 0 ]; then
+          # GNU compiler
+          FC_BUFR_OPT="$F90_OPT -DUNDERSCORE -fno-second-underscore"
+       else
+          echo "[error] unrecognized Fortran compiler. Exit..."
+          exit 1
+       fi
+
+       # setting C compiler flags
+       if [ "$($CC --version|grep -i 'intel\|icc'|wc -l)0" -gt 0 ]; then
+          # Intel compiler
+          CC_BUFR_OPT="-DUNDERSCORE"
+       elif [ "$($CC --version|grep -i 'gnu\|gcc'|wc -l)0" -gt 0 ]; then
+          # GNU compiler
+          CC_BUFR_OPT="-std=c90 -DUNDERSCORE"
+       else
+          echo "[error] unrecognized C compiler. Exit..."
+          exit 1
+       fi
+
+       # print config
+       echo "======================================="
+       echo "config for building BUFRLIBS"
+       echo "CC=$CC"
+       echo "CC_BUFR_OPT=$CC_BUFR_OPT"
+       echo "FC=$FC"
+       echo "FC_BUFR_OPT=$FC_BUFR_OPT"
+
+       # build libs
+       $CC $CC_BUFR_OPT -c *.c
+       $FC $FC_BUFR_OPT -c *.f
+       ar crv libbufr.a *.o
+   fi
+
+   cd $BUILD_DIR
+
+   BUFR_DIR="$BUFR_PATH"
+   BUFR_LIB="-L$BUFR_DIR/ -lbufr" 
+
+   echo "BUFR_DIR=$BUFR_DIR"
+   echo "BUFR_LIB=$BUFR_LIB"
+
+fi
+

config/ESSIC.fortran.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+F90=mpif90
+F90s="gfortran"   #used to compile BUFRLIBS
+#F90_OPT='-O2 -ffree-form -ffree-line-length-none'
+#STEVE: -mcmodel=medium needed for large model grid sizes (e.g. higher than 1 degree resolution of om3_core3)
+# explanation of -mcmodel=medium and -shared-intel: http://software.intel.com/en-us/forums/showthread.php?t=43717#18089
+#F90_OPT='-O2 -ffree-line-length-none -frecord-marker=4 -finit-local-zero' #CDA: for gfortran version < 10
+#F90_OPT='-O2 -ffree-line-length-none -frecord-marker=4 -finit-local-zero -fbacktrace -fcheck=bounds' #CDA: for gfortran version < 10
+F90_OPT='-O2 -ffree-line-length-none -frecord-marker=4 -finit-local-zero -fbacktrace -fcheck=mem,pointer,bounds -fno-realloc-lhs' #CDA: for gfortran version < 10
+F90_OPT="$F90_OPT -fallow-argument-mismatch" #CDA: for gfortran version >= 10
+F90_INLINE=
+F90_DEBUG=
+F90_IEEE= #'-Kieee' #'-fltconsistency'
+F90_OBJECT_FLAG='-c' #STEVE: for some reason, mpxlf doesn't use -c, but rather -g
+BLAS=0
+
+F90_FPP='-cpp'  # for gfortran
+

config/ESSIC.mpi.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+MPI_DIR=
+MPI_INC=
+MPI_LIB= #-lmpi

config/ESSIC.netcdf.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+#NETCDF_DIR=/opt/local
+#NETCDFF_DIR=/opt/local
+#NETCDF_LIB="-L/opt/local/lib -lnetcdff -L/opt/local/lib -Wl,-headerpad_max_install_names -Wl,-syslibroot,/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk -arch x86_64 -lnetcdf -lnetcdf -lm"
+#NETCDF_INC="-I/opt/local/include"
+
+#NETCDF_DIR=/opt/local
+#NETCDFF_DIR=/opt/local
+NETCDF_LIB=`nf-config --flibs`
+NETCDF_INC=`nf-config --fflags`
+echo "[$0] NETCDF_LIB=$NETCDF_LIB"
+echo "[$0] NETCDF_INC=$NETCDF_INC"

config/LOCAL_GFORTRAN.fortran.sh

@@ -6,8 +6,9 @@ F90s="gfortran"   #used to compile BUFRLIBS
 #STEVE: -mcmodel=medium needed for large model grid sizes (e.g. higher than 1 degree resolution of om3_core3)
 # explanation of -mcmodel=medium and -shared-intel: http://software.intel.com/en-us/forums/showthread.php?t=43717#18089
 #F90_OPT='-O2 -ffree-line-length-none -frecord-marker=4 -finit-local-zero' #CDA: for gfortran version < 10
-F90_OPT='-O2 -ffree-line-length-none -frecord-marker=4 -finit-local-zero -fbacktrace -fcheck=mem,pointer,bounds -fno-realloc-lhs' #CDA: for gfortran version < 10
+F90_OPT="-O2 -ffree-line-length-none -frecord-marker=4 -finit-local-zero -fbacktrace -fcheck=mem,pointer,bounds -fno-realloc-lhs" #CDA: for gfortran version < 10
 F90_OPT="$F90_OPT -fallow-argument-mismatch" #CDA: for gfortran version >= 10
+#F90_OPT="$F90_OPT -fprofile-arcs -ftest-coverage"
 F90_INLINE=
 F90_DEBUG=
 F90_IEEE= #'-Kieee' #'-fltconsistency'

config/gen_config_template.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#
+# generate template for machine config files
+#
+
+if [ $# -ne 1 ]; then
+   echo "$0: Generate template for machine config files"
+   echo "usage: "
+   echo "  $0 CONFIG_FILE_PREFIX"
+   exit 1
+fi
+
+machine=$1
+
+if [ -e flist_LOCAL_GFORTRAN ]; then
+    rm -f flist_LOCAL_GFORTRAN
+fi
+
+ls LOCAL_GFORTRAN.*.sh > flist_LOCAL_GFORTRAN
+while read fname_template; do
+    fend=$(echo $fname_template|cut -d "." -f2-)
+    fname_out="${machine}.${fend}"
+    echo "$fname_out <---------- $fname_template"
+    cp $fname_template $fname_out
+done<flist_LOCAL_GFORTRAN
+
+if [ -e flist_LOCAL_GFORTRAN ]; then
+    rm -f flist_LOCAL_GFORTRAN
+fi

src/CMakeLists.txt

@@ -26,6 +26,10 @@ set (SRCS
   common_all/kdtree.f90
   common_all/netlib.f
   common_all/netlibblas.f
+# from ../support/io/
+  ../support/io/m_ncio.f90
+  ../support/io/nc_rdatt.f90.inc
+  ../support/io/nc_rdvar.f90.inc
 #--
 # from letkf/
   letkf/params_letkf.f90
@@ -45,10 +49,12 @@ set (SRCS
   model_specific/${MODEL}/common_debug_${MODEL}.f90
 #--
 # from obs/
+  obs/w3movdat_full.f
   obs/params_obs.f90
   obs/compute_profile_error.f90
   obs/read_argo.f90
   obs/read_avhrr_pathfinder.f90
+  obs/read_geostationary.f90
   obs/read_aviso_adt.f90
   obs/read_ice_txt.f90
   obs/gsw_pot_to_insitu.f90
@@ -78,12 +84,15 @@ install(TARGETS OCN.letkf_${MODEL}.x)
 set(operators
   adt
   sst
+  sst_geostationary
   tprof
   sprof
   icefrac
   )
 foreach(operator ${operators})
   add_executable(OCN.obsOp_${MODEL}.${operator}.x obs/obsop_${operator}.f90)
-  target_link_libraries(OCN.obsOp_${MODEL}.${operator}.x PRIVATE OCN.letkf)
+  target_link_libraries(OCN.obsOp_${MODEL}.${operator}.x PRIVATE OCN.letkf NetCDF::NetCDF_Fortran)
+  set_target_properties (OCN.obsOp_${MODEL}.${operator}.x  PROPERTIES Fortran_PREPROCESS ON) 
+  target_compile_definitions(OCN.obsOp_${MODEL}.${operator}.x PRIVATE DYNAMIC)
   install(TARGETS OCN.obsOp_${MODEL}.${operator}.x)
 endforeach()