diff --git a/driver/runObsMon.sh b/driver/runObsMon.sh index f0f53ab..f271f66 100755 --- a/driver/runObsMon.sh +++ b/driver/runObsMon.sh @@ -106,7 +106,7 @@ logfile="${logdir}/OM_setup.log" if [[ -e ${logfile} ]]; then rm ${logfile}; fi case ${MACHINE_ID} in - hera) + hera|orion|hercules) ${SUB} --account ${ACCOUNT} --ntasks=1 --mem=400M --time=0:05:00 \ -J ${jobname} --partition service -o ${logfile} ${jobfile} ;; @@ -116,7 +116,7 @@ case ${MACHINE_ID} in -v "PYTHONPATH=${PYTHONPATH}, PATH=${PATH}, HOMEobsmon=${HOMEobsmon}, COMOUT=${COMOUT}, \ MODEL=${MODEL}, PDY=${PDY}, cyc=${cyc}, DATAROOT=${DATAROOT}, APRUN_PY=${APRUN_PY}, \ MACHINE_ID=${MACHINE_ID}, ACCOUNT=${ACCOUNT}, JOB_QUEUE=${JOB_QUEUE}, SUB=${SUB}, - OM_LOGS=${OM_LOGS}, YAML_FILE=${YAML_FILE}" \ + OM_LOGS=${OM_LOGS}, YAML_FILE=${YAML_FILE}, CARTOPY_DATA_DIR=${CARTOPY_DATA_DIR}" \ -l select=1:mem=500mb -l walltime=0:05:00 -N ${jobname} ${jobfile} ;; esac diff --git a/jobs/JMON_PLOT_OBS b/jobs/JMON_PLOT_OBS index d2450d2..2ff04d6 100755 --- a/jobs/JMON_PLOT_OBS +++ b/jobs/JMON_PLOT_OBS @@ -31,6 +31,11 @@ if [[ -d ${DATAROOT}/${NET} ]]; then rm -rf ${DATAROOT}/${NET}; fi mkdir -p ${DATA} cd ${DATA} +################################## +# Remove workspace when finished +################################## +KEEPDATA=${KEEPDATA:-"NO"} + ################ # Run exscript ################ @@ -46,8 +51,6 @@ if [[ -e "${pgmout}" ]] ; then fi #################### -# Remove workspace +# Remove workspace -- this will have to be a separate job that runs +# after the OM_plot job is complete. #################### -KEEPDATA=${KEEPDATA:-"NO"} -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} diff --git a/modulefiles/obs-monitor/hera.lua b/modulefiles/obs-monitor/hera.lua new file mode 100644 index 0000000..8d792b5 --- /dev/null +++ b/modulefiles/obs-monitor/hera.lua @@ -0,0 +1,39 @@ +help([[ +Load python virtual environment for obs-monitor +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +conflict(pkgName) + + +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") + +load ("stack-intel/2021.5.0") +load ("python/3.10.13") + +local pyenvpath = "/scratch1/NCEPDEV/da/Edward.Safford/noscrub/python/envs/" +local pyenvname = "obs-mon" + +local pyenvactivate = pathJoin(pyenvpath, pyenvname, "bin/activate") + +if (mode() == "load") then + local activate_cmd = "source "..pyenvactivate + execute{cmd=activate_cmd, modeA={"load"}} + prepend_path("PATH", "/scratch1/NCEPDEV/da/Edward.Safford/noscrub/python/envs/obs-mon/bin") + prepend_path("PYTHONPATH", "/scratch1/NCEPDEV/da/Edward.Safford/noscrub/python/envs/obs-mon") + +else + if (mode() == "unload") then + local deactivate_cmd = "deactivate" + execute{cmd=deactivate_cmd, modeA={"unload"}} + end +end + +whatis("Name: ".. pkgName) +whatis("Version: ".. pkgVersion) +whatis("Category: Obs-monitor") +whatis("Description: Load all libraries needed for obs-monitor") + diff --git a/modulefiles/obs-monitor/hercules.lua b/modulefiles/obs-monitor/hercules.lua new file mode 100644 index 0000000..2b3f130 --- /dev/null +++ b/modulefiles/obs-monitor/hercules.lua @@ -0,0 +1,39 @@ +help([[ +Load python virtual environment for obs-monitor +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +conflict(pkgName) + + +prepend_path("MODULEPATH", " /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env-intel-2023.2.4/install/modulefiles/Core") + +load ("stack-intel/2021.10.0") +load ("python/3.10.8") + +local pyenvpath = "/work/noaa/da/esafford/noscrub/python/envs/" +local pyenvname = "obs-mon" + +local pyenvactivate = pathJoin(pyenvpath, pyenvname, "bin/activate") + +if (mode() == "load") then + local activate_cmd = "source "..pyenvactivate + execute{cmd=activate_cmd, modeA={"load"}} + prepend_path("PATH", "/work/noaa/da/esafford/noscrub/python/envs/obs-mon/bin") + prepend_path("PYTHONPATH", "/work/noaa/da/esafford/noscrub/python/envs/obs-mon") + +else + if (mode() == "unload") then + local deactivate_cmd = "deactivate" + execute{cmd=deactivate_cmd, modeA={"unload"}} + end +end + +whatis("Name: ".. pkgName) +whatis("Version: ".. pkgVersion) +whatis("Category: Obs-monitor") +whatis("Description: Load all libraries needed for obs-monitor") + diff --git a/modulefiles/obs-monitor/orion.lua b/modulefiles/obs-monitor/orion.lua new file mode 100644 index 0000000..a835b84 --- /dev/null +++ b/modulefiles/obs-monitor/orion.lua @@ -0,0 +1,39 @@ +help([[ +Load python virtual environment for obs-monitor +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +conflict(pkgName) + + +prepend_path("MODULEPATH", " /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core") + +load ("stack-intel/2021.9.0") +load ("python/3.10.8") + +local pyenvpath = "/work/noaa/da/esafford/noscrub/python/envs/" +local pyenvname = "obs-mon" + +local pyenvactivate = pathJoin(pyenvpath, pyenvname, "bin/activate") + +if (mode() == "load") then + local activate_cmd = "source "..pyenvactivate + execute{cmd=activate_cmd, modeA={"load"}} + prepend_path("PATH", "/work/noaa/da/esafford/noscrub/python/envs/obs-mon/bin") + prepend_path("PYTHONPATH", "/work/noaa/da/esafford/noscrub/python/envs/obs-mon") + +else + if (mode() == "unload") then + local deactivate_cmd = "deactivate" + execute{cmd=deactivate_cmd, modeA={"unload"}} + end +end + +whatis("Name: ".. pkgName) +whatis("Version: ".. pkgVersion) +whatis("Category: Obs-monitor") +whatis("Description: Load all libraries needed for obs-monitor") + diff --git a/modulefiles/obs-monitor/wcoss2.lua b/modulefiles/obs-monitor/wcoss2.lua new file mode 100644 index 0000000..1e9587e --- /dev/null +++ b/modulefiles/obs-monitor/wcoss2.lua @@ -0,0 +1,36 @@ +help([[ +Load python virtual environment for obs-monitor +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +conflict(pkgName) + + +load ("python/3.10.4") + +local pyenvpath = "/lfs/h2/emc/da/noscrub/edward.safford/python/envs/" +local pyenvname = "obs-mon" + +local pyenvactivate = pathJoin(pyenvpath, pyenvname, "bin/activate") + +if (mode() == "load") then + local activate_cmd = "source "..pyenvactivate + execute{cmd=activate_cmd, modeA={"load"}} + prepend_path("PATH", "/lfs/h2/emc/da/noscrub/edward.safford/python/envs/obs-mon/bin") + prepend_path("PYTHONPATH", "/lfs/h2/emc/da/noscrub/edward.safford/python/envs/obs-mon") + +else + if (mode() == "unload") then + local deactivate_cmd = "deactivate" + execute{cmd=deactivate_cmd, modeA={"unload"}} + end +end + +whatis("Name: ".. pkgName) +whatis("Version: ".. pkgVersion) +whatis("Category: Obs-monitor") +whatis("Description: Load all libraries needed for obs-monitor") + diff --git a/parm/OM_config b/parm/OM_config index 7cdc7a0..24b3b09 100644 --- a/parm/OM_config +++ b/parm/OM_config @@ -9,32 +9,61 @@ export JOBSobsmon=${om_dir_root}/jobs source ${om_dir_root}/ush/detect_machine.sh export MACHINE_ID=`echo "${MACHINE_ID%%.*}"` +module use ${HOMEobsmon}/modulefiles + case ${MACHINE_ID} in hera) export SUB="sbatch" export SERVICE_PARTITION="service" + export CARTOPY_DATA_DIR="/scratch1/NCEPDEV/da/Edward.Safford/noscrub/python/envs/obs-mon/share/cartopy" - ptmp="/scratch2/NCEPDEV/stmp3/$USER" - stmp="/scratch2/NCEPDEV/stmp1/$USER" + ptmp="/scratch2/NCEPDEV/stmp3/${USER}" + stmp="/scratch2/NCEPDEV/stmp1/${USER}" queue="" project="" account="da-cpu" - aprun_py="python" ;; wcoss2) export SUB="qsub" + export CARTOPY_DATA_DIR="/lfs/h2/emc/da/noscrub/edward.safford/python/envs/obs-mon/share/cartopy" - ptmp="/lfs/h2/emc/ptmp/$USER" - stmp="/lfs/h2/emc/stmp/$USER" + ptmp="/lfs/h2/emc/ptmp/${USER}" + stmp="/lfs/h2/emc/stmp/${USER}" queue="dev" project="GDAS-DEV" account="GFS-DEV" - aprun_py=`which python` + ;; + + orion) + export SUB="sbatch" + export CARTOPY_DATA_DIR="/work/noaa/da/esafford/noscrub/python/envs/obs-mon/share/cartopy" + + ptmp="/work2/noaa/stmp/${USER}" + stmp="/work/noaa/stmp/${USER}" + + queue="" + project="" + account="da-cpu" + ;; + + hercules) + export SUB="sbatch" + export CARTOPY_DATA_DIR="/work/noaa/da/esafford/noscrub/python/envs/obs-mon/share/cartopy" + + ptmp="/work2/noaa/stmp/${USER}" + stmp="/work/noaa/stmp/${USER}" + + queue="" + project="" + account="da-cpu" ;; esac +module load obs-monitor/${MACHINE_ID} +export APRUN_PY=`which python` + export ACCOUNT=${ACCOUNT:-$account} export PROJECT=${PROJECT:-$project} export JOB_QUEUE=${JOB_QUEUE:-$queue} @@ -42,4 +71,3 @@ export JOB_QUEUE=${JOB_QUEUE:-$queue} export OM_LOGS="${ptmp}/logs/OM_logs" export DATAROOT=${DATAROOT:-${stmp}} export COMOUT=${COMOUT:-${ptmp}} -export APRUN_PY=${APRUN_PY:-${aprun_py}} diff --git a/parm/gfs/gfs_plots.yaml b/parm/gfs/gfs_plots.yaml index 9282523..6be4878 100644 --- a/parm/gfs/gfs_plots.yaml +++ b/parm/gfs/gfs_plots.yaml @@ -2,7 +2,8 @@ model: gfs cycle_interval: 6 -data: '/scratch1/NCEPDEV/da/Edward.Safford/noscrub/test_data/gfs' +#data: '/scratch1/NCEPDEV/da/Edward.Safford/noscrub/test_data/gfs' +data: '/lfs/h2/emc/da/noscrub/edward.safford/test_data/gfs' satellites: - name: aura @@ -38,7 +39,6 @@ satellites: levels: '1' component: ges - - name: ompslp plot_list: - plot: ozn horiz @@ -174,11 +174,10 @@ satellites: times: 121 channels: 'all' component: ges -# test data file corruption here? -# - plot: rad angle -# times: 121 -# channels: 'all' -# component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges - name: mhs plot_list: @@ -321,51 +320,51 @@ satellites: channels: 'all' component: ges -# test data file corruption here -# - name: n19 -# instruments: -# - name: amsua -# plot_list: -# - plot: rad time -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad bcoef -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad angle -# times: 121 -# channels: 'all' -# component: ges -# -# - name: avhrr -# plot_list: -# - plot: rad time -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad bcoef -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad angle -# times: 121 -# channels: 'all' -# component: ges - -# - name: mhs -# plot_list: -# - plot: rad time -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad bcoef -# times: 121 -# channels: 'all' -# component: ges -# - plot: rad angle -# times: 121 + - name: n19 + instruments: + - name: amsua + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: avhrr + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + channels: 'all' + component: ges + + - name: mhs + plot_list: + - plot: rad time + times: 121 + channels: 'all' + component: ges + - plot: rad bcoef + times: 121 + channels: 'all' + component: ges + - plot: rad angle + times: 121 + component: ges - name: n20 instruments: @@ -514,8 +513,6 @@ satellites: channels: 'all' component: ges - - minimization: - model: gfs plot_list: @@ -531,7 +528,6 @@ minimization: run: gdas times: 28 - observations: - obstype: ps plot_list: diff --git a/scripts/exobsmon_plot.sh b/scripts/exobsmon_plot.sh index c2aa55e..42a6543 100755 --- a/scripts/exobsmon_plot.sh +++ b/scripts/exobsmon_plot.sh @@ -41,7 +41,7 @@ if compgen -G "${DATA}/OM_PLOT*.yaml" > /dev/null; then for yaml in ${DATA}/OM_PLOT*.yaml; do echo "processing yaml: $ctr $yaml" case ${MACHINE_ID} in - hera) + hera|orion|hercules) echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml} -p ${PDATE}" >> ${cmdfile} ;; wcoss2) @@ -54,29 +54,29 @@ if compgen -G "${DATA}/OM_PLOT*.yaml" > /dev/null; then if (( ${ctr} > 0 )); then case ${MACHINE_ID} in - hera) + hera|orion|hercules) ${SUB} --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}" ;; wcoss2) - # prepend setup script to $cmdfile - echo -e ". ${USHobsmon}/setup_wcoss2.sh\n $(cat ${cmdfile})" > ${cmdfile} - chmod 775 ${cmdfile} - - mem=$((4*${ctr})) + mem=$((8*${ctr})) echo "submitting ${jobname} on wcoss2, ctr = $ctr, mem = $mem, cmdfile = ${cmdfile}" ${SUB} -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${logfile} \ -v "PYTHONPATH=${PYTHONPATH}, PATH=${PATH}, HOMEobsmon=${HOMEobsmon}, MODEL=${MODEL}, \ - CNTRLobsmon=${CNTRLobsmon}, PARMobsmon=${PARMobsmon}, DATA=${DATA}, \ + CNTRLobsmon=${CNTRLobsmon}, PARMobsmon=${PARMobsmon}, DATA=${DATA}, CARTOPY_DATA_DIR=${CARTOPY_DATA_DIR}, \ LD_LIBRARY_PATH=${LD_LIBRARY_PATH}, cmdfile=${cmdfile}, ncpus=${ctr}" \ - -l place=vscatter,select=1:ncpus=${ctr}:mem=${mem}gb,walltime=1:30:00 -N ${jobname} ${cmdfile} + -l place=vscatter,select=1:ncpus=${ctr}:mem=${mem}gb:prepost=true,walltime=1:00:00 -N ${jobname} ${USHobsmon}/plot_wcoss2.sh ;; esac fi fi +# +# Need a new job to run following the plot job to clean up $DATA +# + #----------------------------- # Copy output to COMOUTplots # diff --git a/ush/setup_wcoss2.sh b/ush/plot_wcoss2.sh similarity index 50% rename from ush/setup_wcoss2.sh rename to ush/plot_wcoss2.sh index 6becb3e..77631f6 100644 --- a/ush/setup_wcoss2.sh +++ b/ush/plot_wcoss2.sh @@ -5,14 +5,21 @@ module reset module load libfabric/1.11.0.0. module load PrgEnv-intel/8.3.3 module load craype +module load cray-mpich/8.1.12 module load cray-pals +module load cfp/2.0.4 + module load git/2.29.0 module load intel/19.1.3.304 +module load netcdf/4.7.4 module load python/3.10.4 -module load ve/evs/1.0 -export PATH=${PATH}:/lfs/h2/emc/da/noscrub/edward.safford/eva/opt/bin -export PYTHONPATH=${PYTHONPATH}:/lfs/h2/emc/da/noscrub/edward.safford/eva/opt/ +module use ${HOMEobsmon}/modulefiles +module load obs-monitor/wcoss2 + +module list +# cd to working directory from which this was launched if [ ! -z ${PBS_O_WORKDIR} ]; then cd ${PBS_O_WORKDIR}; fi +mpiexec -np ${ncpus} --cpu-bind core cfp ${cmdfile}