Add Parallelworks CI scripts to repo

.github/.parallelworks/README.md

@@ -0,0 +1,6 @@
+# .parallelworks Directory
+
+The .parallelworks directory stores the CI scripts that reside on Parallelworks
+These scripts are executed via the GitHub Actions Workflows in .github/workflows
+
+On Parallelworks these scripts are installed at: /contrib/fv3/SHiELD_build_CI

.github/.parallelworks/checkout.sh

@@ -0,0 +1,88 @@
+#!/bin/sh -xe
+
+##############################################################################
+## User set up variables
+## Root directory for CI
+dirRoot=/contrib/fv3
+## Intel version to be used
+intelVersion=2023.2.0
+##############################################################################
+## HPC-ME container
+container=/contrib/containers/noaa-intel-prototype_2023.09.25.sif
+container_env_script=/contrib/containers/load_spack_noaa-intel.sh
+##############################################################################
+## Set up the directories
+# First argument should be $GITHUB_REF which is the reference to the PR/branch
+# to be checked out for SHiELD_build
+if [ -z "$1" ]
+  then
+    echo "No branch/PR supplied; using main"
+    branch=main
+  else
+    echo Branch is ${1}
+    branch=${1}
+fi
+# Second Argument should be $GITHUB_SHA which  is the commit hash of the
+# branch or PR to trigger the CI, if run manually, you do not need a 2nd
+# argument.  This is needed in the circumstance where a PR is created, 
+# then the CI triggers, and before that CI has finished, the developer
+# pushes a newer commit which triggers a second round of CI.  We would
+# like unique directories so that both CI runs do not interfere.
+if [ -z "$2" ]
+  then
+    echo "No second argument"
+    commit=""
+  else
+    echo Commit is ${2}
+    commit=${2}
+fi
+
+testDir=${dirRoot}/${intelVersion}/SHiELD_build/${branch}/${commit}
+logDir=${testDir}/log
+export MODULESHOME=/usr/share/lmod/lmod
+#Define External Libs path
+export EXTERNAL_LIBS=${dirRoot}/${intelVersion}/SHiELD_build/externallibs
+mkdir -p ${EXTERNAL_LIBS}
+## create directories
+rm -rf ${testDir}
+mkdir -p ${logDir}
+# salloc commands to start up 
+#2 tests layout 8,8 (16 nodes)
+#2 tests layout 4,8 (8 nodes)
+#9 tests layout 4,4 (18 nodes)
+#5 tests layout 4,1 (5 nodes)
+#17 tests layout 2,2 (17 nodes)
+#salloc --partition=p2 -N 64 -J ${branch} sleep 20m &
+
+## clone code
+cd ${testDir}
+git clone --recursive https://github.com/NOAA-GFDL/SHiELD_build.git  
+## Check out the PR
+cd ${testDir}/SHiELD_build && git fetch origin ${branch}:toMerge && git merge toMerge
+
+##checkout components
+cd ${testDir}/SHiELD_build && ./CHECKOUT_code
+#Check if we already have FMS compiled
+grep -m 1 "fms_release" ${testDir}/SHiELD_build/CHECKOUT_code > ${logDir}/release.txt
+source ${logDir}/release.txt
+echo ${fms_release}
+echo `cat ${EXTERNAL_LIBS}/FMSversion`
+if [[ ${fms_release} != `cat ${EXTERNAL_LIBS}/FMSversion` ]]
+  then
+    #remove libFMS if it exists
+    if [ -d $EXTERNAL_LIBS/libFMS ]
+      then
+        rm -rf $EXTERNAL_LIBS/libFMS
+    fi
+    if [ -e $EXTERNAL_LIBS/FMSversion ]
+      then
+        rm $EXTERNAL_LIBS/FMSversion
+    fi
+    echo $fms_release > $EXTERNAL_LIBS/FMSversion
+    echo $container > $EXTERNAL_LIBS/FMScontainerversion
+    echo $container_env_script >> $EXTERNAL_LIBS/FMScontainerversion
+    # Build FMS
+    cd ${testDir}/SHiELD_build/Build
+    set -o pipefail
+    singularity exec -B /contrib ${container} ${container_env_script} "./BUILDlibfms intel"
+ fi

.github/.parallelworks/hydrocompile.sh

@@ -0,0 +1,39 @@
+#!/bin/sh -xe
+
+##############################################################################
+## User set up variables
+## Root directory for CI
+dirRoot=/contrib/fv3
+## Intel version to be used
+intelVersion=2023.2.0
+##############################################################################
+## HPC-ME container
+container=/contrib/containers/noaa-intel-prototype_2023.09.25.sif
+container_env_script=/contrib/containers/load_spack_noaa-intel.sh
+##############################################################################
+## Set up the directories
+if [ -z "$1" ]
+  then
+    echo "No branch/PR supplied; using main"
+    branch=main
+  else
+    echo Branch is ${1}
+    branch=${1}
+fi
+if [ -z "$2" ]
+  then
+    echo "No second argument"
+    commit=none
+  else
+    echo Commit is ${2}
+    commit=${2}
+fi
+testDir=${dirRoot}/${intelVersion}/SHiELD_build/${branch}/${commit}
+logDir=${testDir}/log
+# Set up build
+cd ${testDir}/SHiELD_build/Build
+#Define External Libs path
+export EXTERNAL_LIBS=${dirRoot}/externallibs
+# Build SHiELD
+set -o pipefail
+singularity exec -B /contrib ${container} ${container_env_script} "./COMPILE solo hydro 64bit repro intel clean"

.github/.parallelworks/nhcompile.sh

@@ -0,0 +1,39 @@
+#!/bin/sh -xe
+
+##############################################################################
+## User set up variables
+## Root directory for CI
+dirRoot=/contrib/fv3
+## Intel version to be used
+intelVersion=2023.2.0
+##############################################################################
+## HPC-ME container
+container=/contrib/containers/noaa-intel-prototype_2023.09.25.sif
+container_env_script=/contrib/containers/load_spack_noaa-intel.sh
+##############################################################################
+## Set up the directories
+if [ -z "$1" ]
+  then
+    echo "No branch/PR supplied; using main"
+    branch=main
+  else
+    echo Branch is ${1}
+    branch=${1}
+fi
+if [ -z "$2" ]
+  then
+    echo "No second argument"
+    commit=none
+  else
+    echo Commit is ${2}
+    commit=${2}
+fi
+testDir=${dirRoot}/${intelVersion}/SHiELD_build/${branch}/${commit}
+logDir=${testDir}/log
+# Set up build
+cd ${testDir}/SHiELD_build/Build
+#Define External Libs path
+export EXTERNAL_LIBS=${dirRoot}/externallibs
+# Build SHiELD
+set -o pipefail
+singularity exec -B /contrib ${container} ${container_env_script} "./COMPILE solo nh 64bit repro intel clean"

.github/.parallelworks/run_test.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -xe
+ulimit -s unlimited
+##############################################################################
+## User set up veriables
+## Root directory for CI
+dirRoot=/contrib/fv3
+## Intel version to be used
+intelVersion=2023.2.0
+##############################################################################
+## HPC-ME container
+container=/contrib/containers/noaa-intel-prototype_2023.09.25.sif
+container_env_script=/contrib/containers/load_spack_noaa-intel.sh
+
+##Parse Arguments
+#first argument should be the name of the test and is mandatory
+if [ -z "$1" ]
+  then
+    echo "Please run this script with an argument indicating what test to run.  For example:"
+    echo "./run_test.sh C128r20.solo.superC"
+  else
+    echo Test is ${1}
+    test=${1}
+fi
+#second argument is the branch name. This is optional. Default is main branch if none supplied
+if [ -z "$2" ]
+  then
+    echo "No branch supplied; using main"
+    branch=main
+  else
+    echo Branch is ${2}
+    branch=${2}
+fi
+#third argument is the commit hash if running from CI.  This is optional
+if [ -z "$3" ]
+  then
+    echo "No commit being used in file path"
+    commit=""
+  else
+    echo Commit is ${3}
+    commit=${3}
+fi
+
+## Set up the directories
+MODULESHOME=/usr/share/lmod/lmod
+testDir=${dirRoot}/${intelVersion}/SHiELD_build/${branch}/${commit}
+logDir=${testDir}/log
+baselineDir=${dirRoot}/baselines/intel/${intelVersion}
+
+## Run the CI Test
+# Define the builddir testscriptdir and rundir 
+# Set the BUILDDIR for the test script to use
+export BUILDDIR="${testDir}/SHiELD_build"
+testscriptDir=${BUILDDIR}/RTS/CI
+runDir=${BUILDDIR}/CI/BATCH-CI
+
+# Run CI test scripts
+cd ${testscriptDir}
+set -o pipefail
+# Execute the test piping output to log file
+./${test} " --partition=p2 --mpi=pmi2 --job-name=${commit}_${test} singularity exec -B /contrib ${container} ${container_env_script}" |& tee ${logDir}/run_${test}.log
+
+## Compare Restarts to Baseline
+#The following tests are not expectred to have run-to-run reproducibility:
+#d96_2k.solo.bubble
+#d96_2k.solo.bubble.n0
+#d96_2k.solo.bubble.nhK
+if [[ ${test} == "d96_2k.solo.bubble" || ${test} == "d96_2k.solo.bubble.n0" || ${test} == "d96_2k.solo.bubble.nhK" ]]
+  then
+    echo "${test} is not expected to reproduce so answers were not compared"
+  else
+    source $MODULESHOME/init/sh
+    export MODULEPATH=/mnt/shared/manual_modules:/usr/share/modulefiles/Linux:/usr/share/modulefiles/Core:/usr/share/lmod/lmod/modulefiles/Core:/apps/modules/modulefiles:/apps/modules/modulefamilies/intel
+    module load intel/2022.1.2
+    module load netcdf
+    module load nccmp
+    for resFile in `ls ${baselineDir}/${test}`
+    do
+      nccmp -d ${baselineDir}/${test}/${resFile} ${runDir}/${test}/RESTART/${resFile}
+    done
+fi

.github/.parallelworks/swcompile.sh

@@ -0,0 +1,40 @@
+#!/bin/sh -xe
+
+##############################################################################
+## User set up variables
+## Root directory for CI
+dirRoot=/contrib/fv3
+## Intel version to be used
+intelVersion=2023.2.0
+##############################################################################
+## HPC-ME container
+container=/contrib/containers/noaa-intel-prototype_2023.09.25.sif
+container_env_script=/contrib/containers/load_spack_noaa-intel.sh
+##############################################################################
+## Set up the directories
+if [ -z "$1" ]
+  then
+    echo "No branch/PR supplied; using main"
+    branch=main
+  else
+    echo Branch is ${1}
+    branch=${1}
+fi
+if [ -z "$2" ]
+  then
+    echo "No second argument"
+    commit=none
+  else
+    echo Commit is ${2}
+    commit=${2}
+fi
+testDir=${dirRoot}/${intelVersion}/SHiELD_build/${branch}/${commit}
+logDir=${testDir}/log
+# Set up build
+cd ${testDir}/SHiELD_build/Build
+#Define External Libs path
+export EXTERNAL_LIBS=${dirRoot}/externallibs
+# Build SHiELD
+set -o pipefail
+singularity exec -B /contrib ${container} ${container_env_script} "./COMPILE solo sw 64bit repro intel clean"
+