diff --git a/build.sh b/build.sh index 42ec34284..4ebb3666e 100755 --- a/build.sh +++ b/build.sh @@ -103,14 +103,12 @@ WORKFLOW_BUILD=${WORKFLOW_BUILD:-"OFF"} CMAKE_OPTS+=" -DWORKFLOW_TESTS=${WORKFLOW_BUILD}" # JCSDA changed test data things, need to make a dummy CRTM directory -if [[ $BUILD_TARGET == 'hera' ]]; then - if [ -d "$dir_root/bundle/fix/test-data-release/" ]; then rm -rf $dir_root/bundle/fix/test-data-release/; fi - if [ -d "$dir_root/bundle/test-data-release/" ]; then rm -rf $dir_root/bundle/test-data-release/; fi - mkdir -p $dir_root/bundle/fix/test-data-release/ - mkdir -p $dir_root/bundle/test-data-release/ - ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/fix/test-data-release/crtm - ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/test-data-release/crtm -fi +if [ -d "$dir_root/bundle/fix/test-data-release/" ]; then rm -rf $dir_root/bundle/fix/test-data-release/; fi +if [ -d "$dir_root/bundle/test-data-release/" ]; then rm -rf $dir_root/bundle/test-data-release/; fi +mkdir -p $dir_root/bundle/fix/test-data-release/ +mkdir -p $dir_root/bundle/test-data-release/ +ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/fix/test-data-release/crtm +ln -sf $GDASAPP_TESTDATA/crtm $dir_root/bundle/test-data-release/crtm # Configure echo "Configuring ..." diff --git a/ci/driver.sh b/ci/driver.sh index b6572914f..933b9223b 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -1,8 +1,9 @@ #!/bin/bash --login -set -x +echo "Start at $(date)" my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" +echo "Set my_dir ${my_dir}" # ============================================================================== usage() { @@ -53,6 +54,14 @@ esac cd $GDAS_CI_ROOT/repo CI_LABEL="${GDAS_CI_HOST}-RT" gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list + +open_pr=`cat $GDAS_CI_ROOT/open_pr_list | wc -l` +if (( $open_pr == 0 )); then + echo "No open PRs with ${CI_LABEL}, exit." + echo "Finish at $(date)" + exit +fi + open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list) # ============================================================================== @@ -60,72 +69,86 @@ open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list) repo_url="https://github.com/NOAA-EMC/GDASApp.git" # loop through all open PRs for pr in $open_pr_list; do - gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - echo "Processing Pull Request #${pr}" + echo " " + echo "Start processing Pull Request #${pr} at $(date)" # get the branch name used for the PR gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - - # get the fork information - pr_details=$(gh pr view $pr --repo ${repo_url} --json headRepository,headRepositoryOwner,headRefName) - # extract the necessary info - fork_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') - fork_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') - - # construct the fork URL - gdasapp_url="https://github.com/$fork_owner/${fork_name}.git" + # get additional branch information + branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') + pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login') + + # check if any assignee is authorized to run CI + authorized_by="" + for str in ${pr_assignees[@]}; do + grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users + rc=$? + if (( rc == 0 )); then + authorized_by=${str} + echo "FOUND MATCH $str, rc $rc" + break + fi + done + + # Authorized to run CI + if (( rc == 0 )); then + echo "Run CI" + + # update PR label + gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running + + # construct the fork URL + gdasapp_url="https://github.com/$branch_owner/${branch_name}.git" - echo "Fork URL: $gdasapp_url" - echo "Branch Name: $gdasapp_branch" - - # create PR specific directory - if [ -d $GDAS_CI_ROOT/PR/$pr ]; then - rm -rf $GDAS_CI_ROOT/PR/$pr - fi - mkdir -p $GDAS_CI_ROOT/PR/$pr - cd $GDAS_CI_ROOT/PR/$pr - - # clone copy of repo - git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url - cd GDASApp - - # checkout pull request - git pull - gh pr checkout $pr - git submodule update --init --recursive - - # get commit hash - commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit - - # load modules - case ${TARGET} in - hera | orion) - echo "Loading modules on $TARGET" - module purge - module use $GDAS_CI_ROOT/PR/$pr/GDASApp/modulefiles - module load GDAS/$TARGET - module list - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; - esac - - # run build and testing command - $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} - ci_status=$? - gh pr comment $pr --repo ${repo_url} --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} - if [ $ci_status -eq 0 ]; then - gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + echo "GDASApp URL: $gdasapp_url" + echo "GDASApp branch Name: $gdasapp_branch" + echo "CI authorized by $authorized_by at $(date)" + + # create PR specific directory + if [ -d $GDAS_CI_ROOT/PR/$pr ]; then + rm -rf $GDAS_CI_ROOT/PR/$pr + fi + mkdir -p $GDAS_CI_ROOT/PR/$pr + cd $GDAS_CI_ROOT/PR/$pr + pwd + + # clone copy of repo + git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url + cd GDASApp + pwd + + # checkout GDASApp pull request + git pull + gh pr checkout $pr + git submodule update --init --recursive + + # get commit hash + commit=$(git log --pretty=format:'%h' -n 1) + echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit + + # run build and testing command + echo "Execute $my_dir/run_ci.sh for $GDAS_CI_ROOT/PR/$pr/GDASApp at $(date)" + $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} + ci_status=$? + echo "After run_ci.sh with ci_status ${ci_status} at $(date)" + gh pr comment $pr --repo ${repo_url} --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} + if [ $ci_status -eq 0 ]; then + gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + else + gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + fi + + # Not authorized to run CI else - gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + echo "Do NOT run CI" fi + + echo "Finish processing Pull Request #{pr} at $(date)" done # ============================================================================== # scrub working directory for older files find $GDAS_CI_ROOT/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; - +echo "Finish at $(date)" diff --git a/ci/gw_driver.sh b/ci/gw_driver.sh index e85684f84..c40ff4026 100755 --- a/ci/gw_driver.sh +++ b/ci/gw_driver.sh @@ -1,6 +1,9 @@ #!/bin/bash --login +echo "Start at $(date)" + my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" +echo "Set my_dir ${my_dir}" # ============================================================================== usage() { @@ -32,7 +35,7 @@ done case ${TARGET} in hera | orion) - echo "Running automated testing with workflow on $TARGET" + echo "Running Automated GW Testing on $TARGET" source $MODULESHOME/init/sh source $my_dir/${TARGET}.sh module purge @@ -51,69 +54,116 @@ esac cd $GDAS_CI_ROOT/repo CI_LABEL="${GDAS_CI_HOST}-GW-RT" gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list_gw + +open_pr=`cat $GDAS_CI_ROOT/open_pr_list_gw | wc -l` +if (( $open_pr == 0 )); then + echo "No open PRs with ${CI_LABEL}, exit." + echo "Finish at $(date)" + exit +fi + open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list_gw) # ============================================================================== # clone, checkout, build, test, etc. repo_url="https://github.com/NOAA-EMC/GDASApp.git" workflow_url="https://github.com/NOAA-EMC/global-workflow.git" +workflow_branch="develop" # loop through all open PRs for pr in $open_pr_list; do - gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - echo "Processing Pull Request #${pr}" + echo " " + echo "Start processing Pull Request #${pr} at $(date)" # get the branch name used for the PR gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - # check for a companion PR in the global-workflow - companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) - if [ -n "$companion_pr_exists" ]; then - # get the PR number - companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') - - # extract the necessary info - fork_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') - fork_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') - - # Construct the fork URL - workflow_url="https://github.com/$fork_owner/$fork_name.git" - - echo "Fork URL: $workflow_url" - echo "Branch Name: $gdasapp_branch" - fi - - # create PR specific directory - if [ -d $GDAS_CI_ROOT/workflow/PR/$pr ]; then - rm -rf $GDAS_CI_ROOT/workflow/PR/$pr - fi - mkdir -p $GDAS_CI_ROOT/workflow/PR/$pr - cd $GDAS_CI_ROOT/workflow/PR/$pr + # get additional branch information + branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') + pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login') + + # check if any assignee is authorized to run CI + authorized_by="" + for str in ${pr_assignees[@]}; do + grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users + rc=$? + if (( rc == 0 )); then + authorized_by=${str} + echo "FOUND MATCH $str, rc $rc" + break + fi + done + + # Authorized to run CI + if (( rc == 0 )); then + echo "Run CI" + + # update PR label + gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running + + # check for a companion PR in the global-workflow + companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) + if [ -n "$companion_pr_exists" ]; then + # get the PR number + companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') + + # extract the necessary info + branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') + + # Construct fork URL. Update workflow branch name + workflow_url="https://github.com/$branch_owner/$branch_name.git" + workflow_branch=$gdasapp_branch + + fi + + echo "Workflow URL: $workflow_url" + echo "Workflow branch name: $workflow_branch" + echo "GDASApp branch name: $gdasapp_branch" + echo "CI authorized by $authorized_by at $(date)" + + # create PR specific directory + if [ -d $GDAS_CI_ROOT/workflow/PR/$pr ]; then + rm -rf $GDAS_CI_ROOT/workflow/PR/$pr + fi + mkdir -p $GDAS_CI_ROOT/workflow/PR/$pr + cd $GDAS_CI_ROOT/workflow/PR/$pr + pwd - # clone global workflow develop branch - git clone --recursive --jobs 8 --branch dev/gdasapp $workflow_url - - # checkout pull request - cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc/gdas.cd - git checkout develop - git pull - gh pr checkout $pr - git submodule update --init --recursive - - # get commit hash - commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $GDAS_CI_ROOT/workflow/PR/$pr/commit - - $my_dir/run_gw_ci.sh -d $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow -o $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} - ci_status=$? - gh pr comment $pr --body-file $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} - if [ $ci_status -eq 0 ]; then - gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + # clone global workflow develop branch + git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url + + # checkout GDASApp pull request + cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc/gdas.cd + git pull + gh pr checkout $pr + git submodule update --init --recursive + + # get commit hash + commit=$(git log --pretty=format:'%h' -n 1) + echo "$commit" > $GDAS_CI_ROOT/workflow/PR/$pr/commit + + # run build and testing command + echo "Execute $my_dir/run_gw_ci.sh for $GDAS_CI_ROOT/PR/workflow/PR/$pr/global-workflow at $(date)" + $my_dir/run_gw_ci.sh -d $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow -o $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} + ci_status=$? + echo "After run_gw_ci.sh with ci_status ${ci_status} at $(date)" + gh pr comment $pr --body-file $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} + if [ $ci_status -eq 0 ]; then + gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + else + gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + fi + + # Not authorized to run CI else - gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + echo "Do NOT run CI" fi + + echo "Finish processing Pull Request #{pr} at $(date)" done # ============================================================================== # scrub working directory for older files find $GDAS_CI_ROOT/workflow/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; - +echo "Finish at $(date)" diff --git a/ci/run_ci.sh b/ci/run_ci.sh index b62f78a88..1b5f27231 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -1,5 +1,5 @@ #!/bin/bash -#set -eu +set -u # ============================================================================== usage() { @@ -61,7 +61,7 @@ module use $GDAS_MODULE_USE module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile rm -rf log.ctest -ctest -E "manual" -R gdasapp --output-on-failure &>> log.ctest +ctest -R gdasapp --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then diff --git a/ci/run_gw_ci.sh b/ci/run_gw_ci.sh index 59758e37f..ba1874107 100755 --- a/ci/run_gw_ci.sh +++ b/ci/run_gw_ci.sh @@ -1,5 +1,5 @@ #!/bin/bash -#set -eu +set -u # ============================================================================== usage() { @@ -31,7 +31,7 @@ done # ============================================================================== # start output file -echo "Automated Global-Workflow GDASApp Testing Results:" > $outfile +echo "Automated GW GDASApp Testing Results:" > $outfile echo "Machine: ${TARGET}" >> $outfile echo '```' >> $outfile echo "Start: $(date) on $(hostname)" >> $outfile