From 8966cc43b16a732b999823ad7ed667fca149619b Mon Sep 17 00:00:00 2001 From: Helen Kershaw Date: Wed, 12 Jun 2024 10:09:33 -0400 Subject: [PATCH 1/3] feat: github action for running all quickbuilds.sh setup - finds all the quickbuilds skips the *template* model build - matrix of jobs, one for each quickbuild.sh Using the standard dart dependency container, no RTTOV at the moment developer_tests/forward_operators/work observations/obs_converters/GMI/work observations/obs_converters/GOES/work observations/obs_converters/AIRS/work observations/obs_converters/NSIDC/work - only needs HDF5 but this is in the RTTOV container Other failures observations/obs_converters/gps/work - needs WRF files observations/obs_converters/quikscat/work - type mismatch needs to be allowed for JPL code, also needs hdf4 observations/obs_converters/GSI2DART/work - GSI MPI type mismatch observations/obs_converters/gps/work - needs prepbufr library --- .github/workflows/run_all_quickbuilds.yml | 61 +++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/run_all_quickbuilds.yml diff --git a/.github/workflows/run_all_quickbuilds.yml b/.github/workflows/run_all_quickbuilds.yml new file mode 100644 index 0000000000..95af60b500 --- /dev/null +++ b/.github/workflows/run_all_quickbuilds.yml @@ -0,0 +1,61 @@ +name: 'run all quickbuilds' +on: + workflow_dispatch: + +jobs: + setup: + runs-on: ubuntu-latest + container: + image: hkershaw/dart-dep:1.0 + options: "--cap-add=SYS_PTRACE" + outputs: + matrix: ${{ steps.matrix.outputs.value }} + steps: + - uses: actions/checkout@v4 + - id: matrix + run: | + git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} + DART=$(git rev-parse --show-toplevel) + files_to_process=( $(find $DART -executable -type f -name quickbuild.sh | sed -E 's#(\./|quickbuild\.sh)##g') ) + joined_items="" + for item in "${files_to_process[@]}"; do + if [[ $item == *"template"* ]]; then + continue # template model not for compiling + fi + # Append the quoted item and a comma to the string + stripped_item=${item#/__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/} # removes /__w/DART/DART + echo $stripped_item + joined_items+="\"$stripped_item\"," + done + + # Remove the trailing comma from the last item + joined_items="${joined_items%,}" + + echo "value=[ $joined_items ]" >> $GITHUB_OUTPUT + shell: bash + build: + needs: [ setup ] + runs-on: ubuntu-latest + container: + image: hkershaw/dart-dep:1.0 + options: "--cap-add=SYS_PTRACE" + strategy: + fail-fast: false + matrix: + value: ${{fromJSON(needs.setup.outputs.matrix)}} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set checked out repo as a safe git directory + run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} + - name: Creating Makefile template + run: | + cd build_templates + cp mkmf.template.gfortran mkmf.template + echo 'FFLAGS = -g -Wuninitialized -Wunused -ffree-line-length-none -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow $(INCS)' >> mkmf.template + shell: bash + - name: Run quickbuild.sh script + run: | + cd ${{ matrix.value }} + ./quickbuild.sh + shell: bash From fd3d9a8d9b03a5c473936c70b2d022ded83773df Mon Sep 17 00:00:00 2001 From: Helen Kershaw Date: Wed, 12 Jun 2024 14:22:36 -0400 Subject: [PATCH 2/3] using container with hdf, hfd5, hdfeos, rttov use make -j 4 for compiling case select for mkmf.template (mkmf.templates are in container) Note github replaces HOME with github/home so you can not use ~ in the containter --- .github/workflows/run_all_quickbuilds.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_all_quickbuilds.yml b/.github/workflows/run_all_quickbuilds.yml index 95af60b500..3b853d731e 100644 --- a/.github/workflows/run_all_quickbuilds.yml +++ b/.github/workflows/run_all_quickbuilds.yml @@ -6,7 +6,7 @@ jobs: setup: runs-on: ubuntu-latest container: - image: hkershaw/dart-dep:1.0 + image: hkershaw/dart-dep-external:2.0 options: "--cap-add=SYS_PTRACE" outputs: matrix: ${{ steps.matrix.outputs.value }} @@ -37,7 +37,7 @@ jobs: needs: [ setup ] runs-on: ubuntu-latest container: - image: hkershaw/dart-dep:1.0 + image: hkershaw/dart-dep-external:2.0 options: "--cap-add=SYS_PTRACE" strategy: fail-fast: false @@ -51,8 +51,22 @@ jobs: - name: Creating Makefile template run: | cd build_templates - cp mkmf.template.gfortran mkmf.template - echo 'FFLAGS = -g -Wuninitialized -Wunused -ffree-line-length-none -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow $(INCS)' >> mkmf.template + sed -i 's|exec '\''make'\'', '\''-f'\'', \$opt_m if \$opt_x;|exec '\''make'\'', '\''-j'\'', '\''4'\'', '\''-f'\'', \$opt_m if \$opt_x;|' mkmf + case ${{ matrix.value }} in + *quikscat*) + cp /home/mkmf.template.quikscat.gfortran mkmf.template + ;; + *AIRS*) + cp /home/mkmf.template.AIRS.gfortran mkmf.template + ;; + *GMI* | *GOES* | *forward_operators* | *NSIDC* ) + cp /home/mkmf.template.rttov.gfortran mkmf.template + ;; + *) + cp mkmf.template.gfortran mkmf.template + echo 'FFLAGS = -g -Wuninitialized -Wunused -ffree-line-length-none -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow $(INCS)' >> mkmf.template + ;; + esac shell: bash - name: Run quickbuild.sh script run: | From 21399b953b12c444214436bc37a617ee5b2d809e Mon Sep 17 00:00:00 2001 From: Helen Kershaw Date: Fri, 5 Jul 2024 12:15:13 -0400 Subject: [PATCH 3/3] skipping known failures using -fallow-argument-mismatch for GSI2DART. This code is using its own mpi calls not mpi_utilities --- .github/workflows/run_all_quickbuilds.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_all_quickbuilds.yml b/.github/workflows/run_all_quickbuilds.yml index 3b853d731e..a153aa0064 100644 --- a/.github/workflows/run_all_quickbuilds.yml +++ b/.github/workflows/run_all_quickbuilds.yml @@ -22,6 +22,12 @@ jobs: if [[ $item == *"template"* ]]; then continue # template model not for compiling fi + if [[ $item == *"/var/"* ]] || [[ $item == *"/gps/"* ]]; then + # skipping these converters, out of the box quickbuild.sh fails + # var needs wrf files + # gps need prepbuffr bufrlib.a to be built + continue + fi # Append the quoted item and a comma to the string stripped_item=${item#/__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/} # removes /__w/DART/DART echo $stripped_item @@ -53,7 +59,7 @@ jobs: cd build_templates sed -i 's|exec '\''make'\'', '\''-f'\'', \$opt_m if \$opt_x;|exec '\''make'\'', '\''-j'\'', '\''4'\'', '\''-f'\'', \$opt_m if \$opt_x;|' mkmf case ${{ matrix.value }} in - *quikscat*) + *quikscat* | *GSI2DART* ) cp /home/mkmf.template.quikscat.gfortran mkmf.template ;; *AIRS*)