From 0cd476c5eb4b96a1476eaf4ce1b498f5bd405139 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 23 May 2024 16:56:10 -0400 Subject: [PATCH 01/14] changes to work with ufs-dev PR#184 --- ccpp/config/ccpp_prebuild_config.py | 1 + ccpp/framework | 2 +- ccpp/physics | 2 +- scm/src/CMakeLists.txt | 1 + scm/src/GFS_typedefs.F90 | 39 ++++++++++------------------- scm/src/GFS_typedefs.meta | 14 ++++++++--- 6 files changed, 27 insertions(+), 32 deletions(-) diff --git a/ccpp/config/ccpp_prebuild_config.py b/ccpp/config/ccpp_prebuild_config.py index 41736151a..d68aaf98b 100755 --- a/ccpp/config/ccpp_prebuild_config.py +++ b/ccpp/config/ccpp_prebuild_config.py @@ -33,6 +33,7 @@ TYPEDEFS_NEW_METADATA = { 'ccpp_types' : { 'ccpp_types' : '', + 'MPI_Comm' : '', 'ccpp_t' : 'cdata', }, 'machine' : { diff --git a/ccpp/framework b/ccpp/framework index f0b9a18b0..011db4f80 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit f0b9a18b005d950cb9b0038fbc827b6b37500f43 +Subproject commit 011db4f80a02cba6d65958ace56e8efb197be62b diff --git a/ccpp/physics b/ccpp/physics index 6d8fccbea..a3760b8b8 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 6d8fccbea12c11387c0bc457dcb3855574cd29aa +Subproject commit a3760b8b8f4d7c7e35503283d90f5836573e665c diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 2ef20bb84..fdbcd4661 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -257,6 +257,7 @@ endif() # How about using proper compile targets etc? #------------------------------------------------------------------------------ # Configure sources +set(MPI ON) ADD_SUBDIRECTORY(${CCPP_FRAMEWORK_SRC} ${CMAKE_BINARY_DIR}/ccpp/framework) ADD_SUBDIRECTORY(${CCPP_PHYSICS_SRC} ${CMAKE_BINARY_DIR}/ccpp/physics) ADD_DEPENDENCIES(ccpp_physics ccpp_framework) diff --git a/scm/src/GFS_typedefs.F90 b/scm/src/GFS_typedefs.F90 index cf4a6db0c..51e6ae1ed 100644 --- a/scm/src/GFS_typedefs.F90 +++ b/scm/src/GFS_typedefs.F90 @@ -1,5 +1,6 @@ module GFS_typedefs + use mpi_f08 use machine, only: kind_phys, kind_dbl_prec, kind_sngl_prec use module_radsw_parameters, only: topfsw_type, sfcfsw_type @@ -85,7 +86,7 @@ module GFS_typedefs type GFS_init_type integer :: me !< my MPI-rank integer :: master !< master MPI-rank - integer :: fcst_mpi_comm !< forecast tasks mpi communicator + type(MPI_Comm) :: fcst_mpi_comm !< forecast tasks mpi communicator integer :: fcst_ntasks !< total number of forecast tasks integer :: tile_num !< tile number for this MPI rank integer :: isc !< starting i-index for this MPI-domain @@ -693,7 +694,7 @@ module GFS_typedefs integer :: me !< MPI rank designator integer :: master !< MPI rank of master atmosphere processor - integer :: communicator !< MPI communicator + type(MPI_Comm) :: communicator !< MPI communicator integer :: ntasks !< MPI size in communicator integer :: nthreads !< OpenMP threads available for physics integer :: nlunit !< unit for namelist @@ -1153,6 +1154,7 @@ module GFS_typedefs logical :: lheatstrg !< flag for canopy heat storage parameterization logical :: lseaspray !< flag for sea spray parameterization logical :: cnvcld + logical :: xr_cnvcld !< flag for adding suspended convective clouds to Xu-Randall cloud fraction logical :: random_clds !< flag controls whether clouds are random logical :: shal_cnv !< flag for calling shallow convection logical :: do_deep !< whether to do deep convection @@ -2582,7 +2584,6 @@ subroutine sfcprop_create (Sfcprop, IM, Model) allocate(Sfcprop%iceprv (IM)) allocate(Sfcprop%snowprv (IM)) allocate(Sfcprop%graupelprv(IM)) - Sfcprop%iceprv = clear_val Sfcprop%snowprv = clear_val Sfcprop%graupelprv = clear_val @@ -3297,7 +3298,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys), dimension(:), intent(in) :: bk logical, intent(in) :: restart logical, intent(in) :: hydrostatic - integer, intent(in) :: communicator + type(MPI_Comm), intent(in) :: communicator integer, intent(in) :: ntasks integer, intent(in) :: nthreads @@ -3307,9 +3308,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & integer :: seed0 logical :: exists real(kind=kind_phys) :: tem - real(kind=kind_phys) :: rinc(5) - real(kind=kind_sngl_prec) :: rinc4(5) - real(kind=kind_dbl_prec) :: rinc8(5) + real(kind=kind_dbl_prec) :: rinc(5) real(kind=kind_phys) :: wrk(1) real(kind=kind_phys), parameter :: con_hr = 3600. @@ -3673,6 +3672,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & logical :: lheatstrg = .false. !< flag for canopy heat storage parameterization logical :: lseaspray = .false. !< flag for sea spray parameterization logical :: cnvcld = .false. + logical :: xr_cnvcld = .true. !< flag for including suspended convective clouds in Xu-Randall cloud fraction logical :: random_clds = .false. !< flag controls whether clouds are random logical :: shal_cnv = .false. !< flag for calling shallow convection integer :: imfshalcnv = 1 !< flag for mass-flux shallow convection scheme @@ -3979,7 +3979,6 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys) :: radar_tten_limits(2) = (/ limit_unspecified, limit_unspecified /) integer :: itime - integer :: w3kindreal,w3kindint !--- END NAMELIST VARIABLES @@ -4069,8 +4068,8 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & hwrf_samfdeep, hwrf_samfshal,progsigma,betascu,betamcu, & betadcu,h2o_phys, pdfcld, shcnvcw, redrag, hybedmf, satmedmf,& shinhong, do_ysu, dspheat, lheatstrg, lseaspray, cnvcld, & - random_clds, shal_cnv, imfshalcnv, imfdeepcnv, isatmedmf, & - do_deep, jcap, & + xr_cnvcld, random_clds, shal_cnv, imfshalcnv, imfdeepcnv, & + isatmedmf, do_deep, jcap, & cs_parm, flgmin, cgwf, ccwf, cdmbgwd, sup, ctei_rm, crtrh, & dlqf, rbcr, shoc_parm, psauras, prauras, wminras, & do_sppt, do_shum, do_skeb, & @@ -4949,6 +4948,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%lheatstrg = lheatstrg Model%lseaspray = lseaspray Model%cnvcld = cnvcld + Model%xr_cnvcld = xr_cnvcld Model%random_clds = random_clds Model%shal_cnv = shal_cnv Model%imfshalcnv = imfshalcnv @@ -5664,19 +5664,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%cdec = -9999. Model%clstp = -9999 rinc(1:5) = 0 - call w3kind(w3kindreal,w3kindint) - if (w3kindreal == 8) then - rinc8(1:5) = 0 - call w3difdat(jdat,idat,4,rinc8) - rinc = rinc8 - else if (w3kindreal == 4) then - rinc4(1:5) = 0 - call w3difdat(jdat,idat,4,rinc4) - rinc = rinc4 - else - write(0,*)' FATAL ERROR: Invalid w3kindreal' - call abort - endif + call w3difdat(jdat,idat,4,rinc) Model%phour = rinc(4)/con_hr Model%fhour = (rinc(4) + Model%dtp)/con_hr Model%zhour = mod(Model%phour,Model%fhzero) @@ -6251,7 +6239,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & ' do_shoc=', Model%do_shoc, ' nshoc3d=', Model%nshoc_3d, & ' nshoc_2d=', Model%nshoc_2d, ' shoc_cld=', Model%shoc_cld, & ' nkbfshoc=', Model%nkbfshoc, ' nahdshoc=', Model%nahdshoc, & - ' nscfshoc=', Model%nscfshoc, & + ' nscfshoc=', Model%nscfshoc, ' xr_cnvcld=',Model%xr_cnvcld, & ' uni_cld=', Model%uni_cld, & ' ntot3d=', Model%ntot3d, ' ntot2d=', Model%ntot2d, & ' shocaftcnv=',Model%shocaftcnv,' indcld=', Model%indcld, & @@ -6474,7 +6462,7 @@ subroutine control_print(Model) print *, 'basic control parameters' print *, ' me : ', Model%me print *, ' master : ', Model%master - print *, ' communicator : ', Model%communicator + print *, ' communicator : ', Model%communicator%mpi_val print *, ' nlunit : ', Model%nlunit print *, ' fn_nml : ', trim(Model%fn_nml) print *, ' fhzero : ', Model%fhzero @@ -7192,7 +7180,6 @@ subroutine tbd_create (Tbd, IM, Model) allocate (Tbd%h2opl (IM,levh2o,h2o_coeff)) Tbd%h2opl = clear_val Tbd%ozpl = clear_val - !--- ccn and in needs ! DH* allocate only for MG? *DH diff --git a/scm/src/GFS_typedefs.meta b/scm/src/GFS_typedefs.meta index 02f58b37a..2b2d485bf 100644 --- a/scm/src/GFS_typedefs.meta +++ b/scm/src/GFS_typedefs.meta @@ -3346,7 +3346,7 @@ long_name = MPI communicator units = index dimensions = () - type = integer + type = MPI_Comm [ntasks] standard_name = number_of_mpi_tasks long_name = number of MPI tasks in communicator @@ -5424,6 +5424,12 @@ units = flag dimensions = () type = logical +[xr_cnvcld] + standard_name = flag_for_suspended_convective_clouds_in_Xu_Randall + long_name = flag for using suspended convective clouds in Xu Randall + units = flag + dimensions = () + type = logical [shal_cnv] standard_name = flag_for_simplified_arakawa_schubert_shallow_convection long_name = flag for calling shallow convection @@ -10047,7 +10053,7 @@ [ltg1_max] standard_name = lightning_threat_index_1 long_name = lightning threat index 1 - units = flashes 5 min-1 + units = flashes min-1 dimensions = (horizontal_loop_extent) type = real kind = kind_phys @@ -10056,7 +10062,7 @@ [ltg2_max] standard_name = lightning_threat_index_2 long_name = lightning threat index 2 - units = flashes 5 min-1 + units = flashes min-1 dimensions = (horizontal_loop_extent) type = real kind = kind_phys @@ -10065,7 +10071,7 @@ [ltg3_max] standard_name = lightning_threat_index_3 long_name = lightning threat index 3 - units = flashes 5 min-1 + units = flashes min-1 dimensions = (horizontal_loop_extent) type = real kind = kind_phys From 37f627267411183fcccd2c612dbf4cdcfacdfd02 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Tue, 28 May 2024 19:23:54 +0000 Subject: [PATCH 02/14] add initial working dummy MPI statements to SCM code --- scm/src/CMakeLists.txt | 5 ++++- scm/src/run_scm.py | 2 +- scm/src/scm.F90 | 18 ++++++++++++++++-- scm/src/scm_setup.F90 | 5 ++--- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index fdbcd4661..e8e8f7bbb 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -105,6 +105,9 @@ ADD_DEFINITIONS(-DCCPP) # Add host-model specific preprocessor flag (needed for some physics schemes) ADD_DEFINITIONS(-DSCM) +ADD_DEFINITIONS(-DMPI) + + #------------------------------------------------------------------------------ # Add model-specific flags for C/C++/Fortran preprocessor ADD_DEFINITIONS(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM) @@ -257,7 +260,7 @@ endif() # How about using proper compile targets etc? #------------------------------------------------------------------------------ # Configure sources -set(MPI ON) + ADD_SUBDIRECTORY(${CCPP_FRAMEWORK_SRC} ${CMAKE_BINARY_DIR}/ccpp/framework) ADD_SUBDIRECTORY(${CCPP_PHYSICS_SRC} ${CMAKE_BINARY_DIR}/ccpp/physics) ADD_DEPENDENCIES(ccpp_physics ccpp_framework) diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 926fb68ba..2f1db97c3 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -720,7 +720,7 @@ def launch_executable(use_gdb, gdb, ignore_error = False): if use_gdb: cmd = '(cd {scm_run} && {gdb} {executable})'.format(scm_run=SCM_RUN, gdb=gdb, executable=EXECUTABLE) else: - cmd = '(cd {scm_run} && time {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE) + cmd = '(cd {scm_run} && time srun -A gmtb -n 1 {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE) logging.info('Passing control to "{0}"'.format(cmd)) time.sleep(1) # This will abort in 'execute' in the event of an error if ignore_error = False diff --git a/scm/src/scm.F90 b/scm/src/scm.F90 index ba152f1c1..bdc066f7d 100644 --- a/scm/src/scm.F90 +++ b/scm/src/scm.F90 @@ -15,7 +15,8 @@ subroutine scm_main_sub() use scm_time_integration use scm_output use scm_type_defs - + use mpi_f08 + use :: ccpp_static_api, & only: ccpp_physics_init, & ccpp_physics_timestep_init, & @@ -30,6 +31,8 @@ subroutine scm_main_sub() type(scm_input_type), target :: scm_input_instance type(scm_reference_type), target :: scm_reference + type(MPI_Comm) :: fcst_mpi_comm + integer :: i, j, kdt_rad, idtend, itrac real(kind=8) :: rinc(5) !(DAYS, HOURS, MINUTES, SECONDS, MILLISECONDS) integer :: jdat(1:8) @@ -39,6 +42,11 @@ subroutine scm_main_sub() character(len=16) :: logfile_name logical :: in_spinup +#ifdef MPI + call MPI_INIT(ierr) + fcst_mpi_comm = MPI_COMM_WORLD +#endif + call get_config_nml(scm_state) select case(scm_state%input_type) @@ -94,6 +102,7 @@ subroutine scm_main_sub() open(unit=physics%Init_parm%logunit, file=trim(scm_state%output_dir)//'/'//logfile_name, action='write', status='replace') end if + physics%Init_parm%fcst_mpi_comm = fcst_mpi_comm physics%Init_parm%levs = scm_state%n_levels physics%Init_parm%bdat(1) = scm_state%init_year physics%Init_parm%bdat(2) = scm_state%init_month @@ -120,7 +129,7 @@ subroutine scm_main_sub() call GFS_suite_setup(physics%Model, physics%Statein, physics%Stateout, & physics%Sfcprop, physics%Coupling, physics%Grid, & physics%Tbd, physics%Cldprop, physics%Radtend, & - physics%Diag, physics%Interstitial, 0, 1, 1, & + physics%Diag, physics%Interstitial, 1, 1, & physics%Init_parm, scm_state%n_cols, scm_state%lon, & scm_state%lat, scm_state%area) @@ -429,5 +438,10 @@ end module scm_main !! subroutine \ref scm_main_sub above. program scm use scm_main + +#ifdef MPI + use mpi_f08 +#endif + call scm_main_sub() end program scm diff --git a/scm/src/scm_setup.F90 b/scm/src/scm_setup.F90 index 30ebf3d47..5f4423e29 100644 --- a/scm/src/scm_setup.F90 +++ b/scm/src/scm_setup.F90 @@ -279,7 +279,7 @@ end subroutine patch_in_ref !-------------- subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop, & Coupling, Grid, Tbd, Cldprop, Radtend, Diag, & - Interstitial, communicator, ntasks, nthreads, & + Interstitial, ntasks, nthreads, & Init_parm, n_cols, lon, lat, area) use machine, only: kind_phys @@ -311,7 +311,6 @@ subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop, type(GFS_interstitial_type), intent(inout) :: Interstitial type(GFS_init_type), intent(in) :: Init_parm - integer, intent(in) :: communicator integer, intent(in) :: ntasks, nthreads, n_cols real(kind=dp), dimension(n_cols), intent(in) :: lon, lat, area @@ -334,7 +333,7 @@ subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop, Init_parm%input_nml_file, Init_parm%tile_num, & Init_parm%blksz, Init_parm%ak, Init_parm%bk, & Init_parm%restart, Init_parm%hydrostatic, & - communicator, ntasks, nthreads) + Init_parm%fcst_mpi_comm, ntasks, nthreads) !--- initialize DDTs From 100f9212c247f820cc8c7bc42b15ec7079daef3a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:07:04 +0000 Subject: [PATCH 03/14] Changes fro Grants branch --- .../workflows/ci_build_scm_ubuntu_22.04.yml | 43 ++++++------------- .github/workflows/ci_run_scm_rts.yml | 26 +++++------ scm/src/run_scm.py | 2 +- 3 files changed, 29 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index 6c33a30e5..a5ccb6177 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -21,8 +21,9 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - intel_ROOT: /home/runner/intel suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0,SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps + CC: mpicc + FC: mpif90 # Workflow steps steps: @@ -57,34 +58,6 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - - name: Environment for ifort compiler - if: contains(matrix.fortran-compiler, 'ifort') - run: | - echo "CC=icx" >> $GITHUB_ENV - echo "FC=ifort" >> $GITHUB_ENV - echo "NFVERSION=v4.4.4" >> $GITHUB_ENV - - - name: Cache Intel compilers - id: cache-intel-compilers - if: contains(matrix.fortran-compiler, 'ifort') - uses: actions/cache@v2 - with: - path: /home/runner/intel - key: intel-${{ runner.os }}-compilers-b - - # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-oneapi-toolkits-via-apt.html - # List of packages from Docker file at - # https://github.com/intel/oneapi-containers/blob/master/images/docker/hpckit-devel-ubuntu18.04/Dockerfile - - name: Install Intel compilers and libraries - if: contains(matrix.fortran-compiler, 'ifort') && steps.cache-intel-compilers.outputs.cache-hit != 'true' - run: | - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" - sudo apt-get update - sudo apt-get install intel-hpckit-getting-started intel-oneapi-clck intel-oneapi-common-licensing intel-oneapi-common-vars - sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-itac - - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -141,6 +114,18 @@ jobs: sudo apt-get update sudo apt-get install libnetcdf-dev + - name: Install open mpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache NetCDF Fortran library id: cache-netcdf-fortran uses: actions/cache@v3 diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index f4096681d..f03399439 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -25,6 +25,8 @@ jobs: suites_ps: SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps,SCM_RRFS_v1_ps,SCM_GFS_v17_HR3_ps,SCM_GFS_v17_HR3_RRTMGP_ps dir_rt: /home/runner/work/ccpp-scm/ccpp-scm/test/artifact-${{matrix.build-type}} dir_bl: /home/runner/work/ccpp-scm/ccpp-scm/test/BL-${{matrix.build-type}} + CC: mpicc + FC: mpif90 # Workflow steps steps: @@ -59,18 +61,6 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - - name: Environment for gfortran compiler - if: contains(matrix.fortran-compiler, 'gnu') - run: | - echo "FC=gfortran-11" >> $GITHUB_ENV - echo "CC=gcc-11" >> $GITHUB_ENV - - - name: Environment for ifort compiler - if: contains(matrix.fortran-compiler, 'intel') - run: | - echo "CC=icx" >> $GITHUB_ENV - echo "FC=ifort" >> $GITHUB_ENV - - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -125,6 +115,18 @@ jobs: - name: Install NetCDF C library run: sudo apt-get install libnetcdf-dev + - name: Install open mpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache NetCDF Fortran library id: cache-netcdf-fortran uses: actions/cache@v3 diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 2f1db97c3..773b89054 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -720,7 +720,7 @@ def launch_executable(use_gdb, gdb, ignore_error = False): if use_gdb: cmd = '(cd {scm_run} && {gdb} {executable})'.format(scm_run=SCM_RUN, gdb=gdb, executable=EXECUTABLE) else: - cmd = '(cd {scm_run} && time srun -A gmtb -n 1 {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE) + cmd = '(cd {scm_run} && time mpirun -n 1 {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE) logging.info('Passing control to "{0}"'.format(cmd)) time.sleep(1) # This will abort in 'execute' in the event of an error if ignore_error = False From de62a3db5910320319b0f1f54a799571aff5577e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:13:09 +0000 Subject: [PATCH 04/14] Move openmpi build up in CI workflow --- .../workflows/ci_build_scm_ubuntu_22.04.yml | 24 +++++++++---------- .github/workflows/ci_run_scm_rts.yml | 24 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index a5ccb6177..ed2f8f884 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -58,6 +58,18 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -114,18 +126,6 @@ jobs: sudo apt-get update sudo apt-get install libnetcdf-dev - - name: Install open mpi - run: | - wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz - tar -xvf v4.1.6.tar.gz - cd ompi-4.1.6 - ./autogen.pl - ./configure --prefix=/home/runner/ompi-4.1.6 - make -j4 - make install - echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - - name: Cache NetCDF Fortran library id: cache-netcdf-fortran uses: actions/cache@v3 diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index f03399439..996bb680b 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -61,6 +61,18 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -115,18 +127,6 @@ jobs: - name: Install NetCDF C library run: sudo apt-get install libnetcdf-dev - - name: Install open mpi - run: | - wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz - tar -xvf v4.1.6.tar.gz - cd ompi-4.1.6 - ./autogen.pl - ./configure --prefix=/home/runner/ompi-4.1.6 - make -j4 - make install - echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - - name: Cache NetCDF Fortran library id: cache-netcdf-fortran uses: actions/cache@v3 From 9ffcea06bb3fe775a593df77ba274066c13aa5f4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:22:12 +0000 Subject: [PATCH 05/14] CI changes --- .github/workflows/ci_build_scm_ubuntu_22.04.yml | 7 +++++-- .github/workflows/ci_run_scm_rts.yml | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index ed2f8f884..f2ad6421c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -22,8 +22,6 @@ jobs: w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0,SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps - CC: mpicc - FC: mpif90 # Workflow steps steps: @@ -143,6 +141,11 @@ jobs: sudo make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + ####################################################################################### # Build SCM. ####################################################################################### diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index 996bb680b..aab18a355 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -25,8 +25,6 @@ jobs: suites_ps: SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps,SCM_RRFS_v1_ps,SCM_GFS_v17_HR3_ps,SCM_GFS_v17_HR3_RRTMGP_ps dir_rt: /home/runner/work/ccpp-scm/ccpp-scm/test/artifact-${{matrix.build-type}} dir_bl: /home/runner/work/ccpp-scm/ccpp-scm/test/BL-${{matrix.build-type}} - CC: mpicc - FC: mpif90 # Workflow steps steps: @@ -61,6 +59,7 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi run: | wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz @@ -144,6 +143,11 @@ jobs: sudo make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + ####################################################################################### # Build SCM. Run regression tests. ####################################################################################### From ed1e4d81549cab92b3e764518b555a6130f4676b Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:31:09 +0000 Subject: [PATCH 06/14] CI changes --- .github/workflows/ci_run_scm_DEPHY.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci_run_scm_DEPHY.yml b/.github/workflows/ci_run_scm_DEPHY.yml index eeba4c7fe..326eeb75e 100644 --- a/.github/workflows/ci_run_scm_DEPHY.yml +++ b/.github/workflows/ci_run_scm_DEPHY.yml @@ -54,6 +54,18 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -128,6 +140,11 @@ jobs: ####################################################################################### # Build SCM. Run DEPHYv1 case. ####################################################################################### + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + - name: Download data for SCM run: | cd ${SCM_ROOT} From dcb2e5f1aab65634e37f8f5b2fe605d9ec2bed2c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 21:48:56 +0000 Subject: [PATCH 07/14] CI changes --- .github/workflows/ci_build_scm_ubuntu_22.04.yml | 2 +- .github/workflows/ci_run_scm_rts.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index f2ad6421c..4a57f71a7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -67,7 +67,7 @@ jobs: make install echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index aab18a355..3176274f7 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -147,7 +147,7 @@ jobs: run: | echo "FC=mpif90" >> $GITHUB_ENV echo "CC=mpicc" >> $GITHUB_ENV - + ####################################################################################### # Build SCM. Run regression tests. ####################################################################################### From e54a78adc1d342044e44ce7a586792f671db147c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 29 May 2024 22:09:35 +0000 Subject: [PATCH 08/14] CI changes --- .github/workflows/ci_build_scm_ubuntu_22.04.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index 4a57f71a7..e89dcb50c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -145,7 +145,7 @@ jobs: run: | echo "FC=mpif90" >> $GITHUB_ENV echo "CC=mpicc" >> $GITHUB_ENV - + ####################################################################################### # Build SCM. ####################################################################################### From 0958467c34518a1e8211d2d761df7f0a8fcb8939 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 30 May 2024 14:15:42 -0400 Subject: [PATCH 09/14] remove ifdefs from SCM code since MPI is required now; add MPI_FINALIZE; check for errors for MPI_INIT/FINALIZE --- scm/src/scm.F90 | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scm/src/scm.F90 b/scm/src/scm.F90 index bdc066f7d..d6abe399e 100644 --- a/scm/src/scm.F90 +++ b/scm/src/scm.F90 @@ -42,10 +42,12 @@ subroutine scm_main_sub() character(len=16) :: logfile_name logical :: in_spinup -#ifdef MPI call MPI_INIT(ierr) - fcst_mpi_comm = MPI_COMM_WORLD -#endif + if (ierr/=0) then + write(*,'(a,i0,a)') 'An error occurred in MPI_INIT: ' // ierr // '. Exiting...' + stop 1 + end if + fcst_mpi_comm = MPI_COMM_WORLD call get_config_nml(scm_state) @@ -427,6 +429,12 @@ subroutine scm_main_sub() write(*,'(a,i0,a)') 'An error occurred in ccpp_physics_finalize: ' // trim(cdata%errmsg) // '. Exiting...' stop 1 end if + + call MPI_FINALIZE(ierr) + if (ierr/=0) then + write(*,'(a,i0,a)') 'An error occurred in MPI_FINALIZE: ' // ierr // '. Exiting...' + stop 1 + end if end subroutine scm_main_sub @@ -438,10 +446,7 @@ end module scm_main !! subroutine \ref scm_main_sub above. program scm use scm_main - -#ifdef MPI use mpi_f08 -#endif call scm_main_sub() end program scm From 356e4ee5037be0809f177380f1765e6a32743424 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 30 May 2024 14:33:27 -0400 Subject: [PATCH 10/14] fix compilation error for MPI_INIT/FINALIZE error messages --- scm/src/scm.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scm/src/scm.F90 b/scm/src/scm.F90 index d6abe399e..9a05e974a 100644 --- a/scm/src/scm.F90 +++ b/scm/src/scm.F90 @@ -44,7 +44,7 @@ subroutine scm_main_sub() call MPI_INIT(ierr) if (ierr/=0) then - write(*,'(a,i0,a)') 'An error occurred in MPI_INIT: ' // ierr // '. Exiting...' + write(*,*) 'An error occurred in MPI_INIT: ', ierr stop 1 end if fcst_mpi_comm = MPI_COMM_WORLD @@ -432,7 +432,7 @@ subroutine scm_main_sub() call MPI_FINALIZE(ierr) if (ierr/=0) then - write(*,'(a,i0,a)') 'An error occurred in MPI_FINALIZE: ' // ierr // '. Exiting...' + write(*,*) 'An error occurred in MPI_FINALIZE: ', ierr stop 1 end if From 81296f293a438d59b1300938ae28ee9e16c2dab6 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 30 May 2024 16:20:46 -0400 Subject: [PATCH 11/14] add mpi_command command line argument --- scm/src/run_scm.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 773b89054..11d6b7c0c 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -31,6 +31,9 @@ # Path to default bin directory (relative to scm_root) DEFAULT_BIN_DIR = 'scm/bin' +# Default command string to run MPI apps (number of processes should be 1 since SCM is not set up to use more than 1 yet) +DEFAULT_MPI_COMMAND = 'mpirun -np 1' + # Copy executable to run directory if true (otherwise it will be linked) COPY_EXECUTABLE = False @@ -119,6 +122,7 @@ parser.add_argument('-dt', '--timestep', help='timestep (s)', required=False, type=float) parser.add_argument('-v', '--verbose', help='set logging level to debug and write log to file', action='count', default=0) parser.add_argument('-f', '--file', help='name of file where SCM runs are defined') +parser.add_argument('--mpi_command', help='command used to invoke the executable via MPI (including options)', required=False) ############################################################################### # Functions and subroutines # @@ -185,13 +189,14 @@ def parse_arguments(): run_dir = args.run_dir bin_dir = args.bin_dir timestep = args.timestep + mpi_command = args.mpi_command if not sdf: sdf = DEFAULT_SUITE return (file, case, sdf, namelist, tracers, gdb, runtime, runtime_mult, docker, \ verbose, levels, npz_type, vert_coord_file, case_data_dir, n_itt_out, \ - n_itt_diag, run_dir, bin_dir, timestep) + n_itt_diag, run_dir, bin_dir, timestep, mpi_command) def find_gdb(): """Detect gdb, abort if not found""" @@ -715,12 +720,16 @@ def setup_rundir(self): return os.path.join(SCM_RUN, output_dir) -def launch_executable(use_gdb, gdb, ignore_error = False): +def launch_executable(use_gdb, gdb, ignore_error = False, mpi_command): """Configure model run command and pass control to shell/gdb""" if use_gdb: - cmd = '(cd {scm_run} && {gdb} {executable})'.format(scm_run=SCM_RUN, gdb=gdb, executable=EXECUTABLE) + if not mpi_command: + mpi_command = DEFAULT_MPI_COMMAND + ' xterm -e ' + cmd = '(cd {scm_run} && {mpi_command} {gdb} {executable})'.format(scm_run=SCM_RUN, mpi_command=mpi_command, gdb=gdb, executable=EXECUTABLE) else: - cmd = '(cd {scm_run} && time mpirun -n 1 {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE) + if not mpi_command: + mpi_command = DEFAULT_MPI_COMMAND + cmd = '(cd {scm_run} && time {mpi_command} {executable})'.format(scm_run=SCM_RUN, mpi_command=mpi_command, executable=EXECUTABLE) logging.info('Passing control to "{0}"'.format(cmd)) time.sleep(1) # This will abort in 'execute' in the event of an error if ignore_error = False @@ -756,7 +765,7 @@ def copy_outdir(exp_dir): def main(): (file, case, sdf, namelist, tracers, use_gdb, runtime, runtime_mult, docker, \ verbose, levels, npz_type, vert_coord_file, case_data_dir, n_itt_out, \ - n_itt_diag, run_dir, bin_dir, timestep) = parse_arguments() + n_itt_diag, run_dir, bin_dir, timestep, mpi_command) = parse_arguments() setup_logging(verbose) @@ -873,7 +882,7 @@ def main(): l_ignore_error = MULTIRUN_IGNORE_ERROR else: l_ignore_error = False - (status, time_elapsed) = launch_executable(use_gdb, gdb, ignore_error = l_ignore_error) + (status, time_elapsed) = launch_executable(use_gdb, gdb, ignore_error = l_ignore_error, mpi_command) # if status == 0: logging.info('Process "(case={0}, suite={1}, namelist={2}" completed successfully'. \ From 7a695dd74a8c2f7cb55df7875b00b60c3d28a35c Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 30 May 2024 21:02:24 +0000 Subject: [PATCH 12/14] swap launch_exe arugment order --- scm/src/run_scm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 11d6b7c0c..b499fafac 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -720,7 +720,7 @@ def setup_rundir(self): return os.path.join(SCM_RUN, output_dir) -def launch_executable(use_gdb, gdb, ignore_error = False, mpi_command): +def launch_executable(use_gdb, gdb, mpi_command, ignore_error = False): """Configure model run command and pass control to shell/gdb""" if use_gdb: if not mpi_command: @@ -882,7 +882,7 @@ def main(): l_ignore_error = MULTIRUN_IGNORE_ERROR else: l_ignore_error = False - (status, time_elapsed) = launch_executable(use_gdb, gdb, ignore_error = l_ignore_error, mpi_command) + (status, time_elapsed) = launch_executable(use_gdb, gdb, mpi_command, ignore_error = l_ignore_error) # if status == 0: logging.info('Process "(case={0}, suite={1}, namelist={2}" completed successfully'. \ From 4e1ad0788c2186889cea73e97e2b4f7b2c0191f0 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Fri, 31 May 2024 10:06:55 -0400 Subject: [PATCH 13/14] change len to shape for numpy array checking --- test/plot_scm_out.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plot_scm_out.py b/test/plot_scm_out.py index d8474071e..2c2d50ea8 100755 --- a/test/plot_scm_out.py +++ b/test/plot_scm_out.py @@ -82,7 +82,7 @@ def plot_results(file_bl, file_rt=None, vars2plt=None): if file_rt is not None: x2 = SCM_RT[timeD][:].squeeze()/3600. #seconds - >hours # If temporal dimensions disagree, con't compute deltas from experiments, turn off difference plots. - if (len(x1) != len(x2)): + if (x1.shape != x2.shape): plot_diff = False # end if # end if From 5dee89c1e141bd7b6579c5cdf7764d3b6f6290d2 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Tue, 4 Jun 2024 13:34:37 -0400 Subject: [PATCH 14/14] update ccpp/physics submodule pointer --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index a3760b8b8..76cf93fc2 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit a3760b8b8f4d7c7e35503283d90f5836573e665c +Subproject commit 76cf93fc26d7b843f6d89ed0ba4799fd5f2c21f7