diff --git a/SingleModelOpenMPProto/driver.F90 b/SingleModelOpenMPProto/driver.F90 index 0b58f118..eb51fdcd 100644 --- a/SingleModelOpenMPProto/driver.F90 +++ b/SingleModelOpenMPProto/driver.F90 @@ -98,19 +98,16 @@ subroutine SetModelServices(driver, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#if 1 call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & rc=rc) ! expect 2 PEs per PET in the child VM if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out - call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MinStackSize", & - value=16*1024*1024, rc=rc) ! stack size no less than 16MiB - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out -#if 1 +#endif + +#if 0 call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/OpenMpHandling", & value="SET", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -118,7 +115,7 @@ subroutine SetModelServices(driver, rc) file=__FILE__)) & return ! bail out #endif -#if 1 +#if 0 call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/OpenMpNumThreads", & value=4, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & diff --git a/SingleModelOpenMPUnawareProto/Makefile b/SingleModelOpenMPUnawareProto/Makefile new file mode 100644 index 00000000..c735ee00 --- /dev/null +++ b/SingleModelOpenMPUnawareProto/Makefile @@ -0,0 +1,73 @@ +# GNU Makefile template for user ESMF application + +################################################################################ +################################################################################ +## This Makefile must be able to find the "esmf.mk" Makefile fragment in the ## +## 'include' line below. Following the ESMF User's Guide, a complete ESMF ## +## installation should ensure that a single environment variable "ESMFMKFILE" ## +## is made available on the system. This variable should point to the ## +## "esmf.mk" file. ## +## ## +## This example Makefile uses the "ESMFMKFILE" environment variable. ## +## ## +## If you notice that this Makefile cannot find variable ESMFMKFILE then ## +## please contact the person responsible for the ESMF installation on your ## +## system. ## +## As a work-around you can simply hardcode the path to "esmf.mk" in the ## +## include line below. However, doing so will render this Makefile a lot less ## +## flexible and non-portable. ## +################################################################################ + +ifneq ($(origin ESMFMKFILE), environment) +$(error Environment variable ESMFMKFILE was not set.) +endif + +include $(ESMFMKFILE) + +################################################################################ +################################################################################ + +.SUFFIXES: .f90 .F90 .c .C + +%.o : %.f90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREENOCPP) $< + +%.o : %.F90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) -DESMF_VERSION_MAJOR=$(ESMF_VERSION_MAJOR) $< + +%.o : %.c + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + +%.o : %.C + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + + +# ----------------------------------------------------------------------------- +mainApp: mainApp.o driver.o model.o + $(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -o $@ $^ $(ESMF_F90ESMFLINKLIBS) + +# module dependencies: +mainApp.o: driver.o +driver.o: model.o + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +.PHONY: dust clean distclean info edit +dust: + rm -f PET*.ESMF_LogFile *.nc *.stdout +clean: + rm -f mainApp *.o *.mod +distclean: dust clean + +info: + @echo ================================================================== + @echo ESMFMKFILE=$(ESMFMKFILE) + @echo ================================================================== + @cat $(ESMFMKFILE) + @echo ================================================================== + +edit: + nedit mainApp.F90 driver.F90 model.F90 & + +run: + env OMP_NUM_THREADS=3 mpirun -np 4 ./mainApp diff --git a/SingleModelOpenMPUnawareProto/README b/SingleModelOpenMPUnawareProto/README new file mode 100644 index 00000000..e9fc385a --- /dev/null +++ b/SingleModelOpenMPUnawareProto/README @@ -0,0 +1,45 @@ +README for single Model OpenMP Unaware NUOPC prototype +------------------------------------------------------- + +Very simple NUOPC app with a Driver controlling a single Model component. The +model component uses OpenMP if compiled with support, but does so outside of +NUOPC control. This is called NUOPC-unaware threading or resource control. + +Description: + + A single model component driven by a single driver component. + + The driver assumes that each PET launched is a master thread for its own + OpenMP thead team. NUOPC/ESMF is unaware of the OpenMP threading inside of + the model component. The number of OpenMP threads is set via the standard + OMP_NUM_THREADS environment variable, which is set equal to 3 inside the + Makefile run target. + + The ESM driver component uses the default run sequence to drive the MODEL + component. + + No connector components are present. + + The MODEL component uses a simple two-phase initialization, + consisting of advertise and realize. + + +Build: + - Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF + installation. + - gmake + +Execution: + - Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON. + - Set OMP_NUM_THREADS if directly calling into mpirun as per below. + - mpirun -np X ./esmApp (where X is the total number of PETs, typically 4) + +Output: + - PET*.Log files containing compliance checker output if turned on. + - The prototype outputs time stepping information to stdout. + +Code structure: + - Makefile - Makefile that is based on the standard esmf.mk mechanism. + - model.F90 - The MODEL component, specializing generic NUOPC_Model. + - driver.F90 - Driver specializing NUOPC_Driver, driving the single MODEL. + - mainApp.F90 - Main application. diff --git a/SingleModelOpenMPUnawareProto/driver.F90 b/SingleModelOpenMPUnawareProto/driver.F90 new file mode 100644 index 00000000..9f58bf12 --- /dev/null +++ b/SingleModelOpenMPUnawareProto/driver.F90 @@ -0,0 +1,190 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module driver + + !----------------------------------------------------------------------------- + ! Code that specializes generic NUOPC_Driver + !----------------------------------------------------------------------------- + + use MPI + use ESMF + use NUOPC + use NUOPC_Driver, & + driverSS => SetServices + + use MODEL, only: & + modelSS => SetServices + + implicit none + + private + + ! private module data --> ONLY PARAMETERS + integer, parameter :: stepCount = 5 + real(ESMF_KIND_R8), parameter :: stepTime = 30.D0 ! step time [s] + ! should be parent step + + public SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Driver + call NUOPC_CompDerive(driver, driverSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize driver + call NUOPC_CompSpecialize(driver, specLabel=label_SetModelServices, & + specRoutine=SetModelServices, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set driver verbosity + call NUOPC_CompAttributeSet(driver, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine SetModelServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + ! local variables + type(ESMF_GridComp) :: child + type(ESMF_CplComp) :: connector + type(ESMF_Time) :: startTime + type(ESMF_Time) :: stopTime + type(ESMF_TimeInterval) :: timeStep + type(ESMF_Clock) :: internalClock + + ! - diagnostics - + type(ESMF_VM) :: vm + logical :: isFlag + character(80) :: msgString + integer :: mpiComm, size, ierr + + rc = ESMF_SUCCESS + + ! SetServices for MODEL component + call NUOPC_DriverAddComp(driver, "MODEL", modelSS, comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(child, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! - diagnostics - + isFlag = ESMF_GridCompIsPetLocal(child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + write(msgString,*) "GridCompIsPetLocal: ", isFlag + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_GridCompGet(child, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + isFlag = ESMF_VMIsCreated(vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + write(msgString,*) "VmIsCreated: ", isFlag + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_VMGet(vm, mpiCommunicator=mpiComm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + if (mpiComm==MPI_COMM_NULL) then + write(msgString,*) "MPI_COMM_NULL" + else + call MPI_Comm_size(mpiComm, size, ierr) + write(msgString,*) "valid MPI_COMM with size=",size + endif + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set the driver clock + call ESMF_TimeSet(startTime, s = 0, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_TimeSet(stopTime, s_r8 = stepTime * stepCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_TimeIntervalSet(timeStep, s_r8 = stepTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + internalClock = ESMF_ClockCreate(name="Driver Clock", & + timeStep=timeStep, startTime=startTime, stopTime=stopTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_GridCompSet(driver, clock=internalClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/SingleModelOpenMPUnawareProto/mainApp.F90 b/SingleModelOpenMPUnawareProto/mainApp.F90 new file mode 100644 index 00000000..eb5ee6ff --- /dev/null +++ b/SingleModelOpenMPUnawareProto/mainApp.F90 @@ -0,0 +1,120 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +#define EXPLICIT_MPI_INIT + +program mainApp + + !----------------------------------------------------------------------------- + ! Generic ESMF Main + !----------------------------------------------------------------------------- + + use ESMF +#ifdef EXPLICIT_MPI_INIT + use MPI +#endif + + use driver, only: & + driver_SS => SetServices + + implicit none + + integer :: rc, userRc + type(ESMF_GridComp) :: drvComp + + ! Initialize MPI/ESMF +#ifdef EXPLICIT_MPI_INIT + ! This prototype implements ESMF-aware resource management for threading. + ! Therefore must call ESMF_InitializePreMPI() before MPI_Init*() if later is + ! called explicitly from user level! + call ESMF_InitializePreMPI(rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + call MPI_Init_thread(MPI_THREAD_MULTIPLE, userRc, rc) +#endif + call ESMF_Initialize(defaultCalkind=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_LogWrite("mainApp STARTING", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + !----------------------------------------------------------------------------- + + ! -> CREATE THE DRIVER + drvComp = ESMF_GridCompCreate(name="driver", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! -> SET DRIVER SERVICES + call ESMF_GridCompSetServices(drvComp, driver_SS, userRc=userRc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=userRc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! INITIALIZE THE DRIVER + call ESMF_GridCompInitialize(drvComp, userRc=userRc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=userRc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! RUN THE DRIVER + call ESMF_GridCompRun(drvComp, userRc=userRc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=userRc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! FINALIZE THE DRIVER + call ESMF_GridCompFinalize(drvComp, userRc=userRc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=userRc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + !----------------------------------------------------------------------------- + + call ESMF_LogWrite("mainApp FINISHED", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Finalize ESMF + call ESMF_Finalize() + +end program diff --git a/SingleModelOpenMPUnawareProto/model.F90 b/SingleModelOpenMPUnawareProto/model.F90 new file mode 100644 index 00000000..cc49f64b --- /dev/null +++ b/SingleModelOpenMPUnawareProto/model.F90 @@ -0,0 +1,306 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module MODEL + + !----------------------------------------------------------------------------- + ! MODEL Component. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetVM, SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize model + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_RealizeProvided, & + specRoutine=Realize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_DataInitialize, & + specRoutine=DataInitialize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! Enabeling the following macro, i.e. renaming it to WITHIMPORTFIELDS, + ! will result in a model component that advertise import Field dependencies. + ! In the single model case, where there isn't another model to satisfy these + ! dependencies, it is expected to be caught by the compatability checking. +#define WITHIMPORTFIELDS___disable +#ifdef WITHIMPORTFIELDS + ! importable field: sea_surface_temperature + call NUOPC_Advertise(importState, & + StandardName="sea_surface_temperature", name="sst", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + + ! exportable field: air_pressure_at_sea_level + call NUOPC_Advertise(exportState, & + StandardName="air_pressure_at_sea_level", name="pmsl", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: surface_net_downward_shortwave_flux + call NUOPC_Advertise(exportState, & + StandardName="surface_net_downward_shortwave_flux", name="rsns", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Realize(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + type(ESMF_Field) :: field + type(ESMF_Grid) :: gridIn + type(ESMF_Grid) :: gridOut + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create a Grid object for Fields + gridIn = ESMF_GridCreateNoPeriDimUfrm(maxIndex=(/10, 100/), & + minCornerCoord=(/10._ESMF_KIND_R8, 20._ESMF_KIND_R8/), & + maxCornerCoord=(/100._ESMF_KIND_R8, 200._ESMF_KIND_R8/), & + coordSys=ESMF_COORDSYS_CART, staggerLocList=(/ESMF_STAGGERLOC_CENTER/), & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + gridOut = gridIn ! for now out same as in + +#ifdef WITHIMPORTFIELDS + ! importable field: sea_surface_temperature + field = ESMF_FieldCreate(name="sst", grid=gridIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + + ! exportable field: air_pressure_at_sea_level + field = ESMF_FieldCreate(name="pmsl", grid=gridOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: surface_net_downward_shortwave_flux + field = ESMF_FieldCreate(name="rsns", grid=gridOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_VMLogMemInfo(prefix="After Realize:", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine DataInitialize(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + integer, save :: inHere=1 + + rc = ESMF_SUCCESS + + if (inHere > 1) then + ! indicate that data initialization is complete (breaking out of init-loop) + call NUOPC_CompAttributeSet(model, & + name="InitializeDataComplete", value="true", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + endif + + inHere = inHere + 1 + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) +!$ use omp_lib + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + type(ESMF_VM) :: vm + integer :: currentSsiPe + character(len=160) :: msgString + + rc = ESMF_SUCCESS + + ! query for clock, importState and exportState + call ESMF_GridCompGet(model, clock=clock, importState=importState, & + exportState=exportState, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! Now can use OpenMP for fine grained parallelism... + ! Here just write info about the PET-local OpenMP threads to Log. +!$omp parallel private(msgString, currentSsiPe) +!$omp critical +!$ call ESMF_VMGet(vm, currentSsiPe=currentSsiPe) +!$ write(msgString,'(A,I4,A,I4,A,I4,A,I4,A,I4)') & +!$ "thread_num=", omp_get_thread_num(), & +!$ " currentSsiPe=", currentSsiPe, & +!$ " num_threads=", omp_get_num_threads(), & +!$ " max_threads=", omp_get_max_threads(), & +!$ " num_procs=", omp_get_num_procs() +!$ call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) +!$omp end critical +!$omp end parallel + + ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep + + ! Because of the way that the internal Clock was set by default, + ! its timeStep is equal to the parent timeStep. As a consequence the + ! currTime + timeStep is equal to the stopTime of the internal Clock + ! for this call of the Advance() routine. + + call ESMF_ClockPrint(clock, options="currTime", & + preString="---->Advancing Model from: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_ClockPrint(clock, options="stopTime", & + preString="---------------------> to: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/testProtos.sh b/testProtos.sh index dab835f8..e9b80f4b 100755 --- a/testProtos.sh +++ b/testProtos.sh @@ -298,6 +298,8 @@ TestProto NestingSingleProto mainApp TestProto NestingTelescopeMultipleProto mainApp TestProto SingleModelProto mainApp TestProto SingleModelOpenMPProto mainApp +export OMP_NUM_THREADS=3 +TestProto SingleModelOpenMPUnawareProto mainApp echo "== TEST SUMMARY START ==" i=1