Skip to content

Commit

Permalink
Add SingleModelOpenMPUnawareProto.
Browse files Browse the repository at this point in the history
  • Loading branch information
@theurich
theurich committed Apr 16, 2021
1 parent a8e4029 commit 3c71ab2
Show file tree
Hide file tree
Showing 7 changed files with 741 additions and 8 deletions.
13 changes: 5 additions & 8 deletions SingleModelOpenMPProto/driver.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,27 +98,24 @@ subroutine SetModelServices(driver, rc)
line=__LINE__, &
file=__FILE__)) &
return ! bail out
#if 1
call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, &
rc=rc) ! expect 2 PEs per PET in the child VM
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out
call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MinStackSize", &
value=16*1024*1024, rc=rc) ! stack size no less than 16MiB
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out
#if 1
#endif

#if 0
call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/OpenMpHandling", &
value="SET", rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out
#endif
#if 1
#if 0
call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/OpenMpNumThreads", &
value=4, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
Expand Down
73 changes: 73 additions & 0 deletions SingleModelOpenMPUnawareProto/Makefile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# GNU Makefile template for user ESMF application

################################################################################
################################################################################
## This Makefile must be able to find the "esmf.mk" Makefile fragment in the ##
## 'include' line below. Following the ESMF User's Guide, a complete ESMF ##
## installation should ensure that a single environment variable "ESMFMKFILE" ##
## is made available on the system. This variable should point to the ##
## "esmf.mk" file. ##
## ##
## This example Makefile uses the "ESMFMKFILE" environment variable. ##
## ##
## If you notice that this Makefile cannot find variable ESMFMKFILE then ##
## please contact the person responsible for the ESMF installation on your ##
## system. ##
## As a work-around you can simply hardcode the path to "esmf.mk" in the ##
## include line below. However, doing so will render this Makefile a lot less ##
## flexible and non-portable. ##
################################################################################

ifneq ($(origin ESMFMKFILE), environment)
$(error Environment variable ESMFMKFILE was not set.)
endif

include $(ESMFMKFILE)

################################################################################
################################################################################

.SUFFIXES: .f90 .F90 .c .C

%.o : %.f90
$(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREENOCPP) $<

%.o : %.F90
$(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) -DESMF_VERSION_MAJOR=$(ESMF_VERSION_MAJOR) $<

%.o : %.c
$(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $<

%.o : %.C
$(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $<


# -----------------------------------------------------------------------------
mainApp: mainApp.o driver.o model.o
$(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -o $@ $^ $(ESMF_F90ESMFLINKLIBS)

# module dependencies:
mainApp.o: driver.o
driver.o: model.o

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
.PHONY: dust clean distclean info edit
dust:
rm -f PET*.ESMF_LogFile *.nc *.stdout
clean:
rm -f mainApp *.o *.mod
distclean: dust clean

info:
@echo ==================================================================
@echo ESMFMKFILE=$(ESMFMKFILE)
@echo ==================================================================
@cat $(ESMFMKFILE)
@echo ==================================================================

edit:
nedit mainApp.F90 driver.F90 model.F90 &

run:
env OMP_NUM_THREADS=3 mpirun -np 4 ./mainApp
45 changes: 45 additions & 0 deletions SingleModelOpenMPUnawareProto/README
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
README for single Model OpenMP Unaware NUOPC prototype
-------------------------------------------------------

Very simple NUOPC app with a Driver controlling a single Model component. The
model component uses OpenMP if compiled with support, but does so outside of
NUOPC control. This is called NUOPC-unaware threading or resource control.

Description:

A single model component driven by a single driver component.

The driver assumes that each PET launched is a master thread for its own
OpenMP thead team. NUOPC/ESMF is unaware of the OpenMP threading inside of
the model component. The number of OpenMP threads is set via the standard
OMP_NUM_THREADS environment variable, which is set equal to 3 inside the
Makefile run target.

The ESM driver component uses the default run sequence to drive the MODEL
component.

No connector components are present.

The MODEL component uses a simple two-phase initialization,
consisting of advertise and realize.


Build:
- Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF
installation.
- gmake

Execution:
- Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON.
- Set OMP_NUM_THREADS if directly calling into mpirun as per below.
- mpirun -np X ./esmApp (where X is the total number of PETs, typically 4)

Output:
- PET*.Log files containing compliance checker output if turned on.
- The prototype outputs time stepping information to stdout.

Code structure:
- Makefile - Makefile that is based on the standard esmf.mk mechanism.
- model.F90 - The MODEL component, specializing generic NUOPC_Model.
- driver.F90 - Driver specializing NUOPC_Driver, driving the single MODEL.
- mainApp.F90 - Main application.
190 changes: 190 additions & 0 deletions SingleModelOpenMPUnawareProto/driver.F90
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
!==============================================================================
! Earth System Modeling Framework
! Copyright 2002-2021, University Corporation for Atmospheric Research,
! Massachusetts Institute of Technology, Geophysical Fluid Dynamics
! Laboratory, University of Michigan, National Centers for Environmental
! Prediction, Los Alamos National Laboratory, Argonne National Laboratory,
! NASA Goddard Space Flight Center.
! Licensed under the University of Illinois-NCSA License.
!==============================================================================

module driver

!-----------------------------------------------------------------------------
! Code that specializes generic NUOPC_Driver
!-----------------------------------------------------------------------------

use MPI
use ESMF
use NUOPC
use NUOPC_Driver, &
driverSS => SetServices

use MODEL, only: &
modelSS => SetServices

implicit none

private

! private module data --> ONLY PARAMETERS
integer, parameter :: stepCount = 5
real(ESMF_KIND_R8), parameter :: stepTime = 30.D0 ! step time [s]
! should be parent step

public SetServices

!-----------------------------------------------------------------------------
contains
!-----------------------------------------------------------------------------

subroutine SetServices(driver, rc)
type(ESMF_GridComp) :: driver
integer, intent(out) :: rc

rc = ESMF_SUCCESS

! derive from NUOPC_Driver
call NUOPC_CompDerive(driver, driverSS, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

! specialize driver
call NUOPC_CompSpecialize(driver, specLabel=label_SetModelServices, &
specRoutine=SetModelServices, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

! set driver verbosity
call NUOPC_CompAttributeSet(driver, name="Verbosity", value="high", rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

end subroutine

!-----------------------------------------------------------------------------

subroutine SetModelServices(driver, rc)
type(ESMF_GridComp) :: driver
integer, intent(out) :: rc

! local variables
type(ESMF_GridComp) :: child
type(ESMF_CplComp) :: connector
type(ESMF_Time) :: startTime
type(ESMF_Time) :: stopTime
type(ESMF_TimeInterval) :: timeStep
type(ESMF_Clock) :: internalClock

! - diagnostics -
type(ESMF_VM) :: vm
logical :: isFlag
character(80) :: msgString
integer :: mpiComm, size, ierr

rc = ESMF_SUCCESS

! SetServices for MODEL component
call NUOPC_DriverAddComp(driver, "MODEL", modelSS, comp=child, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out
call NUOPC_CompAttributeSet(child, name="Verbosity", value="high", rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

! - diagnostics -
isFlag = ESMF_GridCompIsPetLocal(child, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out
write(msgString,*) "GridCompIsPetLocal: ", isFlag
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

call ESMF_GridCompGet(child, vm=vm, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out
isFlag = ESMF_VMIsCreated(vm, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

write(msgString,*) "VmIsCreated: ", isFlag
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

call ESMF_VMGet(vm, mpiCommunicator=mpiComm, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

if (mpiComm==MPI_COMM_NULL) then
write(msgString,*) "MPI_COMM_NULL"
else
call MPI_Comm_size(mpiComm, size, ierr)
write(msgString,*) "valid MPI_COMM with size=",size
endif
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

! set the driver clock
call ESMF_TimeSet(startTime, s = 0, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
call ESMF_Finalize(endflag=ESMF_END_ABORT)

call ESMF_TimeSet(stopTime, s_r8 = stepTime * stepCount, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
call ESMF_Finalize(endflag=ESMF_END_ABORT)

call ESMF_TimeIntervalSet(timeStep, s_r8 = stepTime, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
call ESMF_Finalize(endflag=ESMF_END_ABORT)

internalClock = ESMF_ClockCreate(name="Driver Clock", &
timeStep=timeStep, startTime=startTime, stopTime=stopTime, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
call ESMF_Finalize(endflag=ESMF_END_ABORT)

call ESMF_GridCompSet(driver, clock=internalClock, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, &
file=__FILE__)) &
return ! bail out

end subroutine

!-----------------------------------------------------------------------------

end module
Loading

0 comments on commit 3c71ab2

Please sign in to comment.