From 559e22c6ee9bb0e561daf526ca19b2d8c4169bad Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Thu, 9 May 2024 15:17:31 +0000 Subject: [PATCH 01/10] Make FIELD_*RB an alias for either FIELD_*RM or FIELD_*RD --- CMakeLists.txt | 4 +-- dev_alloc_module.fypp | 2 +- fieldType.py | 8 ++---- field_array_module.fypp | 57 ++---------------------------------- field_module.fypp | 64 +++++++++++++++++++++++++++++++++++++---- host_alloc_module.fypp | 2 +- tests/main.F90 | 16 +++++------ 7 files changed, 74 insertions(+), 79 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f8072f0..3a6d3fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,10 +114,10 @@ if(HAVE_CUDA) endif() ## preprocess fypp files -foreach (SUFF IN ITEMS IM RM RB RD LM) +foreach (SUFF IN ITEMS IM RM RD LM) string (TOLOWER ${SUFF} suff) foreach (RANK RANGE 1 5) - foreach (FUNC IN ITEMS "" _shuffle _access _util _array_util _gang _factory _gather _data) + foreach (FUNC IN ITEMS "" _shuffle _access _util _array _array_util _gang _factory _gather _data) add_custom_command (OUTPUT field_${RANK}${suff}${FUNC}_module.F90 COMMAND ${FYPP} -DRANK=${RANK} -DSUFF='${SUFF}' ${fypp_defines} -m os -M ${CMAKE_CURRENT_SOURCE_DIR} -m fieldType ${CMAKE_CURRENT_SOURCE_DIR}/field_RANKSUFF${FUNC}_module.fypp > field_${RANK}${suff}${FUNC}_module.F90 diff --git a/dev_alloc_module.fypp b/dev_alloc_module.fypp index ddd86e1..46c82a5 100644 --- a/dev_alloc_module.fypp +++ b/dev_alloc_module.fypp @@ -9,7 +9,7 @@ MODULE DEV_ALLOC_MODULE -#:set fieldTypeList = list (filter (lambda ft: not ft.alias, fieldType.getFieldTypeList ())) +#:set fieldTypeList = fieldType.getFieldTypeList () ${fieldType.useParkind1 ()}$ diff --git a/fieldType.py b/fieldType.py index 5d286ef..fa4cd86 100755 --- a/fieldType.py +++ b/fieldType.py @@ -20,7 +20,6 @@ def __init__ (self, **kwargs): self.type = th[tt] + '(KIND=' + self.kind + ')' self.default = td[tt] + '_' + self.kind - self.alias = self.suffix == 'RB' self.name = 'FIELD_%s%s' % (self.rank, self.suffix); self.rank = int (self.rank) @@ -32,18 +31,15 @@ def __init__ (self, **kwargs): self.ganged = self.rank > 2 -kinds = ['JPRM', 'JPRB', 'JPRD', 'JPIM', 'JPLM'] +kinds = ['JPRM', 'JPRD', 'JPIM', 'JPLM'] -def getFieldTypeList (ranks=[1,2,3,4,5], kinds=kinds, hasView=None, alias=None, ganged=None): +def getFieldTypeList (ranks=[1,2,3,4,5], kinds=kinds, hasView=None, ganged=None): l = [fieldType (kind=kind, rank=rank) for (kind) in kinds for rank in ranks] if hasView != None: l = [ft for ft in l if ft.hasView == hasView] - if alias != None: - l = [ft for ft in l if ft.alias == alias] - if ganged != None: l = [ft for ft in l if ft.ganged == ganged] diff --git a/field_array_module.fypp b/field_array_module.fypp index f567a01..13fad25 100644 --- a/field_array_module.fypp +++ b/field_array_module.fypp @@ -11,63 +11,12 @@ MODULE FIELD_ARRAY_MODULE #:set fieldTypeList = fieldType.getFieldTypeList (hasView=True) -USE FIELD_MODULE -USE FIELD_FACTORY_MODULE -${fieldType.useParkind1 ()}$ - -IMPLICIT NONE - -PRIVATE - #:for ft in fieldTypeList -TYPE ${ft.name}$_ARRAY - CLASS (${ft.name}$), POINTER :: F_P => NULL () - ${ft.type}$, POINTER :: P (${ft.viewShape}$) => NULL() -CONTAINS - PROCEDURE :: INIT => INIT_${ft.name}$_ARRAY - PROCEDURE :: FINAL => FINAL_${ft.name}$_ARRAY -#:if ft.hasView - PROCEDURE :: UPDATE_VIEW => UPDATE_VIEW_${ft.name}$_ARRAY -#:endif -END TYPE - -PUBLIC :: ${ft.name}$_ARRAY - +USE ${ft.name}$_ARRAY_MODULE #:endfor -CONTAINS - -#:for ft in fieldTypeList -SUBROUTINE INIT_${ft.name}$_ARRAY (SELF, P, LBOUNDS) - -CLASS (${ft.name}$_ARRAY) :: SELF -${ft.type}$, INTENT (IN), TARGET :: P (${ft.shape}$) -INTEGER (KIND=JPIM), INTENT (IN), OPTIONAL :: LBOUNDS (${ft.rank}$) - -CALL FIELD_NEW (SELF%F_P, DATA=P, PERSISTENT=.TRUE., LBOUNDS=LBOUNDS) - -END SUBROUTINE - -SUBROUTINE UPDATE_VIEW_${ft.name}$_ARRAY (SELF, BLOCK_INDEX) -CLASS (${ft.name}$_ARRAY) :: SELF -INTEGER (KIND=JPIM), INTENT (IN) :: BLOCK_INDEX - -IF (ASSOCIATED (SELF%F_P)) THEN - SELF%P => SELF%F_P%GET_VIEW (BLOCK_INDEX) -ENDIF - -END SUBROUTINE - -SUBROUTINE FINAL_${ft.name}$_ARRAY (SELF) -CLASS (${ft.name}$_ARRAY) :: SELF - -IF (ASSOCIATED (SELF%F_P)) THEN - CALL FIELD_DELETE (SELF%F_P) - SELF%P => NULL () -ENDIF - -END SUBROUTINE +${fieldType.useParkind1 ()}$ -#:endfor +IMPLICIT NONE END MODULE diff --git a/field_module.fypp b/field_module.fypp index 31168ae..4d9b19d 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -10,11 +10,53 @@ MODULE FIELD_MODULE #:set fieldTypeList = fieldType.getFieldTypeList () +#:set fieldTypeListRD = fieldType.getFieldTypeList (kinds=['JPRD']) +#:set fieldTypeListRM = fieldType.getFieldTypeList (kinds=['JPRM']) USE DEV_ALLOC_MODULE +#ifdef PARKIND1_SINGLE + +#:for ft in fieldTypeListRM +#:set ftn = "FIELD_" + str (ft.rank) + "RB" +USE ${ft.name}$_MODULE, ONLY : & + & ${ftn}$ => ${ft.name}$, & + & ${ftn}$_PTR => ${ft.name}$_PTR, & + & ${ft.name}$, & + & ${ft.name}$_PTR, & + & ${ft.name}$_OWNER, & + & ${ft.name}$_WRAPPER +#:if ft.hasView +USE ${ft.name}$_MODULE, ONLY : & + & ${ftn}$_VIEW => ${ft.name}$_VIEW, & + & ${ft.name}$_VIEW +#:endif +#:endfor + +#else + +#:for ft in fieldTypeListRD +#:set ftn = "FIELD_" + str (ft.rank) + "RB" +USE ${ft.name}$_MODULE, ONLY : & + & ${ftn}$ => ${ft.name}$, & + & ${ftn}$_PTR => ${ft.name}$_PTR, & + & ${ft.name}$, & + & ${ft.name}$_PTR, & + & ${ft.name}$_OWNER, & + & ${ft.name}$_WRAPPER +#:if ft.hasView +USE ${ft.name}$_MODULE, ONLY : & + & ${ftn}$_VIEW => ${ft.name}$_VIEW, & + & ${ft.name}$_VIEW +#:endif +#:endfor + +#endif + #:for ft in fieldTypeList +#:if ft.kind != 'JPRB' and ft.kind != 'JPRD' USE ${ft.name}$_MODULE +#:endif #:endfor ${fieldType.useParkind1 ()}$ @@ -25,22 +67,32 @@ PRIVATE #:for ft in fieldTypeList PUBLIC :: ${ft.name}$ -PUBLIC :: ${ft.name}$_OWNER -PUBLIC :: ${ft.name}$_WRAPPER PUBLIC :: ${ft.name}$_PTR #:if ft.hasView PUBLIC :: ${ft.name}$_VIEW #:endif +PUBLIC :: ${ft.name}$_OWNER +PUBLIC :: ${ft.name}$_WRAPPER +#:endfor + + +#:for ft in fieldTypeListRD +#:set ftn = "FIELD_" + str (ft.rank) + "RB" +PUBLIC :: ${ftn}$ +PUBLIC :: ${ftn}$_PTR +#:if ft.hasView +PUBLIC :: ${ftn}$_VIEW +#:endif #:endfor PUBLIC :: WAIT_FOR_ASYNC_QUEUE CONTAINS - SUBROUTINE WAIT_FOR_ASYNC_QUEUE(QUEUE) - INTEGER(KIND=JPIM), INTENT(IN) :: QUEUE - !Wait for all data transfer initiated on queue by the current thread +SUBROUTINE WAIT_FOR_ASYNC_QUEUE(QUEUE) +INTEGER(KIND=JPIM), INTENT(IN) :: QUEUE +!Wait for all data transfer initiated on queue by the current thread !$acc wait (QUEUE) - END SUBROUTINE WAIT_FOR_ASYNC_QUEUE +END SUBROUTINE WAIT_FOR_ASYNC_QUEUE END MODULE FIELD_MODULE diff --git a/host_alloc_module.fypp b/host_alloc_module.fypp index b5364f9..f198378 100644 --- a/host_alloc_module.fypp +++ b/host_alloc_module.fypp @@ -9,7 +9,7 @@ MODULE HOST_ALLOC_MODULE -#:set fieldTypeList = fieldType.getFieldTypeList (alias=False) +#:set fieldTypeList = fieldType.getFieldTypeList () ${fieldType.useParkind1 ()}$ USE ISO_C_BINDING diff --git a/tests/main.F90 b/tests/main.F90 index 7bd5963..cc66652 100644 --- a/tests/main.F90 +++ b/tests/main.F90 @@ -10,6 +10,7 @@ PROGRAM MAIN USE FIELD_MODULE +USE FIELD_FACTORY_MODULE USE PARKIND1 IMPLICIT NONE @@ -18,8 +19,8 @@ PROGRAM MAIN INTEGER (KIND=JPIM) :: JBLK,JLEV,JLON,I REAL (KIND=JPRB), ALLOCATABLE :: ZDATA3 (:,:,:) -TYPE (FIELD_3RB_WRAPPER), POINTER :: YLF3 -TYPE (FIELD_4RB_OWNER), POINTER :: YLF4 +CLASS (FIELD_3RB), POINTER :: YLF3 +CLASS (FIELD_4RB), POINTER :: YLF4 REAL (KIND=JPRB), POINTER :: Z3 (:,:,:), Z4 (:,:,:,:) @@ -38,9 +39,7 @@ PROGRAM MAIN ENDDO ENDDO -ALLOCATE (YLF3) - -CALL YLF3%INIT(DATA=ZDATA3 (:,1:3,:), PERSISTENT=.TRUE.) +CALL FIELD_NEW (YLF3, DATA=ZDATA3 (:,1:3,:), PERSISTENT=.TRUE.) PRINT *, '----------- HOST ----------- R ' WRITE (*, '(B32.32)') YLF3%GET_STATUS () @@ -82,10 +81,9 @@ PROGRAM MAIN PRINT *, JLON, Z3 (JLON,1,1) ENDDO -CALL YLF3%FINAL +CALL FIELD_DELETE (YLF3) -ALLOCATE(YLF4) -CALL YLF4%INIT(LBOUNDS=[1, 0, 1, 1], UBOUNDS=[10, 15, 3, 5], PERSISTENT=.TRUE.) +CALL FIELD_NEW (YLF4, LBOUNDS=[1, 0, 1, 1], UBOUNDS=[10, 15, 3, 5], PERSISTENT=.TRUE.) CALL YLF4%GET_HOST_DATA_RDWR (Z4) WRITE (*, *) " LBOUND (Z4) = ", LBOUND (Z4) @@ -107,6 +105,6 @@ PROGRAM MAIN PRINT *, Z4 (1,:,1,1) -CALL YLF4%FINAL +CALL FIELD_DELETE (YLF4) END PROGRAM From 793d0854941c1444ecae4d279997e9fc9f3d29fe Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 10 May 2024 10:25:42 +0000 Subject: [PATCH 02/10] Compile faster (do not use FIELD_MODULE in field_RANKSUFF_*.fypp --- field_RANKSUFF_access_module.fypp | 3 ++- field_RANKSUFF_factory_module.fypp | 10 ++++++++-- field_RANKSUFF_gather_module.fypp | 6 +++--- field_RANKSUFF_shuffle_module.fypp | 17 ++++++++++++++--- field_RANKSUFF_util_module.fypp | 12 ++++++++++-- field_module.fypp | 4 ++-- 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/field_RANKSUFF_access_module.fypp b/field_RANKSUFF_access_module.fypp index 315d766..9853704 100644 --- a/field_RANKSUFF_access_module.fypp +++ b/field_RANKSUFF_access_module.fypp @@ -12,7 +12,8 @@ MODULE FIELD_${RANK}$${SUFF}$_ACCESS_MODULE #:set fieldTypeList = fieldType.getFieldTypeList (ranks=[RANK], kinds=['JP' + SUFF]) #:set dumsize = '1' -USE FIELD_MODULE +USE FIELD_${RANK}$${SUFF}$_MODULE + ${fieldType.useParkind1 ()}$ IMPLICIT NONE diff --git a/field_RANKSUFF_factory_module.fypp b/field_RANKSUFF_factory_module.fypp index 2ef2373..d0275e4 100644 --- a/field_RANKSUFF_factory_module.fypp +++ b/field_RANKSUFF_factory_module.fypp @@ -11,8 +11,14 @@ MODULE FIELD_${RANK}$${SUFF}$_FACTORY_MODULE #:set fieldTypeList = fieldType.getFieldTypeList (ranks=[RANK], kinds=['JP' + SUFF]) -USE FIELD_MODULE -USE FIELD_GANG_MODULE +#:for ft in fieldTypeList +USE ${ft.name}$_MODULE +#:if ft.ganged +USE FIELD_${ft.rank-1}$${SUFF}$_MODULE +USE ${ft.name}$_GANG_MODULE +#:endif +#:endfor + ${fieldType.useParkind1 ()}$ IMPLICIT NONE diff --git a/field_RANKSUFF_gather_module.fypp b/field_RANKSUFF_gather_module.fypp index 304e87c..48c173b 100644 --- a/field_RANKSUFF_gather_module.fypp +++ b/field_RANKSUFF_gather_module.fypp @@ -11,9 +11,9 @@ MODULE FIELD_${RANK}$${SUFF}$_GATHER_MODULE #:set fieldTypeList = fieldType.getFieldTypeList (ranks=[RANK], kinds=['JP' + str (SUFF)], hasView=True) -USE FIELD_MODULE -USE FIELD_ACCESS_MODULE -USE FIELD_FACTORY_MODULE +USE FIELD_${RANK}$${SUFF}$_MODULE +USE FIELD_${RANK}$${SUFF}$_ACCESS_MODULE +USE FIELD_${RANK}$${SUFF}$_FACTORY_MODULE USE FIELD_SHUFFLE_TYPE_MODULE ${fieldType.useParkind1 ()}$ diff --git a/field_RANKSUFF_shuffle_module.fypp b/field_RANKSUFF_shuffle_module.fypp index bcb589c..15a9438 100644 --- a/field_RANKSUFF_shuffle_module.fypp +++ b/field_RANKSUFF_shuffle_module.fypp @@ -11,9 +11,10 @@ MODULE FIELD_${RANK}$${SUFF}$_SHUFFLE_MODULE #:set fieldTypeList = fieldType.getFieldTypeList (ranks=[RANK], kinds=['JP' + SUFF], hasView=True) -USE FIELD_MODULE -USE FIELD_ACCESS_MODULE -USE FIELD_FACTORY_MODULE +USE FIELD_${RANK}$${SUFF}$_MODULE +USE FIELD_${RANK}$${SUFF}$_ACCESS_MODULE +USE FIELD_${RANK}$${SUFF}$_FACTORY_MODULE + ${fieldType.useParkind1 ()}$ IMPLICIT NONE @@ -67,6 +68,11 @@ CONTAINS #:for mode in ['RDONLY', 'RDWR'] FUNCTION PAIR_${ft.name}$_GATHER_${what}$_DATA_${mode}$ (YDPAIR, LDNULL, LDFULL, KLON, KGPBLKS, YDFINDS, YLF, KBLKMIN, KBLKMAX) RESULT (PTR) + +USE FIELD_3IM_MODULE +USE FIELD_3IM_ACCESS_MODULE +USE FIELD_3IM_FACTORY_MODULE + TYPE (${ft.name}$_SHUFFLE_PAIR), POINTER :: YDPAIR LOGICAL, INTENT (IN) :: LDNULL, LDFULL INTEGER (KIND=JPIM), INTENT (IN) :: KLON, KGPBLKS @@ -185,6 +191,11 @@ END FUNCTION PAIR_${ft.name}$_GATHER_${what}$_DATA_${mode}$ #:for ft in fieldTypeList SUBROUTINE PAIR_${ft.name}$_SCATTER_DATA (YDPAIR, YDFINDS) + +USE FIELD_3IM_MODULE +USE FIELD_3IM_ACCESS_MODULE +USE FIELD_3IM_FACTORY_MODULE + TYPE (${ft.name}$_SHUFFLE_PAIR), POINTER :: YDPAIR CLASS (FIELD_3IM), POINTER :: YDFINDS diff --git a/field_RANKSUFF_util_module.fypp b/field_RANKSUFF_util_module.fypp index d8af305..02d8145 100644 --- a/field_RANKSUFF_util_module.fypp +++ b/field_RANKSUFF_util_module.fypp @@ -11,8 +11,10 @@ MODULE FIELD_${RANK}$${SUFF}$_UTIL_MODULE #:set fieldTypeList = fieldType.getFieldTypeList (ranks=[RANK], kinds=['JP' + SUFF]) -USE FIELD_MODULE -USE FIELD_ACCESS_MODULE +USE FIELD_${RANK}$${SUFF}$_MODULE +USE FIELD_${RANK}$${SUFF}$_ACCESS_MODULE + +${fieldType.useParkind1 ()}$ IMPLICIT NONE @@ -58,6 +60,7 @@ END FUNCTION SUBROUTINE LOAD_${ft.name}$ (KLUN, YD) USE FIELD_ABORT_MODULE +USE PARKIND1, ONLY : JPIM INTEGER (KIND=JPIM), INTENT (IN) :: KLUN CLASS (${ft.name}$), POINTER :: YD @@ -68,6 +71,7 @@ END SUBROUTINE SUBROUTINE SAVE_${ft.name}$ (KLUN, YD) USE FIELD_ABORT_MODULE +USE PARKIND1, ONLY : JPIM INTEGER (KIND=JPIM), INTENT (IN) :: KLUN CLASS (${ft.name}$), POINTER :: YD @@ -115,12 +119,14 @@ END SUBROUTINE #:if ft.hasView SUBROUTINE LOAD_${ft.name}$_VIEW (KLUN, YD) +USE PARKIND1, ONLY : JPIM INTEGER (KIND=JPIM), INTENT (IN) :: KLUN CLASS (${ft.name}$_VIEW) :: YD ! Do nothing END SUBROUTINE SUBROUTINE SAVE_${ft.name}$_VIEW (KLUN, YD) +USE PARKIND1, ONLY : JPIM INTEGER (KIND=JPIM), INTENT (IN) :: KLUN CLASS (${ft.name}$_VIEW) :: YD ! Do nothing @@ -148,6 +154,7 @@ END SUBROUTINE SUBROUTINE LOAD_${ft.name}$_PTR (KLUN, YD) USE FIELD_ABORT_MODULE +USE PARKIND1, ONLY : JPIM INTEGER (KIND=JPIM), INTENT (IN) :: KLUN CLASS (${ft.name}$_PTR) :: YD @@ -158,6 +165,7 @@ END SUBROUTINE SUBROUTINE SAVE_${ft.name}$_PTR (KLUN, YD) USE FIELD_ABORT_MODULE +USE PARKIND1, ONLY : JPIM INTEGER (KIND=JPIM), INTENT (IN) :: KLUN CLASS (${ft.name}$_PTR) :: YD diff --git a/field_module.fypp b/field_module.fypp index 4d9b19d..2213541 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -71,8 +71,8 @@ PUBLIC :: ${ft.name}$_PTR #:if ft.hasView PUBLIC :: ${ft.name}$_VIEW #:endif -PUBLIC :: ${ft.name}$_OWNER -PUBLIC :: ${ft.name}$_WRAPPER +!PUBLIC :: ${ft.name}$_OWNER +!PUBLIC :: ${ft.name}$_WRAPPER #:endfor From dae3b81e84db0a5561c9c04654f39630a617fb57 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 10 May 2024 11:56:37 +0000 Subject: [PATCH 03/10] Use macro PARKIND1_SINGLE & test with SP --- CMakeLists.txt | 4 ++++ field_module.fypp | 27 +++++++++++++++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a6d3fb..a6f9645 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,6 +170,10 @@ foreach(prec ${precisions}) INTERFACE $ INTERFACE $ ) + + if( prec MATCHES sp ) + target_compile_definitions( ${LIBNAME}_${prec} PRIVATE PARKIND1_SINGLE ) + endif() # set install location for .mod files install(DIRECTORY ${CMAKE_BINARY_DIR}/include/${LIBNAME}_${prec} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/field_module.fypp b/field_module.fypp index 2213541..f9a4b8d 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -10,6 +10,8 @@ MODULE FIELD_MODULE #:set fieldTypeList = fieldType.getFieldTypeList () + +#:set fieldTypeListRB = fieldType.getFieldTypeList (kinds=['JPRB']) #:set fieldTypeListRD = fieldType.getFieldTypeList (kinds=['JPRD']) #:set fieldTypeListRM = fieldType.getFieldTypeList (kinds=['JPRM']) @@ -17,7 +19,7 @@ USE DEV_ALLOC_MODULE #ifdef PARKIND1_SINGLE -#:for ft in fieldTypeListRM +#:for ft, fta in zip (fieldTypeListRM, fieldTypeListRB) #:set ftn = "FIELD_" + str (ft.rank) + "RB" USE ${ft.name}$_MODULE, ONLY : & & ${ftn}$ => ${ft.name}$, & @@ -33,9 +35,13 @@ USE ${ft.name}$_MODULE, ONLY : & #:endif #:endfor +#:for ft in fieldTypeListRD +USE ${ft.name}$_MODULE +#:endfor + #else -#:for ft in fieldTypeListRD +#:for ft, fta in zip (fieldTypeListRD, fieldTypeListRB) #:set ftn = "FIELD_" + str (ft.rank) + "RB" USE ${ft.name}$_MODULE, ONLY : & & ${ftn}$ => ${ft.name}$, & @@ -51,10 +57,14 @@ USE ${ft.name}$_MODULE, ONLY : & #:endif #:endfor +#:for ft in fieldTypeListRM +USE ${ft.name}$_MODULE +#:endfor + #endif #:for ft in fieldTypeList -#:if ft.kind != 'JPRB' and ft.kind != 'JPRD' +#:if ft.kind != 'JPRM' and ft.kind != 'JPRD' USE ${ft.name}$_MODULE #:endif #:endfor @@ -71,17 +81,14 @@ PUBLIC :: ${ft.name}$_PTR #:if ft.hasView PUBLIC :: ${ft.name}$_VIEW #:endif -!PUBLIC :: ${ft.name}$_OWNER -!PUBLIC :: ${ft.name}$_WRAPPER #:endfor -#:for ft in fieldTypeListRD -#:set ftn = "FIELD_" + str (ft.rank) + "RB" -PUBLIC :: ${ftn}$ -PUBLIC :: ${ftn}$_PTR +#:for ft in fieldTypeListRB +PUBLIC :: ${ft.name}$ +PUBLIC :: ${ft.name}$_PTR #:if ft.hasView -PUBLIC :: ${ftn}$_VIEW +PUBLIC :: ${ft.name}$_VIEW #:endif #:endfor From 66e1623ce8c5ef780bc8137c592151ba35dd61e6 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 10 May 2024 12:07:55 +0000 Subject: [PATCH 04/10] Introduce & use fypp macro to generate the correct associations --- field_module.fypp | 59 +++++++++++++++++------------------------------ 1 file changed, 21 insertions(+), 38 deletions(-) diff --git a/field_module.fypp b/field_module.fypp index f9a4b8d..c3c0d0d 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -17,50 +17,33 @@ MODULE FIELD_MODULE USE DEV_ALLOC_MODULE -#ifdef PARKIND1_SINGLE - -#:for ft, fta in zip (fieldTypeListRM, fieldTypeListRB) -#:set ftn = "FIELD_" + str (ft.rank) + "RB" -USE ${ft.name}$_MODULE, ONLY : & - & ${ftn}$ => ${ft.name}$, & - & ${ftn}$_PTR => ${ft.name}$_PTR, & - & ${ft.name}$, & - & ${ft.name}$_PTR, & - & ${ft.name}$_OWNER, & - & ${ft.name}$_WRAPPER -#:if ft.hasView -USE ${ft.name}$_MODULE, ONLY : & - & ${ftn}$_VIEW => ${ft.name}$_VIEW, & - & ${ft.name}$_VIEW +#:def useAliasRB (ftlRB, ftlTA, ftlOT) + +#:for fta, ftb in zip (ftlTA, ftlRB) +USE ${fta.name}$_MODULE, ONLY : & + & ${ftb.name}$ => ${fta.name}$, & + & ${ftb.name}$_PTR => ${fta.name}$_PTR, & + & ${fta.name}$, & + & ${fta.name}$_PTR, & + & ${fta.name}$_OWNER, & + & ${fta.name}$_WRAPPER +#:if fta.hasView +USE ${fta.name}$_MODULE, ONLY : & + & ${ftb.name}$_VIEW => ${fta.name}$_VIEW, & + & ${fta.name}$_VIEW #:endif #:endfor -#:for ft in fieldTypeListRD -USE ${ft.name}$_MODULE -#:endfor - -#else - -#:for ft, fta in zip (fieldTypeListRD, fieldTypeListRB) -#:set ftn = "FIELD_" + str (ft.rank) + "RB" -USE ${ft.name}$_MODULE, ONLY : & - & ${ftn}$ => ${ft.name}$, & - & ${ftn}$_PTR => ${ft.name}$_PTR, & - & ${ft.name}$, & - & ${ft.name}$_PTR, & - & ${ft.name}$_OWNER, & - & ${ft.name}$_WRAPPER -#:if ft.hasView -USE ${ft.name}$_MODULE, ONLY : & - & ${ftn}$_VIEW => ${ft.name}$_VIEW, & - & ${ft.name}$_VIEW -#:endif +#:for fto in ftlOT +USE ${fto.name}$_MODULE #:endfor -#:for ft in fieldTypeListRM -USE ${ft.name}$_MODULE -#:endfor +#:enddef +#ifdef PARKIND1_SINGLE +$:useAliasRB (fieldTypeListRB, fieldTypeListRM, fieldTypeListRD) +#else +$:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM) #endif #:for ft in fieldTypeList From 1ec777195db4de4b507dcd7266ce231b52409076 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 10 May 2024 12:11:07 +0000 Subject: [PATCH 05/10] Cleaning --- field_module.fypp | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/field_module.fypp b/field_module.fypp index c3c0d0d..c6b7d45 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -9,11 +9,11 @@ MODULE FIELD_MODULE -#:set fieldTypeList = fieldType.getFieldTypeList () +#:set fieldTypeList = fieldType.getFieldTypeList (kinds=['JPIM', 'JPLM']) +#:set fieldTypeListRM = fieldType.getFieldTypeList (kinds=['JPRM']) #:set fieldTypeListRB = fieldType.getFieldTypeList (kinds=['JPRB']) #:set fieldTypeListRD = fieldType.getFieldTypeList (kinds=['JPRD']) -#:set fieldTypeListRM = fieldType.getFieldTypeList (kinds=['JPRM']) USE DEV_ALLOC_MODULE @@ -47,9 +47,7 @@ $:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM) #endif #:for ft in fieldTypeList -#:if ft.kind != 'JPRM' and ft.kind != 'JPRD' USE ${ft.name}$_MODULE -#:endif #:endfor ${fieldType.useParkind1 ()}$ @@ -58,16 +56,7 @@ IMPLICIT NONE PRIVATE -#:for ft in fieldTypeList -PUBLIC :: ${ft.name}$ -PUBLIC :: ${ft.name}$_PTR -#:if ft.hasView -PUBLIC :: ${ft.name}$_VIEW -#:endif -#:endfor - - -#:for ft in fieldTypeListRB +#:for ft in fieldTypeList + fieldTypeListRM + fieldTypeListRB + fieldTypeListRD PUBLIC :: ${ft.name}$ PUBLIC :: ${ft.name}$_PTR #:if ft.hasView From ae42fe1603f8a6c40cdea404be6619e511bea659 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 10 May 2024 12:12:42 +0000 Subject: [PATCH 06/10] Cleaning --- field_module.fypp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/field_module.fypp b/field_module.fypp index c6b7d45..d198d84 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -41,15 +41,11 @@ USE ${fto.name}$_MODULE #:enddef #ifdef PARKIND1_SINGLE -$:useAliasRB (fieldTypeListRB, fieldTypeListRM, fieldTypeListRD) +$:useAliasRB (fieldTypeListRB, fieldTypeListRM, fieldTypeListRD + fieldTypeList) #else -$:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM) +$:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM + fieldTypeList) #endif -#:for ft in fieldTypeList -USE ${ft.name}$_MODULE -#:endfor - ${fieldType.useParkind1 ()}$ IMPLICIT NONE From 3caf8c03757316791e9c64c5f2153c41d8c765f8 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 10 May 2024 13:13:41 +0000 Subject: [PATCH 07/10] Cleaning --- field_module.fypp | 2 -- 1 file changed, 2 deletions(-) diff --git a/field_module.fypp b/field_module.fypp index d198d84..f3f3684 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -15,8 +15,6 @@ MODULE FIELD_MODULE #:set fieldTypeListRB = fieldType.getFieldTypeList (kinds=['JPRB']) #:set fieldTypeListRD = fieldType.getFieldTypeList (kinds=['JPRD']) -USE DEV_ALLOC_MODULE - #:def useAliasRB (ftlRB, ftlTA, ftlOT) #:for fta, ftb in zip (ftlTA, ftlRB) From bd1630a159280cd3774e9c1b2c333299187f1f40 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Fri, 10 May 2024 14:13:06 +0000 Subject: [PATCH 08/10] Add missing field_RANKSUFF_array_module.fypp --- field_RANKSUFF_array_module.fypp | 82 ++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 field_RANKSUFF_array_module.fypp diff --git a/field_RANKSUFF_array_module.fypp b/field_RANKSUFF_array_module.fypp new file mode 100644 index 0000000..b66e1f0 --- /dev/null +++ b/field_RANKSUFF_array_module.fypp @@ -0,0 +1,82 @@ +#! (C) Copyright 2022- ECMWF. +#! (C) Copyright 2022- Meteo-France. +#! +#! This software is licensed under the terms of the Apache Licence Version 2.0 +#! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +#! In applying this licence, ECMWF does not waive the privileges and immunities +#! granted to it by virtue of its status as an intergovernmental organisation +#! nor does it submit to any jurisdiction. + +MODULE FIELD_${RANK}$${SUFF}$_ARRAY_MODULE + +#:set fieldTypeList = fieldType.getFieldTypeList (ranks=[RANK], kinds=['JP' + str (SUFF)]) + +#:for ft in fieldTypeList +USE ${ft.name}$_MODULE +USE ${ft.name}$_FACTORY_MODULE +#:endfor + +${fieldType.useParkind1 ()}$ + +IMPLICIT NONE + +PRIVATE + +#:for ft in fieldTypeList +TYPE ${ft.name}$_ARRAY + CLASS (${ft.name}$), POINTER :: F_P => NULL () +#:if ft.hasView + ${ft.type}$, POINTER :: P (${ft.viewShape}$) => NULL() +#:endif +CONTAINS + PROCEDURE :: INIT => INIT_${ft.name}$_ARRAY + PROCEDURE :: FINAL => FINAL_${ft.name}$_ARRAY +#:if ft.hasView + PROCEDURE :: UPDATE_VIEW => UPDATE_VIEW_${ft.name}$_ARRAY +#:endif +END TYPE + +PUBLIC :: ${ft.name}$_ARRAY + +#:endfor + +CONTAINS + +#:for ft in fieldTypeList +SUBROUTINE INIT_${ft.name}$_ARRAY (SELF, P, LBOUNDS) + +CLASS (${ft.name}$_ARRAY) :: SELF +${ft.type}$, INTENT (IN), TARGET :: P (${ft.shape}$) +INTEGER (KIND=JPIM), INTENT (IN), OPTIONAL :: LBOUNDS (${ft.rank}$) + +CALL FIELD_NEW (SELF%F_P, DATA=P, PERSISTENT=.TRUE., LBOUNDS=LBOUNDS) + +END SUBROUTINE + +#:if ft.hasView +SUBROUTINE UPDATE_VIEW_${ft.name}$_ARRAY (SELF, BLOCK_INDEX) +CLASS (${ft.name}$_ARRAY) :: SELF +INTEGER (KIND=JPIM), INTENT (IN) :: BLOCK_INDEX + +IF (ASSOCIATED (SELF%F_P)) THEN + SELF%P => SELF%F_P%GET_VIEW (BLOCK_INDEX) +ENDIF + +END SUBROUTINE +#:endif + +SUBROUTINE FINAL_${ft.name}$_ARRAY (SELF) +CLASS (${ft.name}$_ARRAY) :: SELF + +IF (ASSOCIATED (SELF%F_P)) THEN + CALL FIELD_DELETE (SELF%F_P) +#:if ft.hasView + SELF%P => NULL () +#:endif +ENDIF + +END SUBROUTINE + +#:endfor + +END MODULE From 84bf9a3fb303d49aa11cfd85a6c533e11cc2ba9f Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Mon, 13 May 2024 08:38:54 +0000 Subject: [PATCH 09/10] Finalize JPRB aliasing --- field_array_module.fypp | 24 +++++++++++++++++++++--- field_array_util_module.fypp | 29 ++++++++++++++++++++++++++--- field_module.fypp | 13 +++++-------- field_util_module.fypp | 35 ++++++++++++++++++++++++++++++++--- tests/test_crc64.F90 | 1 + 5 files changed, 85 insertions(+), 17 deletions(-) diff --git a/field_array_module.fypp b/field_array_module.fypp index 13fad25..837fc62 100644 --- a/field_array_module.fypp +++ b/field_array_module.fypp @@ -9,12 +9,30 @@ MODULE FIELD_ARRAY_MODULE -#:set fieldTypeList = fieldType.getFieldTypeList (hasView=True) +#:set fieldTypeListOT = fieldType.getFieldTypeList (hasView=True, kinds=['JPIM', 'JPLM']) +#:set fieldTypeListRM = fieldType.getFieldTypeList (hasView=True, kinds=['JPRM']) +#:set fieldTypeListRB = fieldType.getFieldTypeList (hasView=True, kinds=['JPRB']) +#:set fieldTypeListRD = fieldType.getFieldTypeList (hasView=True, kinds=['JPRD']) -#:for ft in fieldTypeList -USE ${ft.name}$_ARRAY_MODULE +#:def useAliasRB (ftlRB, ftlTA, ftlOT) + +#:for fta, ftb in zip (ftlTA, ftlRB) +USE ${fta.name}$_ARRAY_MODULE, ONLY : & + & ${ftb.name}$_ARRAY => ${fta.name}$_ARRAY, & + & ${fta.name}$_ARRAY +#:endfor + +#:for fto in ftlOT +USE ${fto.name}$_ARRAY_MODULE #:endfor +#:enddef + +#ifdef PARKIND1_SINGLE +$:useAliasRB (fieldTypeListRB, fieldTypeListRM, fieldTypeListRD + fieldTypeListOT) +#else +$:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM + fieldTypeListOT) +#endif ${fieldType.useParkind1 ()}$ IMPLICIT NONE diff --git a/field_array_util_module.fypp b/field_array_util_module.fypp index 48b81bf..b4a6048 100644 --- a/field_array_util_module.fypp +++ b/field_array_util_module.fypp @@ -8,12 +8,35 @@ #! nor does it submit to any jurisdiction. MODULE FIELD_ARRAY_UTIL_MODULE -#:set fieldTypeList = fieldType.getFieldTypeList () +#:set fieldTypeListOT = fieldType.getFieldTypeList (hasView=True, kinds=['JPIM', 'JPLM']) +#:set fieldTypeListRM = fieldType.getFieldTypeList (hasView=True, kinds=['JPRM']) +#:set fieldTypeListRB = fieldType.getFieldTypeList (hasView=True, kinds=['JPRB']) +#:set fieldTypeListRD = fieldType.getFieldTypeList (hasView=True, kinds=['JPRD']) -#:for ft in fieldTypeList -USE ${ft.name}$_ARRAY_UTIL_MODULE +#:def useAliasRB (ftlRB, ftlTA, ftlOT) + +#:for fta, ftb in zip (ftlTA, ftlRB) +USE ${fta.name}$_ARRAY_UTIL_MODULE, ONLY : & +#:for method in ['LOAD', 'SAVE', 'COPY', 'WIPE', 'HOST'] + & ${method}$_${ftb.name}$_ARRAY => ${method}$_${fta.name}$_ARRAY, ${method}$_${fta.name}$_ARRAY, ${method}$, & +#:endfor + & COPY +#:endfor + +#:for fto in ftlOT +USE ${fto.name}$_ARRAY_UTIL_MODULE #:endfor +#:enddef + +#ifdef PARKIND1_SINGLE +$:useAliasRB (fieldTypeListRB, fieldTypeListRM, fieldTypeListRD + fieldTypeListOT) +#else +$:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM + fieldTypeListOT) +#endif + +${fieldType.useParkind1 ()}$ + IMPLICIT NONE END MODULE diff --git a/field_module.fypp b/field_module.fypp index f3f3684..75b739b 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -9,8 +9,7 @@ MODULE FIELD_MODULE -#:set fieldTypeList = fieldType.getFieldTypeList (kinds=['JPIM', 'JPLM']) - +#:set fieldTypeListOT = fieldType.getFieldTypeList (kinds=['JPIM', 'JPLM']) #:set fieldTypeListRM = fieldType.getFieldTypeList (kinds=['JPRM']) #:set fieldTypeListRB = fieldType.getFieldTypeList (kinds=['JPRB']) #:set fieldTypeListRD = fieldType.getFieldTypeList (kinds=['JPRD']) @@ -22,9 +21,7 @@ USE ${fta.name}$_MODULE, ONLY : & & ${ftb.name}$ => ${fta.name}$, & & ${ftb.name}$_PTR => ${fta.name}$_PTR, & & ${fta.name}$, & - & ${fta.name}$_PTR, & - & ${fta.name}$_OWNER, & - & ${fta.name}$_WRAPPER + & ${fta.name}$_PTR #:if fta.hasView USE ${fta.name}$_MODULE, ONLY : & & ${ftb.name}$_VIEW => ${fta.name}$_VIEW, & @@ -39,9 +36,9 @@ USE ${fto.name}$_MODULE #:enddef #ifdef PARKIND1_SINGLE -$:useAliasRB (fieldTypeListRB, fieldTypeListRM, fieldTypeListRD + fieldTypeList) +$:useAliasRB (fieldTypeListRB, fieldTypeListRM, fieldTypeListRD + fieldTypeListOT) #else -$:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM + fieldTypeList) +$:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM + fieldTypeListOT) #endif ${fieldType.useParkind1 ()}$ @@ -50,7 +47,7 @@ IMPLICIT NONE PRIVATE -#:for ft in fieldTypeList + fieldTypeListRM + fieldTypeListRB + fieldTypeListRD +#:for ft in fieldTypeListOT + fieldTypeListRM + fieldTypeListRB + fieldTypeListRD PUBLIC :: ${ft.name}$ PUBLIC :: ${ft.name}$_PTR #:if ft.hasView diff --git a/field_util_module.fypp b/field_util_module.fypp index 13f547f..64500de 100644 --- a/field_util_module.fypp +++ b/field_util_module.fypp @@ -9,12 +9,41 @@ MODULE FIELD_UTIL_MODULE -#:set fieldTypeList = fieldType.getFieldTypeList () +#:set fieldTypeListOT = fieldType.getFieldTypeList (kinds=['JPIM', 'JPLM']) +#:set fieldTypeListRM = fieldType.getFieldTypeList (kinds=['JPRM']) +#:set fieldTypeListRB = fieldType.getFieldTypeList (kinds=['JPRB']) +#:set fieldTypeListRD = fieldType.getFieldTypeList (kinds=['JPRD']) -#:for ft in fieldTypeList -USE ${ft.name}$_UTIL_MODULE +#:def useAliasRB (ftlRB, ftlTA, ftlOT) + +#:for fta, ftb in zip (ftlTA, ftlRB) +USE ${fta.name}$_UTIL_MODULE, ONLY : & +#:for method in ['LOAD', 'SAVE', 'DIFF', 'COPY', 'WIPE', 'HOST', 'CRC64'] + & ${method}$_${ftb.name}$ => ${method}$_${fta.name}$, ${method}$_${fta.name}$, ${method}$, & +#:if method not in ['DIFF', 'CRC64'] + & ${method}$_${ftb.name}$_PTR => ${method}$_${fta.name}$_PTR, ${method}$_${fta.name}$_PTR, & +#:if fta.hasView + & ${method}$_${ftb.name}$_VIEW => ${method}$_${fta.name}$_VIEW, ${method}$_${fta.name}$_VIEW, & +#:endif +#:endif +#:endfor + & COPY +#:endfor + +#:for fto in ftlOT +USE ${fto.name}$_UTIL_MODULE #:endfor +#:enddef + +#ifdef PARKIND1_SINGLE +$:useAliasRB (fieldTypeListRB, fieldTypeListRM, fieldTypeListRD + fieldTypeListOT) +#else +$:useAliasRB (fieldTypeListRB, fieldTypeListRD, fieldTypeListRM + fieldTypeListOT) +#endif + +${fieldType.useParkind1 ()}$ + IMPLICIT NONE END MODULE diff --git a/tests/test_crc64.F90 b/tests/test_crc64.F90 index 68c443a..9bbe13e 100644 --- a/tests/test_crc64.F90 +++ b/tests/test_crc64.F90 @@ -11,6 +11,7 @@ PROGRAM TEST_CRC64 USE FIELD_MODULE USE FIELD_FACTORY_MODULE USE FIELD_UTIL_MODULE + USE FIELD_ACCESS_MODULE USE OMP_LIB USE PARKIND1 USE FIELD_ABORT_MODULE From 8e7c1b46633f4b328217639e25f4743149ca74f0 Mon Sep 17 00:00:00 2001 From: Philippe Marguinaud Date: Mon, 13 May 2024 09:08:49 +0000 Subject: [PATCH 10/10] Move WAIT_FOR_ASYNC in field_async_module.F90 --- CMakeLists.txt | 2 +- field_async_module.F90 | 25 +++++++++++++++++++++++++ field_module.fypp | 10 ---------- tests/async_host.F90 | 1 + 4 files changed, 27 insertions(+), 11 deletions(-) create mode 100644 field_async_module.F90 diff --git a/CMakeLists.txt b/CMakeLists.txt index a6f9645..ec80e4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,7 +90,7 @@ if( NOT fiat_FOUND ) list(APPEND srcs ${ABOR1_PATH} ${OML_PATH} ${PARKIND1_PATH}) endif() -list(APPEND srcs field_basic_module.F90 field_defaults_module.F90 dev_alloc.c c_malloc.c field_constants_module.F90 field_abort_module.F90 field_statistics_module.F90) +list(APPEND srcs field_async_module.F90 field_basic_module.F90 field_defaults_module.F90 dev_alloc.c c_malloc.c field_constants_module.F90 field_abort_module.F90 field_statistics_module.F90) ## check for CUDA include(CheckLanguage) diff --git a/field_async_module.F90 b/field_async_module.F90 new file mode 100644 index 0000000..a7bf2ca --- /dev/null +++ b/field_async_module.F90 @@ -0,0 +1,25 @@ +! (C) Copyright 2022- ECMWF. +! (C) Copyright 2022- Meteo-France. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation +! nor does it submit to any jurisdiction. + +MODULE FIELD_ASYNC_MODULE + +IMPLICIT NONE + +CONTAINS + +SUBROUTINE WAIT_FOR_ASYNC_QUEUE (QUEUE) + +USE PARKIND1, ONLY : JPIM + +INTEGER(KIND=JPIM), INTENT(IN) :: QUEUE +!Wait for all data transfer initiated on queue by the current thread +!$acc wait (QUEUE) +END SUBROUTINE WAIT_FOR_ASYNC_QUEUE + +END MODULE FIELD_ASYNC_MODULE diff --git a/field_module.fypp b/field_module.fypp index 75b739b..1539505 100644 --- a/field_module.fypp +++ b/field_module.fypp @@ -55,14 +55,4 @@ PUBLIC :: ${ft.name}$_VIEW #:endif #:endfor -PUBLIC :: WAIT_FOR_ASYNC_QUEUE - -CONTAINS - -SUBROUTINE WAIT_FOR_ASYNC_QUEUE(QUEUE) -INTEGER(KIND=JPIM), INTENT(IN) :: QUEUE -!Wait for all data transfer initiated on queue by the current thread -!$acc wait (QUEUE) -END SUBROUTINE WAIT_FOR_ASYNC_QUEUE - END MODULE FIELD_MODULE diff --git a/tests/async_host.F90 b/tests/async_host.F90 index 614d6cf..5644859 100644 --- a/tests/async_host.F90 +++ b/tests/async_host.F90 @@ -9,6 +9,7 @@ PROGRAM SYNC_HOST USE FIELD_MODULE + USE FIELD_ASYNC_MODULE USE FIELD_FACTORY_MODULE USE PARKIND1 USE FIELD_ABORT_MODULE