GPU Path optimizations

AUTHORS

@@ -1,20 +1,21 @@
 Authors and Contributors
 ========================
 
+- P. Courtier (ECMWF)
 - W. Deconinck (ECMWF)
+- D. Degrauwe (RMI)
 - D. Dent (ECMWF)
 - P. Dueben (ECMWF)
 - R. El Khatib (Meteo France)
+- D. Giard (Meteo France)
 - J. Hague (ECMWF)
 - M. Hamrud (ECMWF)
+- M. Hortal (ECMWF)
 - L. Isaksen (ECMWF)
-- G. Mozdzynski (ECMWF)
 - P. Marguinaud (Meteo France)
+- L. Mosimann (NVIDIA)
+- G. Mozdzynski (ECMWF)
 - A. Mueller (ECMWF)
-- M. Hortal (ECMWF)
-- P. Courtier (ECMWF)
-- D. Degrauwe (RMI)
-- D. Giard (Meteo France)
 - G. Radnoti (ECMWF)
 - D. Salmond (ECMWF)
 - Y. Seity (Meteo France)

CMakeLists.txt

@@ -9,7 +9,7 @@
 cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild )
 
-project( ectrans LANGUAGES C Fortran )
+project( ectrans LANGUAGES C CXX Fortran )
 include( ectrans_macros )
 
 ecbuild_enable_fortran( REQUIRED NO_MODULE_DIRECTORY )

src/programs/CMakeLists.txt

@@ -1,4 +1,5 @@
 # (C) Copyright 2020- ECMWF.
+# (C) Copyright 2022- NVIDIA.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -25,20 +26,16 @@ if( HAVE_TOOLS AND TARGET eccodes_f90 )
       LIBS ${trans} eccodes_f90
       LINKER_LANGUAGE Fortran
       DEFINITIONS ECTRANS_TOOLS_RTABLE_PATH="${ECTRANS_TOOLS_RTABLE_PATH}" )
-
   endforeach()
-
-
 endif()
 
-
 set( HAVE_dp ${HAVE_DOUBLE_PRECISION} )
 set( HAVE_sp ${HAVE_SINGLE_PRECISION} )
 
 if( HAVE_GPU )
   foreach( prec sp dp )
     if( HAVE_${prec} )
-      ecbuild_add_executable(TARGET  driver-spectrans-${prec}
+      ecbuild_add_executable(TARGET  driver-spectrans-CA-${prec}
                              SOURCES driver-spectraltransform.F90
                              INCLUDES
                                   ${MPI_Fortran_INCLUDE_PATH}
@@ -47,14 +44,18 @@ if( HAVE_GPU )
                                   fiat parkind_${prec}
                                   eccodes_f90 eccodes_memfs
                                   ${MPI_Fortran_LIBRARIES}
-                                  trans_gpu_static_${prec}
+                                  trans_gpu_static_CA_${prec}
                                   gpu
                                   OpenACC::OpenACC_Fortran 
                                   ${LAPACK_LIBRARIES}
                                   nvhpcwrapnvtx
                           )
-	ecbuild_add_executable(TARGET  driver-spectrans-CA-${prec}
-                             SOURCES driver-spectraltransform.F90
+      set_property( TARGET driver-spectrans-CA-${prec} PROPERTY CUDA_ARCHITECTURES 70 )
+      target_compile_options( driver-spectrans-CA-${prec} PRIVATE $<$<COMPILE_LANGUAGE:Fortran>:-g -acc -Minfo=acc -gpu=cc70,lineinfo,deepcopy,fastmath,nordc,pinned -cudalib=cufft,cublas -fpic>  )
+      set_target_properties(driver-spectrans-CA-${prec} PROPERTIES LINK_FLAGS "-acc -cudalib=cufft,cublas -fpic -gpu=cc70,pinned")
+
+      ecbuild_add_executable(TARGET  driver-spectrans-CA-${prec}-indiv
+                             SOURCES driver-spectraltransform_indiv.F90
                              INCLUDES
                                   ${MPI_Fortran_INCLUDE_PATH}
                                   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../trans/gpu/include/ectrans>
@@ -68,17 +69,9 @@ if( HAVE_GPU )
                                   ${LAPACK_LIBRARIES}
                                   nvhpcwrapnvtx
                           )
-		  #trans_gpu_static_${prec}
-		  #gpu
-		  #${CMAKE_BINARY_DIR}/lib/libtrans_gpu_static_${prec}.a
-		  #${CMAKE_BINARY_DIR}/lib/libgpu.a
-      #target_link_libraries( driver-spectrans PRIVATE OpenACC::OpenACC_Fortran )
-      set_property( TARGET driver-spectrans-${prec} PROPERTY CUDA_ARCHITECTURES 70 )
-      set_property( TARGET driver-spectrans-CA-${prec} PROPERTY CUDA_ARCHITECTURES 70 )
-      target_compile_options( driver-spectrans-${prec} PRIVATE $<$<COMPILE_LANGUAGE:Fortran>:-g -acc -Minfo=acc -gpu=cc70,lineinfo,deepcopy,fastmath,nordc -cudalib=cufft,cublas -fpic>  )
-      target_compile_options( driver-spectrans-CA-${prec} PRIVATE $<$<COMPILE_LANGUAGE:Fortran>:-g -acc -Minfo=acc -gpu=cc70,lineinfo,deepcopy,fastmath,nordc -cudalib=cufft,cublas -fpic>  )
-      set_target_properties(driver-spectrans-${prec} PROPERTIES LINK_FLAGS "-acc -cudalib=cufft,cublas -fpic")
-      set_target_properties(driver-spectrans-CA-${prec} PROPERTIES LINK_FLAGS "-acc -cudalib=cufft,cublas -fpic")
+      set_property( TARGET driver-spectrans-CA-${prec}-indiv PROPERTY CUDA_ARCHITECTURES 70 )
+      target_compile_options( driver-spectrans-CA-${prec}-indiv PRIVATE $<$<COMPILE_LANGUAGE:Fortran>:-g -acc -Minfo=acc -gpu=cc70,lineinfo,deepcopy,fastmath,nordc,pinned -cudalib=cufft,cublas -fpic>  )
+      set_target_properties(driver-spectrans-CA-${prec}-indiv PROPERTIES LINK_FLAGS "-acc -cudalib=cufft,cublas -fpic -gpu=cc70,pinned")
       message("Building ${prec} GPU driver")
     endif()
   endforeach()

src/programs/driver-spectraltransform.F90

@@ -1,4 +1,5 @@
 ! (C) Copyright 2014- ECMWF.
+! (C) Copyright 2022- NVIDIA.
 ! 
 ! This software is licensed under the terms of the Apache Licence Version 2.0
 ! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -76,13 +77,17 @@ PROGRAM TRANSFORM_TEST
 REAL(KIND=JPRB), POINTER :: ZT(:,:,:) => NULL()
 REAL(KIND=JPRB), ALLOCATABLE :: ZSP(:,:)
 
+REAL(KIND=JPRB),ALLOCATABLE :: PAVE(:)
+REAL(KIND=JPRB),ALLOCATABLE :: PMIN(:)
+REAL(KIND=JPRB),ALLOCATABLE :: PMAX(:)
+
 LOGICAL :: LSTACK
 LOGICAL :: LDONE,LSTDEV
 LOGICAL :: LUSERPNM, LKEEPRPNM, LUSEFLT
 LOGICAL :: LTRACE_STATS,LSTATS_OMP, LSTATS_COMMS, LSTATS_MPL
 LOGICAL :: LSTATS,LBARRIER_STATS, LBARRIER_STATS2, LDETAILED_STATS
 LOGICAL :: LSTATS_ALLOC, LSYNCSTATS, LSTATSCPU, LSTATS_MEM
-LOGICAL :: LXML_STATS
+LOGICAL :: LXML_STATS, LDUMP
 LOGICAL :: LFFTW
 INTEGER(KIND=JPIM) :: NSTATS_MEM, NTRACE_STATS, NPRNT_STATS
 ! 0 - no output, 1 - init and final result, 2 - every timestep
@@ -140,14 +145,15 @@ PROGRAM TRANSFORM_TEST
                  & LUSERPNM, LKEEPRPNM, LUSEFLT, NQ, NLIN, IMAX_FLDS_IN, &
                  & NPRINTNORMS, ITERS, ZMAXERR_CHECK, NPROMA, NPROMATR, LEQ_REGIONS, &
                  & NPRINTLEV, NPRTRW, NPRTRV, NSPECRESMIN, NFLEVG, MBX_SIZE, LSTACK, &
-                 & LFFTW
+                 & LFFTW, LDUMP
 
 !     ------------------------------------------------------------------
 
 #include "setup_trans0.h"
 #include "setup_trans.h"
 #include "inv_trans.h"
 #include "dir_trans.h"
+#include "gpnorm_trans.h"
 #include "dist_spec.h"
 #include "gath_grid.h"
 #include "trans_inq.h"
@@ -200,6 +206,7 @@ PROGRAM TRANSFORM_TEST
 LBARRIER_STATS2=.FALSE.
 LSTATSCPU=.FALSE.
 LSYNCSTATS=.FALSE.
+LDUMP=.TRUE.
 LXML_STATS=.FALSE.
 LTRACE_STATS=.FALSE.
 NSTATS_MEM=0
@@ -243,7 +250,7 @@ PROGRAM TRANSFORM_TEST
 ! Participating processors limited by -P option
 
 !--------------------------
-CALL MPL_INIT()
+CALL MPL_INIT(LDENV=.false.)
 !IF( LSTATS ) CALL GSTATS(0,0)
 ZTINIT=TIMEF()
 
@@ -308,7 +315,11 @@ PROGRAM TRANSFORM_TEST
     IF( NPRTRV*NPRTRW /= NPROC ) CYCLE
     IF( NPRTRV > NPRTRW ) EXIT
     IF( NPRTRW > NSPECRESMIN ) CYCLE
+! With CUDA AWARE MPI we don't need any OpenMP so there is no need for this! Effectively this is even
+! undesireable because it may trigger different domain decompositions for no reasons on different machines
+#ifndef USE_CUDA_AWARE_MPI_FT
     IF( NPRTRW <= NSPECRESMIN/(2*OML_MAX_THREADS()) ) EXIT
+#endif
   ENDDO
   ! GO FOR APPROX SQUARE PARTITION FOR BACKUP
   IF( NPRTRV*NPRTRW /= NPROC .OR. NPRTRW > NSPECRESMIN .OR. NPRTRV > NPRTRW ) THEN
@@ -771,6 +782,10 @@ PROGRAM TRANSFORM_TEST
 ALLOCATE(ZGMV(NPROMA,NFLEVG,NDIMGMV,NGPBLKS))
 ALLOCATE(ZGMVS(NPROMA,NDIMGMVS,NGPBLKS))
 
+ALLOCATE(PMIN(NFLEVG))
+ALLOCATE(PMAX(NFLEVG))
+ALLOCATE(PAVE(NFLEVG))
+
 ALLOCATE(ZNORMSP(1))
 ALLOCATE(ZNORMSP1(1))
 ALLOCATE(ZNORMVOR(NFLEVG))
@@ -857,10 +872,9 @@ PROGRAM TRANSFORM_TEST
   ZTSTEP1(JSTEP)=(TIMEF()-ZTSTEP1(JSTEP))/1000.0_JPRD
 
   ! Dump a field to a binary file
-  CALL DUMP_GRIDPOINT_FIELD(JSTEP, MYPROC, NPROMA, NGPBLKS, ZGMVS(:,1,:), 'S', NOUTDUMP)
-  CALL DUMP_GRIDPOINT_FIELD(JSTEP, MYPROC, NPROMA, NGPBLKS, ZWINDS(:,NFLEVG,3,:),  'U', NOUTDUMP)
-  CALL DUMP_GRIDPOINT_FIELD(JSTEP, MYPROC, NPROMA, NGPBLKS, ZWINDS(:,NFLEVG,4,:),  'V', NOUTDUMP)
-  CALL DUMP_GRIDPOINT_FIELD(JSTEP, MYPROC, NPROMA, NGPBLKS, ZGMV(:,NFLEVG,5,:),  'T', NOUTDUMP)
+  IF (LDUMP) CALL DUMP_GRIDPOINT_FIELD_3D(JSTEP, MYPROC, ZGMVS(:,:,:), 'S', NOUTDUMP)
+  IF (LDUMP) CALL DUMP_GRIDPOINT_FIELD_4D(JSTEP, MYPROC, ZWINDS(:,:,:,:),  'W', NOUTDUMP)
+  IF (LDUMP) CALL DUMP_GRIDPOINT_FIELD_4D(JSTEP, MYPROC, ZGMV(:,:,:,:),  'M', NOUTDUMP)
 
   ZTSTEP2(JSTEP)=TIMEF()
   CALL DIR_TRANS(PSPVOR=ZVOR,PSPDIV=ZDIV,&
@@ -871,6 +885,12 @@ PROGRAM TRANSFORM_TEST
       & PGP3A=ZGMV(:,:,5:5,:))
   ZTSTEP2(JSTEP)=(TIMEF()-ZTSTEP2(JSTEP))/1000.0_JPRD
 
+  ! Dump a field to a binary file
+  IF (LDUMP) CALL DUMP_GRIDPOINT_FIELD_2D(JSTEP, MYPROC, ZVOR(:,:), 'V', NOUTDUMP)
+  IF (LDUMP) CALL DUMP_GRIDPOINT_FIELD_2D(JSTEP, MYPROC, ZDIV(:,:),  'D', NOUTDUMP)
+  IF (LDUMP) CALL DUMP_GRIDPOINT_FIELD_2D(JSTEP, MYPROC, ZSP(:,:), 'P', NOUTDUMP)
+  IF (LDUMP) CALL DUMP_GRIDPOINT_FIELD_3D(JSTEP, MYPROC, ZT(:,:,:),  'T', NOUTDUMP)
+
   ZTSTEP(JSTEP)=(TIMEF()-ZTSTEP(JSTEP))/1000.0_JPRD
 
   ZTSTEPAVG=ZTSTEPAVG+ZTSTEP(JSTEP)
@@ -921,8 +941,32 @@ PROGRAM TRANSFORM_TEST
   ELSE
     WRITE(NOUT,'("time step ",I6," took", F8.4)') JSTEP,ZTSTEP(JSTEP)
   ENDIF
+  flush(nout)
+  ! call acc_present_dump()
+  ! print *, "going to free in 3 seconds"
+  ! call sleep (1)
+  ! print *, "going to free in 2 seconds"
+  ! call sleep (1)
+  ! print *, "going to free in 1 seconds"
+  ! call sleep (1)
+  ! !call acc_clear_freelists()
+  ! call sleep (5)
+  ! !call acc_present_dump()
+  ! !call sleep (10000)
 ENDDO
 
+CALL GPNORM_TRANS(ZWINDS(:,:,2,:),NFLEVG,KPROMA=NPROMA,PAVE=PAVE,PMIN=PMIN,PMAX=PMAX,LDAVE_ONLY=.false.,KRESOL=1)
+if (myproc == 1) then
+    OPEN(800+myproc, FORM="UNFORMATTED")
+    write(800+myproc) "pave", sum(pave)/size(pave)
+    write(800+myproc) "pmin", sum(pmin)/size(pmin)
+    write(800+myproc) "pmax", sum(pmax)/size(pmax)
+    close(800+myproc)
+    print *, "pave", sum(pave)/size(pave)
+    print *, "pmin", sum(pmin)/size(pmin)
+    print *, "pmax", sum(pmax)/size(pmax)
+endif
+
 ZTLOOP=(TIMEF()-ZTLOOP)/1000.0_JPRD
 
 WRITE(NOUT,'(" ")')
@@ -1266,28 +1310,77 @@ SUBROUTINE SORT(A, N)
 
 !     ------------------------------------------------------------------
 
-SUBROUTINE DUMP_GRIDPOINT_FIELD(JSTEP, MYPROC, NPROMA, NGPBLKS, FLD, FLDCHAR, NOUTDUMP)
+SUBROUTINE DUMP_GRIDPOINT_FIELD_2D(JSTEP, MYPROC, FLD, FLDCHAR, NOUTDUMP)
 
 ! Dump a 2D field to a binary file.
 
 INTEGER(KIND=JPIM), INTENT(IN) :: JSTEP ! Time step, used for naming file
 INTEGER(KIND=JPIM), INTENT(IN) :: MYPROC ! MPI rank, used for naming file
-INTEGER(KIND=JPIM), INTENT(IN) :: NPROMA ! Size of NPROMA
-INTEGER(KIND=JPIM), INTENT(IN) :: NGPBLKS ! Number of NPROMA blocks
-REAL(KIND=JPRB)   , INTENT(IN) :: FLD(NPROMA,NGPBLKS) ! 2D field
+REAL(KIND=JPRB)   , INTENT(IN) :: FLD(:,:) ! 2D field
+CHARACTER         , INTENT(IN) :: FLDCHAR ! Single character field identifier
+INTEGER(KIND=JPIM), INTENT(IN) :: NOUTDUMP ! Unit number for output file
+
+CHARACTER(LEN=14) :: FILENAME = "X.XXX.XXXX.dat"
+CHARACTER(LEN=60) :: DUMP_DIR
+
+WRITE(FILENAME(1:1),'(A1)') FLDCHAR
+WRITE(FILENAME(3:5),'(I3.3)') JSTEP
+WRITE(FILENAME(7:10),'(I4.4)') MYPROC
+
+CALL GETENV("DUMP_DIR", DUMP_DIR)
+IF (TRIM(DUMP_DIR) == "") CALL GETCWD(DUMP_DIR)
+OPEN(NOUTDUMP, FILE=TRIM(DUMP_DIR)//'/'//FILENAME, FORM="UNFORMATTED")
+WRITE(NOUTDUMP) FLD
+CLOSE(NOUTDUMP)
+
+END SUBROUTINE DUMP_GRIDPOINT_FIELD_2D
+SUBROUTINE DUMP_GRIDPOINT_FIELD_3D(JSTEP, MYPROC, FLD, FLDCHAR, NOUTDUMP)
+
+! Dump a 3D field to a binary file.
+
+INTEGER(KIND=JPIM), INTENT(IN) :: JSTEP ! Time step, used for naming file
+INTEGER(KIND=JPIM), INTENT(IN) :: MYPROC ! MPI rank, used for naming file
+REAL(KIND=JPRB)   , INTENT(IN) :: FLD(:,:,:) ! 3D field
+CHARACTER         , INTENT(IN) :: FLDCHAR ! Single character field identifier
+INTEGER(KIND=JPIM), INTENT(IN) :: NOUTDUMP ! Unit number for output file
+
+CHARACTER(LEN=14) :: FILENAME = "X.XXX.XXXX.dat"
+CHARACTER(LEN=60) :: DUMP_DIR
+
+WRITE(FILENAME(1:1),'(A1)') FLDCHAR
+WRITE(FILENAME(3:5),'(I3.3)') JSTEP
+WRITE(FILENAME(7:10),'(I4.4)') MYPROC
+
+CALL GETENV("DUMP_DIR", DUMP_DIR)
+IF (TRIM(DUMP_DIR) == "") CALL GETCWD(DUMP_DIR)
+OPEN(NOUTDUMP, FILE=TRIM(DUMP_DIR)//'/'//FILENAME, FORM="UNFORMATTED")
+WRITE(NOUTDUMP) FLD
+CLOSE(NOUTDUMP)
+
+END SUBROUTINE DUMP_GRIDPOINT_FIELD_3D
+SUBROUTINE DUMP_GRIDPOINT_FIELD_4D(JSTEP, MYPROC, FLD, FLDCHAR, NOUTDUMP)
+
+! Dump a 4D field to a binary file.
+
+INTEGER(KIND=JPIM), INTENT(IN) :: JSTEP ! Time step, used for naming file
+INTEGER(KIND=JPIM), INTENT(IN) :: MYPROC ! MPI rank, used for naming file
+REAL(KIND=JPRB)   , INTENT(IN) :: FLD(:,:,:,:) ! 4D field
 CHARACTER         , INTENT(IN) :: FLDCHAR ! Single character field identifier
 INTEGER(KIND=JPIM), INTENT(IN) :: NOUTDUMP ! Unit number for output file
 
 CHARACTER(LEN=14) :: FILENAME = "X.XXX.XXXX.dat"
+CHARACTER(LEN=60) :: DUMP_DIR
 
 WRITE(FILENAME(1:1),'(A1)') FLDCHAR
 WRITE(FILENAME(3:5),'(I3.3)') JSTEP
 WRITE(FILENAME(7:10),'(I4.4)') MYPROC
 
-OPEN(NOUTDUMP, FILE=FILENAME, FORM="UNFORMATTED")
-WRITE(NOUTDUMP) RESHAPE(FLD, (/ NPROMA*NGPBLKS /))
+CALL GETENV("DUMP_DIR", DUMP_DIR)
+IF (TRIM(DUMP_DIR) == "") CALL GETCWD(DUMP_DIR)
+OPEN(NOUTDUMP, FILE=TRIM(DUMP_DIR)//'/'//FILENAME, FORM="UNFORMATTED")
+WRITE(NOUTDUMP) FLD
 CLOSE(NOUTDUMP)
 
-END SUBROUTINE DUMP_GRIDPOINT_FIELD
+END SUBROUTINE DUMP_GRIDPOINT_FIELD_4D
 
 END PROGRAM TRANSFORM_TEST