Skip to content

Commit

Permalink
Added global offset index. Need to reconcile iblks, as computed throu…
Browse files Browse the repository at this point in the history 
…gh kf_gp, with zgft being targeted for kf_fs, which results in a crash due to too many batches being dispatched
  • Loading branch information
@dmitrypek
dmitrypek committed Aug 9, 2024
1 parent 44d59cf commit b3d35ea
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 11 deletions.
69 changes: 62 additions & 7 deletions src/trans/cpu/internal/dir_trans_ctl_mod.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,15 @@ MODULE DIR_TRANS_CTL_MOD
USE PARKIND1, ONLY: JPIM, JPRB
USE OVERLAP_TYPES_MOD, ONLY: BATCH
USE OVERLAP_TYPES_MOD, ONLY: BATCHLIST
USE PE2SET_MOD
!USE COMMON

IMPLICIT NONE

TYPE(BATCHLIST) :: ACTIVE_BATCHES
REAL(KIND=JPRB), ALLOCATABLE :: BGTF(:,:)
INTEGER(KIND=JPIM), ALLOCATABLE :: IREQ_RECV(:,:)
REAL(KIND=JPRB), ALLOCATABLE :: ZCOMBUFR(:,:)

CONTAINS
SUBROUTINE DIR_TRANS_CTL(KF_UV_G,KF_SCALARS_G,KF_GP,KF_FS,KF_UV,KF_SCALARS,&
Expand Down Expand Up @@ -80,7 +84,7 @@ SUBROUTINE DIR_TRANS_CTL(KF_UV_G,KF_SCALARS_G,KF_GP,KF_FS,KF_UV,KF_SCALARS,&
USE TPM_GEN, ONLY: NPROMATR, NOUT
USE ABORT_TRANS_MOD, ONLY: ABORT_TRANS
USE LINKED_LIST_M, ONLY: LINKEDLISTNODE
USE TPM_DISTR, ONLY: D
USE TPM_DISTR, ONLY: D,NPROC,MYPROC,MYSETW

IMPLICIT NONE

Expand Down Expand Up @@ -111,8 +115,11 @@ SUBROUTINE DIR_TRANS_CTL(KF_UV_G,KF_SCALARS_G,KF_GP,KF_FS,KF_UV,KF_SCALARS,&

TYPE(BATCH) :: NEW_BATCH
INTEGER(KIND=JPIM) :: IBLKS,JBLK
INTEGER(KIND=JPIM) :: NDONE
INTEGER(KIND=JPIM) :: NDONE,KNRECV,KRECVCOUNT,IGL,JGL,J,ISETW
INTEGER(KIND=JPIM) :: IFIRSTLAT, ILASTLAT, IPOS, ISETA, ISETB, IRECV, ISETV,JFLD,JROC
INTEGER(KIND=JPIM), ALLOCATABLE :: KRECVTOT(:)
TYPE(LINKEDLISTNODE), POINTER :: IB
INTEGER(KIND=JPIM), ALLOCATABLE :: NPTRFS(:)

! ------------------------------------------------------------------

Expand All @@ -135,12 +142,59 @@ SUBROUTINE DIR_TRANS_CTL(KF_UV_G,KF_SCALARS_G,KF_GP,KF_FS,KF_UV,KF_SCALARS,&
ALLOCATE(BGTF(KF_FS,D%NLENGTF))
ENDIF

IBLKS = (KF_GP - 1) / NPROMATR + 1
! IBLKS = (KF_GP - 1) / NPROMATR + 1
IBLKS = (KF_FS - 1) / NPROMATR + 1

! CALL COMPUTE_RECV_PARAMS(KNRECV,KRECVCOUNT)
KNRECV = 0
ALLOCATE(KRECVTOT(NPROC))
DO JROC=1,NPROC

CALL PE2SET(JROC,ISETA,ISETB,ISETW,ISETV)


IFIRSTLAT = MAX(D%NPTRLS(MYSETW),D%NFRSTLAT(ISETA))
ILASTLAT = MIN(D%NPTRLS(MYSETW)+D%NULTPP(MYSETW)-1,D%NLSTLAT(ISETA))

IPOS = 0
DO JGL=IFIRSTLAT,ILASTLAT
IGL = D%NPTRFRSTLAT(ISETA)+JGL-D%NFRSTLAT(ISETA)
IPOS = IPOS+D%NONL(IGL,ISETB)
ENDDO

KRECVTOT(JROC) = IPOS*KF_FS
IF(KRECVTOT(JROC) > 0 .AND. MYPROC /= JROC) THEN
KNRECV = KNRECV + 1
ENDIF

ENDDO
KRECVCOUNT=0
DO J=1,NPROC
KRECVCOUNT=MAX(KRECVCOUNT,KRECVTOT(J))
ENDDO
DEALLOCATE(KRECVTOT)


IF (ALLOCATED(IREQ_RECV) .AND. SIZE(IREQ_RECV,1) /= KNRECV .AND. SIZE(IREQ_RECV,2) /= IBLKS) THEN
DEALLOCATE(IREQ_RECV)
ENDIF
IF(.NOT. ALLOCATED(IREQ_RECV)) THEN
ALLOCATE(IREQ_RECV(KNRECV,IBLKS))
endif

IF (ALLOCATED(ZCOMBUFR) .AND. SIZE(ZCOMBUFR,2) /= KNRECV .AND. SIZE(ZCOMBUFR,1) /= KRECVCOUNT) THEN
DEALLOCATE(ZCOMBUFR)
ENDIF
IF(.NOT. ALLOCATED(ZCOMBUFR)) THEN
ALLOCATE(ZCOMBUFR(KRECVCOUNT,KNRECV))
endif

ALLOCATE(NPTRFS(IBLKS+1))

JBLK = 1
NDONE = 0

CALL ACTIVATE(JBLK, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC, PGP, BGTF)
CALL ACTIVATE(JBLK, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC, PGP, BGTF,NPTRFS)

DO WHILE (NDONE < IBLKS)

Expand All @@ -160,7 +214,7 @@ SUBROUTINE DIR_TRANS_CTL(KF_UV_G,KF_SCALARS_G,KF_GP,KF_FS,KF_UV,KF_SCALARS,&

IF (JBLK < IBLKS) THEN
JBLK = JBLK + 1
CALL ACTIVATE(JBLK, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC, PGP, BGTF)
CALL ACTIVATE(JBLK, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC, PGP, BGTF,NPTRFS)
ENDIF
ENDDO
ELSE
Expand All @@ -174,7 +228,7 @@ SUBROUTINE DIR_TRANS_CTL(KF_UV_G,KF_SCALARS_G,KF_GP,KF_FS,KF_UV,KF_SCALARS,&

END SUBROUTINE DIR_TRANS_CTL

SUBROUTINE ACTIVATE(N, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC, PGP, PGTF)
SUBROUTINE ACTIVATE(N, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC, PGP, PGTF,NPTRFS)
INTEGER, INTENT(IN) :: N
INTEGER(KIND=JPIM), INTENT(IN) :: KF_GP
INTEGER(KIND=JPIM), INTENT(IN) :: KF_SCALARS_G
Expand All @@ -183,11 +237,12 @@ SUBROUTINE ACTIVATE(N, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC, PGP, PGTF
INTEGER(KIND=JPIM), OPTIONAL, INTENT(IN) :: KVSETSC(:)
REAL(KIND=JPRB), OPTIONAL, INTENT(IN) :: PGP(:,:,:)
REAL(KIND=JPRB), INTENT(OUT) :: PGTF(:,:)
INTEGER(KIND=JPIM), ALLOCATABLE :: NPTRFS(:)

CLASS(BATCH), POINTER :: NEW_BATCH

! Add a new Batch to the list
CALL ACTIVE_BATCHES%APPEND(BATCH(N, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC))
CALL ACTIVE_BATCHES%APPEND(BATCH(N, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC,NPTRFS))

SELECT TYPE (NEW_BATCH => ACTIVE_BATCHES%TAIL%VALUE)
TYPE IS (BATCH)
Expand Down
19 changes: 15 additions & 4 deletions src/trans/cpu/internal/overlap_types_mod.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,21 +65,23 @@ MODULE OVERLAP_TYPES_MOD
! Batch methods
! -----------------------------------------------------------------------------

FUNCTION BATCH_CONSTRUCTOR(KBLK, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC) RESULT(THIS)
FUNCTION BATCH_CONSTRUCTOR(KBLK, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC,NPTRFS) RESULT(THIS)
USE SHUFFLE_MOD, ONLY: SHUFFLE
USE FIELD_SPLIT_MOD, ONLY: FIELD_SPLIT
USE TPM_GEN, ONLY: NPROMATR, NOUT
USE TPM_DISTR, ONLY: D, NPRTRNS, NPROC
USE TPM_TRANS, ONLY: NGPBLKS
USE TRGTOL_MOD, ONLY: TRGTOL_PROLOG

! USE COMMON_MOD

INTEGER(KIND=JPIM), INTENT(IN) :: KBLK
INTEGER(KIND=JPIM), INTENT(IN) :: KF_GP
INTEGER(KIND=JPIM), INTENT(IN) :: KF_SCALARS_G
INTEGER(KIND=JPIM), INTENT(IN) :: KF_UV_G
! Not actually optional!
INTEGER(KIND=JPIM), OPTIONAL, INTENT(IN) :: KVSETUV(:)
INTEGER(KIND=JPIM), OPTIONAL, INTENT(IN) :: KVSETSC(:)
INTEGER(KIND=JPIM), ALLOCATABLE :: NPTRFS(:)

INTEGER :: JFLD, IST

Expand Down Expand Up @@ -122,7 +124,12 @@ FUNCTION BATCH_CONSTRUCTOR(KBLK, KF_GP, KF_SCALARS_G, KF_UV_G, KVSETUV, KVSETSC)
ENDDO

! Compute this batch's offset into PGTF
THIS%IOFFGTF = (THIS%NBLK - 1) * NPROMATR + 1
! THIS%IOFFGTF = (THIS%NBLK - 1) * NPROMATR + 1
IF(KBLK .EQ. 1) THEN
NPTRFS(1) = 1
ENDIF
NPTRFS(KBLK+1) = NPTRFS(KBLK) + THIS%NF_FS
THIS%IOFFGTF = NPTRFS(KBLK)

ALLOCATE(THIS%NVSET(THIS%NF_GP))
IST = 1
Expand Down Expand Up @@ -161,11 +168,15 @@ END FUNCTION BATCH_CONSTRUCTOR

SUBROUTINE START_COMM(THIS, PGP, PGTF)
USE TRGTOL_MOD, ONLY: TRGTOL_COMM_SEND

USE TPM_GEN, ONLY: NOUT

CLASS(BATCH), INTENT(INOUT) :: THIS
REAL(KIND=JPRB), OPTIONAL, INTENT(IN) :: PGP(:,:,:)
REAL(KIND=JPRB), INTENT(OUT) :: PGTF(:,:)

WRITE(NOUT,*) "DEBUG", THIS%IOFFGTF, THIS%NF_FS
FLUSH(NOUT)

CALL TRGTOL_COMM_SEND(PGTF(THIS%IOFFGTF:THIS%IOFFGTF+THIS%NF_FS-1,:), THIS%NF_FS, &
& THIS%NF_GP, THIS%NF_SCALARS_G, THIS%NVSET, THIS%NSENDCOUNT, &
& THIS%NRECVCOUNT, THIS%NNSEND, THIS%NNRECV, THIS%NSENDTOT, &
Expand Down

0 comments on commit b3d35ea

Please sign in to comment.