remove copy of state into mpi window

.gitignore

@@ -203,6 +203,7 @@ test_quad_reg_interp
 test_table_read
 test_ran_unif
 test_kde_dist
+test_window
 
 # Directories to NOT IGNORE ... same as executable names
 # as far as I know, these must be listed after the executables

assimilation_code/modules/utilities/distributed_state_mod.f90

@@ -110,7 +110,7 @@ subroutine get_fwd(x, my_index, state_ens_handle)
       !x = get_local_state(element_index)
       x = state_ens_handle%copies(1:data_count, element_index)
    else
-     call get_from_fwd(owner_of_state, state_win, element_index, data_count, x)
+     call get_from_fwd(owner_of_state, state_win, element_index, state_ens_handle%num_copies, data_count, x)
    endif
 endif
 

assimilation_code/modules/utilities/mpi_utilities_mod.f90

@@ -1968,13 +1968,14 @@ end subroutine get_from_mean
 
 !-----------------------------------------------------------------------------
 
-subroutine get_from_fwd(owner, window, mindex, num_rows, x)
+subroutine get_from_fwd(owner, window, mindex, rows_in_window, num_rows, x)
 
 integer,  intent(in)  :: owner    ! task in the window that owns the memory
 integer,  intent(in)  :: window   ! window object
 integer,  intent(in)  :: mindex   ! index in the tasks memory
-integer,  intent(in)  :: num_rows ! number of rows in the window
-real(r8), intent(out) :: x(:)     ! result
+integer,  intent(in)  :: rows_in_window ! number of rows in the window
+integer,  intent(in)  :: num_rows ! number of rows to get from the window
+real(r8), intent(out) :: x(num_rows)     ! result
 
 integer(KIND=MPI_ADDRESS_KIND) :: target_disp
 integer :: errcode
@@ -1983,7 +1984,7 @@ subroutine get_from_fwd(owner, window, mindex, num_rows, x)
 ! to have occured until the call to mpi_win_unlock. 
 ! => Don't do anything with x in between mpi_get and mpi_win_lock
 
-target_disp = (mindex - 1)*num_rows
+target_disp = (mindex - 1)*rows_in_window
 call mpi_win_lock(MPI_LOCK_SHARED, owner, 0, window, errcode)
 call mpi_get(x, num_rows, datasize, owner, target_disp, num_rows, datasize, window, errcode)
 call mpi_win_unlock(owner, window, errcode)

assimilation_code/modules/utilities/mpif08_utilities_mod.f90

@@ -1969,12 +1969,13 @@ end subroutine get_from_mean
 
 !-----------------------------------------------------------------------------
 
-subroutine get_from_fwd(owner, window, mindex, num_rows, x)
+subroutine get_from_fwd(owner, window, mindex, rows_in_window, num_rows, x)
 
 integer,  intent(in)  :: owner    ! task in the window that owns the memory
 type(MPI_Win),  intent(in)  :: window   ! window object
 integer,  intent(in)  :: mindex   ! index in the tasks memory
-integer,  intent(in)  :: num_rows ! number of rows in the window
+integer,  intent(in)  :: rows_in_window ! number of rows in the window
+integer,  intent(in)  :: num_rows ! number of rows to get from the window
 real(r8), intent(out) :: x(num_rows)     ! result
 
 integer(KIND=MPI_ADDRESS_KIND) :: target_disp
@@ -1984,7 +1985,7 @@ subroutine get_from_fwd(owner, window, mindex, num_rows, x)
 ! to have occured until the call to mpi_win_unlock. 
 ! => Don't do anything with x in between mpi_get and mpi_win_lock
 
-target_disp = (mindex - 1)*num_rows
+target_disp = (mindex - 1)*rows_in_window
 call mpi_win_lock(MPI_LOCK_SHARED, owner, 0, window, errcode)
 call mpi_get(x, num_rows, datasize, owner, target_disp, num_rows, datasize, window, errcode)
 call mpi_win_unlock(owner, window, errcode)

assimilation_code/modules/utilities/null_mpi_utilities_mod.f90

@@ -634,13 +634,14 @@ end subroutine get_from_mean
 
 !-----------------------------------------------------------------------------
 
-subroutine get_from_fwd(owner, window, mindex, num_rows, x)
+subroutine get_from_fwd(owner, window, mindex, rows_in_window, num_rows, x)
 
 integer,  intent(in)  :: owner    ! task in the window that owns the memory
 integer,  intent(in)  :: window   ! window object
 integer,  intent(in)  :: mindex   ! index in the tasks memory
-integer,  intent(in)  :: num_rows ! number of rows in the window
-real(r8), intent(out) :: x(num_rows) ! result
+integer,  intent(in)  :: rows_in_window ! number of rows in the window
+integer,  intent(in)  :: num_rows ! number of rows to get from the window
+real(r8), intent(out) :: x(num_rows)     ! result
 
 call error_handler(E_ERR,'get_from_fwd', 'cannot be used in serial mode', source)
 

assimilation_code/modules/utilities/no_cray_win_mod.f90 → assimilation_code/modules/utilities/win_mod.f90

@@ -2,7 +2,6 @@
 ! by UCAR, "as is", without charge, subject to all terms of use at
 ! http://www.image.ucar.edu/DAReS/DART/DART_download
 
-!> Window without cray pointer. Should you point the window at contigous memory?
 module window_mod
 
 !> \defgroup window window_mod
@@ -30,21 +29,16 @@ module window_mod
 integer :: current_win !< keep track of current window, start out assuming an invalid window
 
 ! parameters for keeping track of which window is open
-!>@todo should this be in the window_mod?  you will have to change in both cray 
-!> and non cray versions 
 integer, parameter :: NO_WINDOW    = -1
 integer, parameter :: MEAN_WINDOW  = 0 
 integer, parameter :: STATE_WINDOW = 2 
 
-integer :: data_count !! number of copies in the window
+integer :: data_count !! number of copies required
 integer(KIND=MPI_ADDRESS_KIND) :: window_size
 logical :: use_distributed_mean = .false. ! initialize to false
 
 ! Global memory to stick the mpi window to.
 ! Need a simply contiguous piece of memory to pass to mpi_win_create
-! Openmpi 1.10.0 will not compile with ifort 16 if
-! you create a window with a 2d array.
-real(r8), allocatable :: contiguous_fwd(:)
 real(r8), allocatable :: mean_1d(:)
 
 type(ensemble_type) :: mean_ens_handle
@@ -65,8 +59,7 @@ subroutine create_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handl
 integer :: bytesize !< size in bytes of each element in the window
 integer :: my_num_vars !< my number of vars
 
-! Find out how many copies to put in the window
-! copies_in_window is not necessarily equal to ens_handle%num_copies
+! Find out how many copies to get, maybe different to state_ens_handle%num_copies
 data_count = copies_in_window(state_ens_handle)
 
 if (get_allow_transpose(state_ens_handle)) then
@@ -82,20 +75,15 @@ subroutine create_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handl
    my_num_vars = state_ens_handle%my_num_vars
 
    call mpi_type_size(datasize, bytesize, ierr)
-   window_size = my_num_vars*data_count*bytesize
-
-   allocate(contiguous_fwd(data_count*my_num_vars))
-   contiguous_fwd = reshape(state_ens_handle%copies(1:data_count, :), (/my_num_vars*data_count/))
+   window_size = my_num_vars*state_ens_handle%num_copies*bytesize
 
    ! Expose local memory to RMA operation by other processes in a communicator.
-   call mpi_win_create(contiguous_fwd, window_size, bytesize, MPI_INFO_NULL, get_dart_mpi_comm(), state_win, ierr)
+   call mpi_win_create(state_ens_handle%copies, window_size, bytesize, MPI_INFO_NULL, get_dart_mpi_comm(), state_win, ierr)
 endif
 
 ! Set the current window to the state window
 current_win = STATE_WINDOW
 
-data_count = copies_in_window(state_ens_handle)
-
 end subroutine create_state_window
 
 !-------------------------------------------------------------
@@ -163,7 +151,6 @@ subroutine free_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handle)
 else
    ! close mpi window
    call mpi_win_free(state_win, ierr)
-   deallocate(contiguous_fwd)
 endif
 
 current_win = NO_WINDOW