From 16cf34cca507fde738cf2a663bb3c3ed50fd1d55 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 14 Jul 2023 13:44:37 -0600 Subject: [PATCH] Phalanx: fix exec space instance init for UVM=ON --- packages/phalanx/src/Phalanx_KokkosViewOfViews.hpp | 8 ++++---- .../phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/phalanx/src/Phalanx_KokkosViewOfViews.hpp b/packages/phalanx/src/Phalanx_KokkosViewOfViews.hpp index 19c2dd752a4c..597b2b9a579b 100644 --- a/packages/phalanx/src/Phalanx_KokkosViewOfViews.hpp +++ b/packages/phalanx/src/Phalanx_KokkosViewOfViews.hpp @@ -324,14 +324,14 @@ namespace PHX { /// fence. Be sure to manually fence as needed. template ViewOfViews3(const ExecSpace& exec_space,const std::string name,Extents... extents) - : view_host_(Kokkos::view_alloc(Kokkos::HostSpace(),name),extents...), + : view_host_(Kokkos::view_alloc(typename OuterViewType::HostMirror::execution_space(),name),extents...), view_device_(Kokkos::view_alloc(exec_space,name),extents...), device_view_is_synced_(false), is_initialized_(true), use_count_(0), check_use_count_(true) { - view_host_unmanaged_ = Kokkos::create_mirror_view(Kokkos::HostSpace(),view_device_); + view_host_unmanaged_ = Kokkos::create_mirror_view(Kokkos::view_alloc(typename OuterViewType::HostMirror::execution_space()),view_device_); use_count_ = view_device_.impl_track().use_count(); } @@ -390,9 +390,9 @@ namespace PHX { template void initialize(const ExecSpace& exec_space,const std::string name,Extents... extents) { - view_host_ = typename OuterViewType::HostMirror(Kokkos::view_alloc(Kokkos::HostSpace(),name),extents...); + view_host_ = typename OuterViewType::HostMirror(Kokkos::view_alloc(typename OuterViewType::HostMirror::execution_space(),name),extents...); view_device_ = OuterViewType(Kokkos::view_alloc(exec_space,name),extents...); - view_host_unmanaged_ = Kokkos::create_mirror_view(Kokkos::HostSpace(),view_device_); + view_host_unmanaged_ = Kokkos::create_mirror_view(Kokkos::view_alloc(typename OuterViewType::HostMirror::execution_space()),view_device_); device_view_is_synced_ = false; is_initialized_ = true; use_count_ = view_device_.impl_track().use_count(); diff --git a/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp b/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp index 08f123718393..08e3faa781a6 100644 --- a/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp +++ b/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp @@ -290,6 +290,12 @@ TEUCHOS_UNIT_TEST(PhalanxViewOfViews,ViewOfView3_UserStreamCtor) { TEST_ASSERT(v_of_v.isInitialized()); + // For UVM=ON builds, need to fence here. The outer views are + // being accessed before the initialization is completed on + // device. The failure only shows up if you overload the cuda card + // with multiple tests to slow down the initialization. + streams[3].fence(); // PHX_UVM_ON_FENCE + v_of_v.addView(a,0,0); v_of_v.addView(b,0,1); v_of_v.addView(c,1,0); @@ -363,6 +369,12 @@ TEUCHOS_UNIT_TEST(PhalanxViewOfViews,ViewOfView3_UserStreamInitialize) { v_of_v.initialize(streams[3],"outer host",2,2); TEST_ASSERT(v_of_v.isInitialized()); + // For UVM=ON builds, need to fence here. The outer views are + // being accessed before the initialization is completed on + // device. The failure only shows up if you overload the cuda card + // with multiple tests to slow down the initialization. + streams[3].fence(); // PHX_UVM_ON_FENCE + v_of_v.addView(a,0,0); v_of_v.addView(b,0,1); v_of_v.addView(c,1,0);