From 60313d150481936be9329f5fd20e717aca6607b0 Mon Sep 17 00:00:00 2001
From: Nicolas Morales <nmmoral@sandia.gov>
Date: Wed, 27 Sep 2023 16:23:10 -0700
Subject: [PATCH] add extra debug prints to debug hanging issue

---
 src/bvh/collision_object.cpp      | 18 +++++++++++++-----
 src/bvh/collision_object/impl.hpp |  3 +++
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/bvh/collision_object.cpp b/src/bvh/collision_object.cpp
index 2457390..bd9721b 100644
--- a/src/bvh/collision_object.cpp
+++ b/src/bvh/collision_object.cpp
@@ -131,7 +131,7 @@ namespace bvh
       const auto sbeg = ( i == 0 ) ? 0 : m_impl->splits_h( i - 1 );
       const auto send = ( i == m_impl->num_splits ) ? m_impl->split_indices_h.extent( 0 ) : m_impl->splits_h( i );
       const std::size_t nelements = send - sbeg;
-      ::bvh::vt::debug( "{}: creating broadphase patch size {} from offset {}\n", ::vt::theContext()->getNode(), nelements, sbeg );
+      ::bvh::vt::debug( "{}: creating broadphase patch for body {} size {} from offset {}\n", ::vt::theContext()->getNode(), m_impl->collision_idx, nelements, sbeg );
       m_impl->local_patches[i] = broadphase_patch_type(
         i + rank * od_factor, span< const entity_snapshot >( m_impl->snapshots.data() + sbeg, nelements ) );
     }
@@ -165,6 +165,10 @@ namespace bvh
       msg->patch = m_impl->local_patches.at( _local.x() );
       msg->origin_node = rank;
       msg->local_idx = _local;
+      ::bvh::vt::debug( "{}: sending broadphase patch {} for body {} size {}\n",
+                        ::vt::theContext()->getNode(),
+                        vt_index{ _local.x() + offset }, m_impl->collision_idx,
+                        msg->patch.size() );
       return m_impl->broadphase_patch_collection_proxy[vt_index{ _local.x() + offset }]
       .sendMsg< broadphase_patch_msg, &details::set_broadphase_patches >( msg.get() );
     } );
@@ -175,12 +179,13 @@ namespace bvh
     {
       ::vt::trace::TraceScopedEvent scope(bvh_build_trees_);
       // Tree build needs to be done collectively, everyone needs to finish before the next step
-      m_impl->chainset.nextStepCollective( "build_tree_step", [this, offset]( vt_index _idx ){
+      m_impl->chainset.nextStepCollective( "build_tree_step", [this, offset]( vt_index _idx ) {
+        ::bvh::vt::debug( "{}: building tree reduction for patch {} for body {}\n", ::vt::theContext()->getNode(),
+                          vt_index{ _idx.x() + offset }, m_impl->collision_idx );
         return collision_object_impl::build_trees_top_down( vt_index{ _idx.x() + offset },
             m_impl->objgroup, m_impl->broadphase_patch_collection_proxy );
       } );
     }
-
   }
 
   void
@@ -221,8 +226,11 @@ namespace bvh
     int rank = static_cast< int >( ::vt::theContext()->getNode() );
     std::size_t offset = rank * od_factor;
 
-    m_impl->chainset.nextStepCollective( "start broadphase insertion", [this]( vt_index _local_idx) {
-      if ( _local_idx.x() == 0 ) {
+    m_impl->chainset.nextStepCollective( "start broadphase insertion", [this, &_other]( vt_index _local_idx) {
+      if ( _local_idx.x() == 0 )
+      {
+        ::bvh::vt::debug( "{}: starting broadphase between body {} and {}\n",
+                          ::vt::theContext()->getNode(), m_impl->collision_idx, _other.m_impl->collision_idx );
         auto msg = ::vt::makeMessage< collision_object_impl::messages::modify_msg >();
         return m_impl->objgroup[::vt::theContext()->getNode()].sendMsg< collision_object_impl::messages::modify_msg, &collision_object_impl::collision_object_holder::begin_narrowphase_modification >( msg );
       } else
diff --git a/src/bvh/collision_object/impl.hpp b/src/bvh/collision_object/impl.hpp
index d978f28..e1ffc23 100644
--- a/src/bvh/collision_object/impl.hpp
+++ b/src/bvh/collision_object/impl.hpp
@@ -94,11 +94,14 @@ namespace bvh
       const std::size_t nelements = send - sbeg;
       const std::size_t chunk_data_size = nelements * m_entity_unit_size;
       const int rank = _rank;
+      debug_assert( m_entity_unit_size > 0, "entity unit size must be > 0" );
 
       auto send_msg = ::vt::makeMessageSz< narrowphase_patch_msg >( chunk_data_size );
       send_msg->data_size = chunk_data_size;
 
       std::size_t offset = 0;
+      ::bvh::vt::debug( "{}: sending narrowphase patch {} for body {} size {}\n", ::vt::theContext()->getNode(),
+                        vt_index{ _local_idx + rank * overdecomposition }, collision_idx, nelements );
       // Should be replaced with VT serialization
       for (std::size_t j = sbeg; j < send; ++j)
       {