Allow for multiple Team splits and team cloning

dash/include/dash/Team.h

@@ -17,6 +17,9 @@
 #include <memory>
 #include <type_traits>
 #include <utility>
+#include <vector>
+#include <algorithm>
+#include <mutex>
 
 
 namespace dash {
@@ -135,7 +138,6 @@ class Team
     _myid(t._myid),
     _size(t._size),
     _parent(t._parent),
-    _child(t._child),
     _position(t._position),
     _num_siblings(t._num_siblings),
     _group(t._group),
@@ -144,6 +146,16 @@ class Team
     t._parent = nullptr;
     t._group  = nullptr;
     t._dartid = DART_TEAM_NULL;
+
+    {
+      std::lock_guard<std::mutex> g(_parent->_mutex);
+
+      _children = std::move(t._children);
+      // TODO: this is shaky, there may still be dangling pointers to t._parent
+      for (Team *team : _children) {
+        team->_parent = this;
+      }
+    }
   }
 
   /**
@@ -187,13 +199,23 @@ class Team
       }
     }
 
-    if (_child) {
-      delete(_child);
-      _child = nullptr;
-    }
+    if (!is_null())
+    {
+      // Child teams will deregister themselves, no need for locking here
+      for (Team* team : _children) {
+        delete(team);
+      }
+      _children.clear();
 
-    if (_parent) {
-      _parent->_child = nullptr;
+      // remove the team from its parent
+      if (!is_all())
+      {
+        std::lock_guard<std::mutex> g(_parent->_mutex);
+        if (nullptr != _parent) {
+          _parent->_children.erase(
+            std::find(_parent->_children.begin(), _parent->_children.end(), this));
+        }
+      }
     }
 
     free();
@@ -367,6 +389,32 @@ class Team
              num_parts);
   }
 
+  /**
+   * Create a clone of this team.
+   *
+   * \return A new team instance containing the same set of units as the
+   *         original.
+   */
+  inline Team & clone()
+  {
+    dart_team_t new_dartid;
+    if (is_null()) {
+      return *this;
+    }
+    dart_team_clone(_dartid, &new_dartid);
+    size_t num_siblings;
+    // tell the other teams that they now have a new sibling
+    {
+      std::lock_guard<std::mutex> g(_parent->_mutex);
+      num_siblings = _parent->_children.size();
+      for (Team *team : _parent->_children) {
+        team->_num_siblings++;
+      }
+    }
+    Team  *new_team = new Team(new_dartid, _parent, num_siblings, num_siblings);
+    return *new_team;
+  }
+
   /**
    * Equality comparison operator.
    *
@@ -413,7 +461,7 @@ class Team
    */
   inline bool is_leaf() const
   {
-    return _child == nullptr;
+    return _children.empty();
   }
 
   /**
@@ -455,6 +503,16 @@ class Team
     else { return Null(); }
   }
 
+#if 0
+
+  /**
+   * TODO: These functions have been disabled to allow for tree of teams
+   *       Possible new implementation: gather all teams from all children
+   *       or provide a special iterator to iterate over a specific level /
+   *       the bottom.
+   */
+
+
   Team & sub(size_t level = 1)
   {
     Team * t = this;
@@ -475,6 +533,7 @@ class Team
     }
     return *t;
   }
+#endif // 0
 
   inline void barrier() const
   {
@@ -563,6 +622,7 @@ class Team
 
 private:
 
+  static
   void register_team(Team * team)
   {
     DASH_LOG_DEBUG("Team.register_team",
@@ -574,6 +634,7 @@ class Team
       std::make_pair(team->_dartid, team));
   }
 
+  static
   void unregister_team(Team * team)
   {
     DASH_LOG_DEBUG("Team.unregister_team",
@@ -611,7 +672,8 @@ class Team
   mutable team_unit_t     _myid         = UNDEFINED_TEAM_UNIT_ID;
   mutable size_t          _size         = 0;
   Team                  * _parent       = nullptr;
-  Team                  * _child        = nullptr;
+  std::vector<Team*>      _children;
+  std::mutex              _mutex;
   size_t                  _position     = 0;
   size_t                  _num_siblings = 0;
   mutable dart_group_t    _group        = DART_GROUP_NULL;

dash/src/Team.cc

@@ -55,14 +55,18 @@ Team::Team(
   _num_siblings(nsiblings)
 {
   if (nullptr != parent) {
-    if (parent->_child) {
-      DASH_THROW(
-        dash::exception::InvalidArgument,
-        "child of team "  << parent->dart_id() << " " <<
-        "already set to " << parent->_child->dart_id() << ", " <<
-        "cannot set to "  << id);
-    } else {
-      parent->_child = this;
+    {
+      std::lock_guard<std::mutex> g(parent->_mutex);
+#ifdef DASH_DEBUG
+      auto iter = std::find(parent->_children.begin(),
+                            parent->_children.end(), this);
+      if (iter != parent->_children.end()) {
+        DASH_THROW(
+          dash::exception::InvalidArgument,
+          "Team " << id << " already registered with parent " << parent->dart_id());
+      }
+#endif
+      parent->_children.push_back(this);
     }
   }
   // Do not register static Team instances as static variable _team might

dash/test/team/TeamTest.cc

@@ -106,3 +106,30 @@ TEST_F(TeamTest, SplitTeamSync)
   }
 }
 
+TEST_F(TeamTest, Clone)
+{
+  if (dash::size() < 2) {
+    SKIP_TEST_MSG("Test requires at least 2 units");
+  }
+
+  auto & team_all = dash::Team::All();
+
+  auto & team_core = team_all.split(2);
+
+  ASSERT_EQ_U(team_all, team_core.parent());
+
+  auto *team_to_clone = &team_core;
+  for (int i = 0; i < 10; ++i) {
+    auto & team_clone = team_to_clone->clone();
+
+    ASSERT_EQ_U(team_all, team_clone.parent());
+    ASSERT_EQ_U(i+1, team_clone.num_siblings());
+    ASSERT_EQ_U(i+1, team_clone.position());
+    ASSERT_EQ_U(team_core.size(), team_clone.size());
+    ASSERT_NE_U(team_core.dart_id(), team_clone.dart_id());
+    ASSERT_NE_U(team_core.dart_id(), team_clone.dart_id());
+
+    team_to_clone = &team_clone;
+  }
+}
+