Merge pull request #110 from NexGenAnalytics/96-pamgen-remove-use-of-…

…MPI_COMM_WORLD #96: Remove use of MPI_COMM_WORLD in Pamgen package
NexGenAnalytics · Sep 7, 2023 · e5c4a0b · e5c4a0b
2 parents 104e577 + eb93ea7
commit e5c4a0b
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 6 deletions.
diff --git a/packages/pamgen/extra_functions/pamgen_extras.cpp b/packages/pamgen/extra_functions/pamgen_extras.cpp
@@ -8,6 +8,7 @@
 
 #ifdef HAVE_MPI
 #include <mpi.h>
+#include "pamgen_global_comm.h"
 #endif
 /*****************************************************************************/
 void  Conform_Boundary_IDS(long long ** comm_entities,
@@ -42,13 +43,13 @@ void  Conform_Boundary_IDS(long long ** comm_entities,
   for(unsigned i = 0; i < nncm ;i ++){
     int size = entity_counts[i];
     int proc = proc_ids[i];
-    MPI_Irecv(receive_buffer[i],size, MPI_LONG_LONG_INT, proc, 1, MPI_COMM_WORLD, req + i);
+    MPI_Irecv(receive_buffer[i],size, MPI_LONG_LONG_INT, proc, 1, PAMGEN_NEVADA::get_global_comm(), req + i);
   }
 
   for(unsigned i = 0; i < nncm ;i ++){
     int size = entity_counts[i];
     int proc = proc_ids[i];
-    MPI_Send(send_buffer[i], size, MPI_LONG_LONG_INT, proc, 1,MPI_COMM_WORLD);
+    MPI_Send(send_buffer[i], size, MPI_LONG_LONG_INT, proc, 1, PAMGEN_NEVADA::get_global_comm());
   }
 
   for(unsigned i = 0; i < nncm ;i ++){
@@ -113,15 +114,15 @@ void  Conform_Boundary_IDS_topo_entity(std::vector < std:: vector < topo_entity
     int size = topo_entities[i].size();
     if(size > 0){
       int proc = proc_ids[i];
-      MPI_Irecv(receive_buffer[i],size, MPI_LONG_LONG_INT, proc, 1, MPI_COMM_WORLD, req + i);
+      MPI_Irecv(receive_buffer[i],size, MPI_LONG_LONG_INT, proc, 1, PAMGEN_NEVADA::get_global_comm(), req + i);
     }
   }
 
   for(unsigned i = 0; i < nncm ;i ++){
     int size = topo_entities[i].size();
     if(size > 0){
       int proc = proc_ids[i];
-      MPI_Send(send_buffer[i], size, MPI_LONG_LONG_INT, proc, 1,MPI_COMM_WORLD);
+      MPI_Send(send_buffer[i], size, MPI_LONG_LONG_INT, proc, 1, PAMGEN_NEVADA::get_global_comm());
     }
   }
 
@@ -180,7 +181,7 @@ void calc_global_node_ids(long long * globalNodeIds,
   MPI_Scan(&num_unique_nodes,&start_id,1,
       MPI_LONG_LONG_INT,
       MPI_SUM,
-      MPI_COMM_WORLD);
+      PAMGEN_NEVADA::get_global_comm());
   start_id -= num_unique_nodes;
 #endif
 
@@ -302,7 +303,7 @@ void calc_global_ids(std::vector < topo_entity * > eof_vec,
   MPI_Scan(&owned_entities,&start_id,1,
       MPI_LONG_LONG_INT,
       MPI_SUM,
-      MPI_COMM_WORLD);
+      PAMGEN_NEVADA::get_global_comm());
   start_id -= owned_entities;
 #endif
 

diff --git a/packages/pamgen/extra_functions/pamgen_global_comm.h b/packages/pamgen/extra_functions/pamgen_global_comm.h
@@ -0,0 +1,28 @@
+
+#ifndef PAMGEN_GLOBAL_COMM_HPP
+#define PAMGEN_GLOBAL_COMM_HPP
+
+#ifdef HAVE_MPI
+
+#include <mpi.h>
+#include <mutex>
+
+namespace PAMGEN_NEVADA {
+
+static std::mutex mpi_mutex;
+static MPI_Comm Global_MPI_Comm = MPI_COMM_WORLD;
+
+inline void initialize_global_comm(MPI_Comm comm) {
+    std::lock_guard<std::mutex> guard(mpi_mutex);
+    Global_MPI_Comm = comm;
+}
+
+inline MPI_Comm get_global_comm() {
+    std::lock_guard<std::mutex> guard(mpi_mutex);
+    return Global_MPI_Comm;
+}
+
+}
+
+#endif // HAVE_MPI
+#endif // PAMGEN_GLOBAL_COMM_HPP
diff --git a/packages/pamgen/src/CMakeLists.txt b/packages/pamgen/src/CMakeLists.txt
@@ -159,6 +159,7 @@ APPEND_SET(PE_SOURCES
 
 APPEND_SET(PE_HEADERS
   ../extra_functions/pamgen_extras.h
+  ../extra_functions/pamgen_global_comm.h
   )