MPI (message passing interface) --- run several M2 processes in parallel

.github/workflows/test_build.yml

@@ -49,6 +49,15 @@ jobs:
             compiler: clang-15
             cxx: clang++
             cc: clang
+          # This build tests MPI
+          - build-system: cmake
+            os: macos-12
+            compiler: MPI
+            cxx: clang++
+            cc: clang
+          - build-system: cmake
+            os: ubuntu-22.04
+            compiler: MPI
         exclude:
           - build-system: cmake
             os: macos-13
@@ -68,6 +77,7 @@ jobs:
           brew install --overwrite python
           brew install automake boost tbb ccache ctags texinfo llvm make ninja yasm libffi msolve
           brew install --only-dependencies macaulay2/tap/M2
+          brew install mpich
           brew link factory --force
 
 # ----------------------
@@ -78,6 +88,7 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           sudo add-apt-repository -y -n ppa:macaulay2/macaulay2
+          sudo add-apt-repository -y -n ppa:savoury1/backports
           sudo apt-get update
           sudo apt-get install -y -q --no-install-recommends clang-16 gfortran libtool-bin ninja-build yasm ccache
           sudo apt-get install -y -q --no-install-recommends liblzma-dev libboost-stacktrace-dev \
@@ -86,6 +97,7 @@ jobs:
                   libgivaro-dev libboost-regex-dev fflas-ffpack libflint-dev libmps-dev libfrobby-dev \
                   libsingular-dev singular-data libcdd-dev cohomcalg topcom 4ti2 libnormaliz-dev normaliz coinor-csdp \
                   libnauty-dev nauty lrslib polymake pipx phcpack w3c-markup-validator libtbb-dev qepcad libomp-16-dev msolve
+          sudo apt-get install -y -q --no-install-recommends mpich libmpich-dev
 
 # ----------------------
 #   Steps common to all build variants
@@ -95,6 +107,12 @@ jobs:
         run: |
           echo "CC=${{  matrix.cc  }}" >> $GITHUB_ENV
           echo "CXX=${{ matrix.cxx }}" >> $GITHUB_ENV
+          if [[ "${{ matrix.compiler }}" == "MPI" ]]
+          then echo "CC=mpicc"                    >> $GITHUB_ENV
+               echo "CXX=mpicxx"                  >> $GITHUB_ENV
+               echo "MPICH_CC=${{  matrix.cc  }}" >> $GITHUB_ENV
+               echo "MPICH_CXX=${{ matrix.cxx }}" >> $GITHUB_ENV
+          fi
           if [[ "${{ runner.os }}" == "Linux" ]]
           then shopt -s expand_aliases
                alias llvm-config="/usr/bin/llvm-config-16"
@@ -130,6 +148,7 @@ jobs:
             -DCMAKE_BUILD_TYPE=Release -DBUILD_NATIVE=OFF \
             -DCMAKE_PREFIX_PATH="`brew --prefix`;`brew --prefix libffi`" \
             -DCMAKE_INSTALL_PREFIX=/usr \
+            -DWITH_MPI=`[[ "${{ matrix.compiler }}" == MPI ]] && echo ON || echo OFF` \
             --debug-trycompile
 
       - name: Build libraries using Ninja
@@ -190,6 +209,13 @@ jobs:
           ctest -j4 --output-on-failure -R "ComputationsBook"
           # TODO: add Macaulay2/tests/engine when https://github.com/Macaulay2/M2/issues/1213 is fixed
 
+      - name: Run MPI test examples
+        if: matrix.compiler == 'MPI'
+        run: |
+          mpirun -np 4 ./M2 --script ../../Macaulay2/packages/MPI/example-homotopies.m2
+          mpirun -np 4 ./M2 --script ../../Macaulay2/packages/MPI/example2.m2
+          mpirun -np 4 ./M2 --script ../../Macaulay2/packages/MPI/master-worker-MPI.m2
+
       - name: Run Tests using Autotools
         if: matrix.build-system == 'autotools' && runner.os == 'Linux' && matrix.compiler == 'default'
         run: |

M2/BUILD/MPI/Makefile

@@ -0,0 +1,85 @@
+.PHONY: always
+always:
+
+SHELL:=/bin/bash
+BRANCH := $(shell git branch --contains | grep '^\* ' | sed -e s=^..== -e s=/=.=g )
+
+#######################
+## cmake build files ##
+#######################
+
+cmake-appleclang:
+	echo "git branch is " $(BRANCH)
+	mkdir -p builds.tmp/cmake-appleclang
+	cd builds.tmp/cmake-appleclang; cmake \
+		-GNinja \
+		-DCMAKE_C_COMPILER=mpicc \
+		-DCMAKE_CXX_COMPILER=mpic++ \
+		-DCMAKE_BUILD_TYPE=RelWithDebInfo \
+		-DBUILD_TESTING=on \
+		-DBUILD_DOCS=on \
+		-DBUILD_NATIVE=off\
+		-DWITH_MPI=ON\
+		-DCMAKE_PREFIX_PATH="`brew --prefix libomp`" \
+		../../../..
+#		-DLINTING=on
+
+auto-ubuntu:
+	echo "git branch is " $(BRANCH)
+	mkdir -p builds.tmp/$@
+	cd builds.tmp/$@; ../../../../configure \
+		CC=mpicc CXX=mpic++ \
+		--with-MPI \
+		--prefix="`pwd`/installed" \
+                --disable-strip \
+                --enable-download
+auto-ubuntu-debug:
+	echo "git branch is " $(BRANCH)
+	mkdir -p builds.tmp/$@
+	cd builds.tmp/$@; ../../../../configure \
+		CC=mpicc CXX=mpic++ \
+		--with-MPI \
+		--prefix="`pwd`/installed" \
+                --disable-strip \
+                --enable-download \
+		--enable-debug \
+		--disable-optimize
+cmake-fedora:
+	module load mpi/mpich
+	echo "git branch is " $(BRANCH)
+	mkdir -p builds.tmp/$@
+	cd builds.tmp/$@; cmake \
+		-GNinja \
+		-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+		-DCMAKE_C_COMPILER=mpicc \
+		-DCMAKE_CXX_COMPILER=mpic++ \
+		-DWITH_MPI=ON\
+		-DCMAKE_BUILD_TYPE=RelWithDebInfo \
+		-DBUILD_TESTING=on \
+		-DBUILD_DOCS=on \
+		../../../..
+
+cmake-ubuntu:
+	echo "git branch is " $(BRANCH)
+	mkdir -p builds.tmp/$@
+	cd builds.tmp/$@; cmake \
+		-GNinja \
+		-DCMAKE_C_COMPILER=mpicc \
+		-DCMAKE_CXX_COMPILER=mpic++ \
+		-DWITH_MPI=ON\
+		-DCMAKE_BUILD_TYPE=RelWithDebInfo \
+		-DBUILD_TESTING=on \
+		-DBUILD_DOCS=on \
+		../../../..
+
+cmake-ubuntu-no-WITH_MPI:
+	echo "git branch is " $(BRANCH)
+	mkdir -p builds.tmp/$@
+	cd builds.tmp/$@; cmake \
+		-GNinja \
+		-DCMAKE_C_COMPILER=mpicc \
+		-DCMAKE_CXX_COMPILER=mpic++ \
+		-DCMAKE_BUILD_TYPE=RelWithDebInfo \
+		-DBUILD_TESTING=on \
+		-DBUILD_DOCS=on \
+		../../../..

M2/BUILD/MPI/cmake.batch

@@ -0,0 +1,22 @@
+# run in the root dir of repo
+brew tap macaulay2/tap
+brew install $(brew deps --1 --include-build macaulay2/tap/M2)
+deps=$(brew deps --1 --include-optional macaulay2/tap/M2 | tr '\n' ';')
+paths=$HOMEBREW_PREFIX/opt/${deps//;/;$HOMEBREW_PREFIX/opt/}
+
+cmake -GNinja -S ./M2 -B ./M2/BUILD/build \
+      -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+      -DCMAKE_C_COMPILER=mpicc \
+      -DCMAKE_CXX_COMPILER=mpic++ \
+      -DWITH_MPI=ON \
+      -DBUILD_NATIVE=OFF \
+      -DCMAKE_PREFIX_PATH=$paths \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=/usr
+
+cd ./M2/BUILD/build
+ninja build-libraries
+ninja build-programs
+ninja
+
+# CC=`brew --prefix llvm`/bin/clang CXX=`brew --prefix llvm`/bin/clang++ cmake -GNinja -S ../.. -B . -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_PREFIX_PATH="`brew --prefix`;`brew --prefix libffi`" 

M2/BUILD/mahrud/Makefile

@@ -46,4 +46,14 @@ cmake-build-with-brew:
 	cmake --build cmake --target install-packages
 	cmake --build cmake --target M2-emacs
 
+cmake-build-with-mpi:
+	MPICH_CC=clang MPICH_CXX=clang++ CC=mpicc CXX=mpicxx \
+	cmake -GNinja -S../.. -BMPI -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$(OPT_PREFIX)" -DWITH_MPI=ON
+	cmake --build MPI --target M2-core
+	cd MPI
+	mpirun -np 4 ./M2 --script ../../../Macaulay2/packages/MPI/example-homotopies.m2
+	mpirun -np 4 ./M2 --script ../../../Macaulay2/packages/MPI/example2.m2
+	mpirun -np 4 ./M2 --script ../../../Macaulay2/packages/MPI/master-worker-MPI.m2
+
+
 #TODO: can we skip building factory? use FTABLESDIR=/usr/share/factory/ 

M2/BUILD/mike/Makefile

@@ -301,6 +301,16 @@ darwin64-appleclang : always
 		--enable-download
 #		--enable-build-libraries="cohomcalg"
 
+darwin64-appleclang-mpi : always
+	mkdir -p builds.tmp/darwin64-appleclang-mpi
+	cd builds.tmp/darwin64-appleclang-mpi; ../../../../configure \
+		CC=mpicc CXX=mpic++ \
+		--prefix="`pwd`/installed" \
+		--disable-strip \
+		--enable-download
+
+# mpirun -np 3 M2
+
 darwin64-latestclang : always
 	mkdir -p builds.tmp/darwin64-latestclang
 	cd builds.tmp/darwin64-latestclang; ../../../../configure \

M2/Macaulay2/bin/main.cpp

@@ -25,6 +25,10 @@
 #include <vector>
 #include <flint/flint.h> // for flint_set_abort
 
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
 /* ######################################################################### */
 
 JumpCell abort_jmp;
@@ -66,6 +70,24 @@ static void * GC_start_performance_measurement_0(void *) {
 
 int main(/* const */ int argc, /* const */ char *argv[], /* const */ char *env[])
 {
+#ifdef WITH_MPI
+  // MPI preamble
+  // Initialize the MPI environment
+  if (MPI_Init(NULL, NULL) == MPI_SUCCESS) {
+    // Get the number of processes
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    // Get the rank of the process
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+
+    if (world_rank>0)
+      std::cout << "MPI: initialized process " << world_rank
+		<< " out of " << world_size << " processes" << std::endl;
+  }
+  #endif
+
   /* find the number of environment variables defined */
   int envc = -1;
   while (env[++envc] != NULL) { /* iterate over environ until you hit NULL */ }
@@ -118,7 +140,7 @@ int main(/* const */ int argc, /* const */ char *argv[], /* const */ char *env[]
     waitOnTask(interpTask);
   }
   return 0;
-}
+} //end// main
 
 /* ######################################################################### */
 

M2/Macaulay2/d/CMakeLists.txt

@@ -87,6 +87,7 @@ set(DLIST
   threads.dd
   python.d	# removed unless WITH_PYTHON
   interface.dd interface2.d
+  mpi.d         # removed unless WITH_MPI
   monoid.dd
   monomial_ordering.dd
   texmacs.d
@@ -112,6 +113,9 @@ endif()
 if(NOT WITH_FFI)
   list(REMOVE_ITEM DLIST ffi.d)
 endif()
+if(NOT WITH_MPI)
+  list(REMOVE_ITEM DLIST mpi.d)
+endif()
 
 ###############################################################################
 ## Generate TAGS file

M2/Macaulay2/d/M2lib.c

@@ -9,6 +9,10 @@
 
 #include <readline/readline.h>
 #include <readline/history.h>
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+#include <stdio.h>
 
 extern struct JumpCell abort_jmp;
 extern struct JumpCell interrupt_jmp;
@@ -39,6 +43,80 @@ void system_cpuTime_init(void) {
   startTime = system_cpuTime();
 }
 
+// begin MPI ------------------------------------------------------------------
+#ifdef WITH_MPI
+int MPInumberOfProcesses() {
+  int world_size;
+  MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+  return world_size;
+}
+
+int MPImyProcessNumber() {
+  int world_rank;
+  if (MPI_Comm_rank(MPI_COMM_WORLD, &world_rank) != MPI_SUCCESS)
+    world_rank = -1;
+  return world_rank;
+}
+
+// blocking send 
+int MPIsendString(M2_string s, int p) {
+  char *t = M2_tocharstar(s);
+  int ret = MPI_Send(t, strlen(t)+1, MPI_CHAR, p, 0 /*tag*/, MPI_COMM_WORLD);
+  GC_FREE(t); 
+  return ret;
+}
+
+// blocking receive
+M2_string MPIreceiveString(int p) {
+  MPI_Status status;
+  // Probe for an incoming message from process zero
+  MPI_Probe(p, 0/*tag*/, MPI_COMM_WORLD, &status);
+  // When probe returns, the status object has the size and other
+  // attributes of the incoming message. Get the message size
+  int size;
+  MPI_Get_count(&status, MPI_CHAR, &size);
+  // Allocate a buffer to hold the incoming numbers
+  char* s = (char*) malloc(sizeof(char) * size);
+  // Now receive the message with the allocated buffer
+  MPI_Recv(s, size, MPI_CHAR, p, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+  M2_string ret = M2_tostring(s);
+  free(s);
+  return ret;
+}
+
+
+// nonblocking probe
+int MPIprobeInterrupt() {
+  int flag = 0;
+  int receive_tag;
+  MPI_Status status;
+  const int master = 0;
+  //!!! consider any integer message an interrupt???
+  return MPI_Iprobe(master, MPI_ANY_TAG, MPI_COMM_WORLD, &flag, &status);
+}
+
+// nonblocking send (needed???)
+int MPIsendStringNonblocking(M2_string s, int p) {
+  char *t = M2_tocharstar(s);
+  MPI_Request request;
+  int ret = MPI_Isend(t, strlen(t)+1, MPI_CHAR, p, 0 /*tag*/, MPI_COMM_WORLD, &request);
+  // should remember "request" if e.g. need to check the completion 
+  GC_FREE(t); // is it OK to free? (Should be if MPI still refers to this... but does it?)  
+  return ret;
+}
+
+// interrupt
+int MPIinterrupt(int p) {
+  int buffer;
+  const int MPI_INTERRUPT_TAG = 3210; //!!! is also in e/interrupted.cpp
+  int ret = MPI_Send(&buffer, 1, MPI_INT, p, MPI_INTERRUPT_TAG, MPI_COMM_WORLD);
+  // should remember "request" if e.g. need to check the completion 
+  return ret;
+}
+
+#endif
+// end MPI --------------------------------------------------------------------
+
 double system_threadTime(void) {
   struct timespec t;
   int err = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &t);
@@ -63,6 +141,13 @@ void clean_up(void) {
 #ifndef NDEBUG
   trap();
 #endif
+#ifdef WITH_MPI
+  int n = MPImyProcessNumber();
+  if (n>0)
+      printf("MPI: Bye world from process %d out of %d processes\n",
+	     MPImyProcessNumber(), MPInumberOfProcesses());
+  MPI_Finalize();
+#endif
 }
 
 int system_isReady(int fd) {