This deactivates multithreading, which is the main reason that LaMEM …

…is very slow when run with MUMPS/SuperLU_Dist in parallel. The issue is that the BLAS provided in BinaryBuilder is multithreaded by default, with clashes with PETSc. This can be switched off with the environmental variable OMP_NUM_THREADS=1, now added by default.

Project.toml

@@ -1,7 +1,7 @@
 name = "LaMEM"
 uuid = "2e889f3d-35ce-4a77-8ea2-858aecb630f7"
 authors = ["Boris Kaus <[email protected]>"]
-version = "0.2.2"
+version = "0.2.3"
 
 [deps]
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"

src/run_lamem.jl

@@ -3,8 +3,23 @@
 # Note: This downloads the BinaryBuilder version of LaMEM, which is not necessarily the latest version of LaMEM 
 #       (or the same as the current repository), since we have to manually update the builds.
 
+"""
+    deactivate_multithreading!(cmd)
+
+This deactivates multithreading
+"""
+function deactivate_multithreading(cmd::Cmd)
+    # multithreading of the BLAS libraries that is installed by default with the julia BLAS
+    # does not work well. Switch that off:
+    cmd = addenv(cmd,"OMP_NUM_THREADS"=>1)
+    cmd = addenv(cmd,"VECLIB_MAXIMUM_THREADS"=>1)
+
+    return cmd
+end
+
+
 """ 
-    run_lamem(ParamFile::String, cores::Int64=1, args:String=""; wait=true)
+    run_lamem(ParamFile::String, cores::Int64=1, args:String=""; wait=true; deactivate_multithreads=true)
 
 This starts a LaMEM simulation, for using the parameter file `ParamFile` on `cores` number of cores. 
 Optional additional command-line parameters can be specified with `args`.
@@ -23,17 +38,28 @@ julia> ParamFile="../../input_models/BuildInSetups/FallingBlock_Multigrid.dat";
 julia> run_lamem(ParamFile, 2, "-nstep_max = 1")
 ```
 """
-function run_lamem(ParamFile::String, cores::Int64=1, args::String=""; wait=true)
+function run_lamem(ParamFile::String, cores::Int64=1, args::String=""; wait=true; deactivate_multithreads=true)
 
     if cores==1
         # Run LaMEM on a single core, which does not require a working MPI
-        run(`$(LaMEM_jll.LaMEM()) -ParamFile $(ParamFile) $args`, wait=wait);
+        cmd = `$(LaMEM_jll.LaMEM()) -ParamFile $(ParamFile) $args`
+        if deactivate_multithreads
+            cmd = deactivate_multithreading(cmd)
+        end
+
+        run(cmd, wait=wait);
     else
         # set correct environment
         mpirun = setenv(mpiexec, LaMEM_jll.JLLWrappers.JLLWrappers.LIBPATH_env=>LaMEM_jll.LIBPATH[]);
 
+        # create command-line object
+        cmd = `$(mpirun) -n $cores $(LaMEM_jll.LaMEM_path) -ParamFile $(ParamFile) $args`
+        if deactivate_multithreads
+            cmd = deactivate_multithreading(cmd)
+        end
+
         # Run LaMEM in parallel
-        run(`$(mpirun) -n $cores $(LaMEM_jll.LaMEM_path) -ParamFile $(ParamFile) $args`, wait=wait);
+        run(cmd, wait=wait);
     end
 
     return nothing