Added ExaCA support to all tutorials (#39)

* Added ExaCA support to all tutorials with optional Allrun -withExaCA flag Co-authored-by: Matt Rolchigo <[email protected]> * cleaned up tutorials and added auto-enabled fucntion option with -withExaCA Allrun flag * Update Allrun to remove exaca openmp setting --------- Co-authored-by: Matt Rolchigo <[email protected]>
ORNL · Nov 12, 2024 · b0fa890 · b0fa890
1 parent f484321
commit b0fa890
Show file tree

Hide file tree

Showing 45 changed files with 392 additions and 115 deletions.
diff --git a/applications/solvers/additiveFoam/createFields.H b/applications/solvers/additiveFoam/createFields.H
@@ -21,7 +21,7 @@ volScalarField alpha3
         runTime.timeName(),
         mesh,
         IOobject::READ_IF_PRESENT,
-        IOobject::NO_WRITE
+        IOobject::AUTO_WRITE
     ),
     mesh,
     dimensionedScalar("alpha.powder", dimless, 0.0),

diff --git a/applications/solvers/additiveFoam/functionObjects/ExaCA/ExaCA.C b/applications/solvers/additiveFoam/functionObjects/ExaCA/ExaCA.C
@@ -32,7 +32,6 @@ License
 #include "volFields.H"
 #include "OFstream.H"
 #include "OSspecific.H"
-#include "meshSearch.H"
 #include "labelVector.H"
 #include "pointMVCWeight.H"
 
@@ -76,7 +75,8 @@ Foam::functionObjects::ExaCA::ExaCA
             IOobject::NO_WRITE
         ),
         vpi_.interpolate(T_)
-    )
+    ),
+    searchEngine_(mesh_, polyMesh::CELL_TETS)
 {
     read(dict);
 
@@ -227,13 +227,11 @@ bool Foam::functionObjects::ExaCA::execute()
 
 void Foam::functionObjects::ExaCA::mapPoints()
 {
-    mesh_.time().cpuTimeIncrement();
-
     if (events.size() == 0)
     {
         return;
     }
-              
+
     // find event sub-space before constructing interpolant weights
     const pointField& points = mesh_.points();
 
@@ -260,15 +258,15 @@ void Foam::functionObjects::ExaCA::mapPoints()
     }
 
     // find interpolant weigthts for each point
-    meshSearch searchEngine(mesh_, polyMesh::CELL_TETS);  
-
     label pI = 0;
     label seedi = events[0][0];
 
     pointsInCell.setSize(mesh_.nCells());
 
     const labelVector nPoints(vector::one + box_.span() / dx_);
 
+    Info << "starting point loop" << endl;
+
     for (label k=0; k < nPoints.z(); ++k)
     {
         for (label j=0; j < nPoints.y(); ++j)
@@ -282,7 +280,7 @@ void Foam::functionObjects::ExaCA::mapPoints()
                     // shift point during search to avoid edges in pointMVC
                     const point spt = pt - vector::one*1e-10;
 
-                    label celli = searchEngine.findCell(spt, seedi, true);
+                    label celli = searchEngine_.findCell(spt, seedi, true);
 
                     if (celli != -1)
                     {
@@ -296,7 +294,7 @@ void Foam::functionObjects::ExaCA::mapPoints()
 
                         pI++;
                     }
-                    
+
                     seedi = celli;
                 }
             }
@@ -311,16 +309,10 @@ void Foam::functionObjects::ExaCA::mapPoints()
     {
         pic.shrink();
     }
-
-    Info<< "Successfully mapped points to mesh in: "
-        << returnReduce(mesh_.time().cpuTimeIncrement(), maxOp<scalar>()) << " s"
-        << endl << endl;
 }
 
 void Foam::functionObjects::ExaCA::interpolate()
-{
-    mesh_.time().cpuTimeIncrement();
-
+{    
     if (events.size() == 0)
     {
         return;
@@ -423,24 +415,28 @@ void Foam::functionObjects::ExaCA::interpolate()
         }
         os << data[i][n] << "\n";
     }
-
-    Info<< "Successfully interpolated and wrote ExaCA data in: "
-        << returnReduce(mesh_.time().cpuTimeIncrement(), maxOp<scalar>()) << " s"
-        << endl << endl;
 }
 
 bool Foam::functionObjects::ExaCA::end()
 {
+    //- sort events by cell and in time
     events.shrink();
 
     sort(events);
 
     Info<< "Number of solidification events: "
         << returnReduce(events.size(), sumOp<scalar>()) << endl;
-
-    mapPoints();
 
+    //- map points to cells
+    mesh_.time().cpuTimeIncrement();
+
+    mapPoints();
+
+    Info<< "Successfully mapped points to mesh in: "
+        << returnReduce(mesh_.time().cpuTimeIncrement(), maxOp<scalar>()) << " s"
+        << endl << endl;
 
+    //- interpolate and write ExaCA data in reduced data format
     const fileName exacaPath
     (
         mesh_.time().rootPath()/mesh_.time().globalCaseName()/"ExaCA"
@@ -449,6 +445,10 @@ bool Foam::functionObjects::ExaCA::end()
     mkDir(exacaPath);
 
     interpolate();
+
+    Info<< "Successfully interpolated and wrote ExaCA data in: "
+        << returnReduce(mesh_.time().cpuTimeIncrement(), maxOp<scalar>()) << " s"
+        << endl << endl;
 
     return true;
 }

diff --git a/applications/solvers/additiveFoam/functionObjects/ExaCA/ExaCA.H b/applications/solvers/additiveFoam/functionObjects/ExaCA/ExaCA.H
@@ -36,7 +36,7 @@ SourceFiles
 
 #include "fvMeshFunctionObject.H"
 #include "volPointInterpolation.H"
-
+#include "meshSearch.H"
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
@@ -75,6 +75,8 @@ class ExaCA
         scalar    isoValue_;
 
         scalar    dx_;
+
+        meshSearch searchEngine_;
 
     // Private Member Functions
 

diff --git a/applications/utilities/multiLayer/runLayers b/applications/utilities/multiLayer/runLayers
@@ -153,9 +153,23 @@ do
     fi
 
     thickness="$(foamDictionary -entry linearDirectionCoeffs/thickness -value system/extrudeMeshDict)"
+
+    # set powder height to the layer thickness 
+    foamDictionary -entry regions -set \
+    "(
+        boxToCell
+        {
+            box (-1 -1 -$layerThickness) (1 1 1);
+            fieldValues
+            (
+                volScalarFieldValue alpha.powder 1
+            );
+        }
+    )" system/setFieldsDict
+
     nProcs="$(foamDictionary -entry numberOfSubdomains -value system/decomposeParDict)"
 
-    transformPoints "translate=(0 0 -$thickness)"
+    transformPoints "translate=(0 0 -$thickness)"    
     setFields
     decomposePar
     mpirun -np $nProcs additiveFoam -parallel

diff --git a/clean.sh b/clean.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Function to attempt removing .nfs files, even if in use
+remove_nfs_files() {
+  find "$1" -type f -name '.nfs*' | while read -r nfs_file; do
+    echo "Removing: $nfs_file"
+
+    # Find the process holding the file and kill it
+    lsof_output=$(lsof "$nfs_file" 2>/dev/null)
+    if [[ -n "$lsof_output" ]]; then
+      pid=$(echo "$lsof_output" | awk 'NR>1 {print $2}' | sort -u)
+      if [[ -n "$pid" ]]; then
+        echo "Killing process $pid holding file $nfs_file"
+        kill -9 $pid
+      fi
+    fi
+
+    # Attempt to remove the file again
+    rm -f "$nfs_file"
+  done
+}
+
+# Navigate through every directory from the current one
+echo "Starting search and removal of .nfs files from: $(pwd)"
+remove_nfs_files "$(pwd)"
+
+# Traverse through all subdirectories
+find . -type d | while read -r dir; do
+  remove_nfs_files "$dir"
+done
+
+echo "Finished removing .nfs files."
+
diff --git a/tutorials/AMB2018-02-B/Allclean b/tutorials/AMB2018-02-B/Allclean
@@ -5,6 +5,7 @@ cd ${0%/*} || exit 1    # Run from this directory
 . $WM_PROJECT_DIR/bin/tools/CleanFunctions
 
 cleanCase
-
-rm -rf ExaCA
+#------------------------------------------------------------------------------
+rm -rf ExaCA/Output*
+rm -rf ExaCA/time-temperature.csv
 #------------------------------------------------------------------------------
diff --git a/tutorials/AMB2018-02-B/Allrun b/tutorials/AMB2018-02-B/Allrun
@@ -3,13 +3,41 @@ cd ${0%/*} || exit 1    # Run from this directory
 
 # Source tutorial run functions
 . $WM_PROJECT_DIR/bin/tools/RunFunctions
-
 application=`getApplication`
 
+# Parse arguments
+withExaCA=false
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -withExaCA)
+      withExaCA=true
+      ;;
+  esac
+  shift
+done
+
+if [ "$withExaCA" = true ]; then
+    foamDictionary -entry functions -set "{#includeFunc ExaCA}" system/controlDict
+else
+    foamDictionary -entry functions -set "{}" system/controlDict
+fi
+
+#------------------------------------------------------------------------------
+# AdditiveFOAM
 runApplication blockMesh
 runApplication decomposePar
 runParallel $application
 
 runApplication reconstructPar
 
 #------------------------------------------------------------------------------
+# ExaCA (Version 2.0.1)
+if [ "$withExaCA" = true ]; then
+    echo "x,y,z,tm,ts,cr" > ExaCA/time-temperature.csv
+    cat ExaCA/data_* >> ExaCA/time-temperature.csv
+    rm -rf ExaCA/data_*
+
+    NPROCS="$(foamDictionary -expand -entry numberOfSubdomains -value system/decomposeParDict)"
+    mpirun -np $NPROCS ~/install/exaca/bin/ExaCA ExaCA/input.json
+fi
+#------------------------------------------------------------------------------
diff --git a/tutorials/AMB2018-02-B/ExaCA/input.json b/tutorials/AMB2018-02-B/ExaCA/input.json
@@ -0,0 +1,35 @@
+{
+   "SimulationType": "FromFile",
+   "MaterialFileName": "Inconel625.json",
+   "GrainOrientationFile": "GrainOrientationVectors.csv",
+   "RandomSeed": 0,
+   "Domain": {
+       "CellSize": 2.5,
+       "TimeStep": 0.125,
+       "NumberOfLayers": 1,
+       "LayerOffset": 0
+   },
+   "Nucleation": {
+      "Density": 100,
+      "MeanUndercooling": 5,
+      "StDev": 0.5
+   },
+   "TemperatureData": {
+       "LayerwiseTempRead": false,
+       "TemperatureFiles": ["./ExaCA/time-temperature.csv"]
+   },
+   "Substrate": {
+      "MeanBaseplateGrainSize": 8.3,
+      "MeanPowderGrainSize": 2.5
+   },
+   "Printing": {
+      "PathToOutput": "./ExaCA/",
+      "OutputFile": "Output",
+      "PrintBinary": false,
+      "PrintExaConstitSize": 0,
+      "Interlayer": {
+          "Fields": ["GrainID", "LayerID", "GrainMisorientation", "UndercoolingCurrent", "SolidificationEventCounter"],
+          "Layers": [0, 1]
+      }
+   }
+}
diff --git a/tutorials/AMB2018-02-B/constant/heatSourceDict b/tutorials/AMB2018-02-B/constant/heatSourceDict
@@ -17,6 +17,12 @@ FoamFile
 
 sources (beam);
 
+/*---------------------------------------------------------------------------*\
+Citation:
+    G.L. Knapp, J. Coleman, M. Rolchigo, M. Stoyanov, A. Plotkowski,
+    Calibrating uncertain parameters in melt pool simulations of additive 
+    manufacturing (2023), https://doi.org/10.1016/j.commatsci.2022.11190.
+\*---------------------------------------------------------------------------*/
 beam
 {
     pathName            scanPath;    
@@ -25,7 +31,7 @@ beam
 
     constantCoeffs
     {
-        eta             0.3;
+        eta             0.33;
     }
 
     heatSourceModel     superGaussian;

diff --git a/tutorials/AMB2018-02-B/system/ExaCA b/tutorials/AMB2018-02-B/system/ExaCA
@@ -12,7 +12,7 @@ ExaCA
     libs ("libExaCAFunctionObject.so");
 
     box         (0 -0.0001 -0.0002) (0.002 0.0001 0);
-    dx          5e-6;
+    dx          2.5e-6;
     isoValue    1620;
 }
 

diff --git a/tutorials/AMB2018-02-B/system/controlDict b/tutorials/AMB2018-02-B/system/controlDict
@@ -7,11 +7,11 @@
 \*---------------------------------------------------------------------------*/
 FoamFile
 {
-    version     2.0;
-    format      ascii;
-    class       dictionary;
-    location    "system";
-    object      controlDict;
+    version         2;
+    format          ascii;
+    class           dictionary;
+    location        "system";
+    object          controlDict;
 }
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -25,7 +25,7 @@ stopAt          endTime;
 
 endTime         0.004;
 
-deltaT          1e-7;
+deltaT          1e-07;
 
 writeControl    adjustableRunTime;
 
@@ -48,11 +48,15 @@ runTimeModifiable yes;
 adjustTimeStep  yes;
 
 maxCo           0.5;
+
 maxDi           1;
+
 maxAlphaCo      1;
 
 functions
 {
-    #includeFunc ExaCA
+    #includeFunc    ExaCA
 }
+
+
 // ************************************************************************* //
diff --git a/tutorials/multiBeam/Allclean b/tutorials/multiBeam/Allclean
@@ -5,6 +5,7 @@ cd ${0%/*} || exit 1    # Run from this directory
 . $WM_PROJECT_DIR/bin/tools/CleanFunctions
 
 cleanCase
-
-rm -rf ExaCA
+#------------------------------------------------------------------------------
+rm -rf ExaCA/Output*
+rm -rf ExaCA/time-temperature.csv
 #------------------------------------------------------------------------------