Extend HDF5 test to 8 GPUs

examples/hdf5Test/documentation.yml

@@ -26,7 +26,7 @@ aliases:
 # Run-Time Tests
 tests:
   - name: Checkpoint
-    description: Test only that checkpointing works
+    description: Test that checkpointing works
     cmakeFlag: 0
     cfgFile:   0001gpus.cfg
     post-run: &cpPostRun
@@ -35,7 +35,7 @@ tests:
         - *validateOpenPMD
         - python3 "${TEST_INSTALL_PATH}/testData/validateCheckpoint.py"
   - name: Restart
-    description: Restart from the last checkpoint of BaseCheckpoint
+    description: Restart from the last checkpoint of Checkpoint
     cmakeFlag: 0
     cfgFile:   0001gpusRestart.cfg
     dependency: Checkpoint
@@ -59,12 +59,41 @@ tests:
     description: Uses HDF5 files to interpolate the field between timesteps
     cmakeFlag: 2
     cfgFile:   0001gpusHDF5Field.cfg
-    pre-run:
+    pre-run: &createHDF5Field
         - mkdir "${TEST_INSTALL_PATH}/testData/$TEST_NAME"
         - python3 "${TEST_INSTALL_PATH}/testData/createHDF5Filesets.py" --folder "${TEST_INSTALL_PATH}/testData/$TEST_NAME" --size ${TEST_GRID_SIZE} --dt "2e-16" --time "1.5e-15"
-    post-run:
+    post-run: &validateHDF5Field
         - *validateOpenPMD
         - python3 "${TEST_INSTALL_PATH}/testData/validateCheckpoint.py"
         - python3 "${TEST_INSTALL_PATH}/testData/validateFieldInterpolator.py"
 
+# Same but for 8 GPUs
+  - name: Checkpoint8
+    description: Test that checkpointing works for 8 GPUs
+    cmakeFlag: 0
+    cfgFile:   0008gpus.cfg
+    post-run: *cpPostRun
+  - name: Restart8
+    description: Restart from the last checkpoint of Checkpoint8
+    cmakeFlag: 0
+    cfgFile:   0008gpusRestart.cfg
+    dependency: Checkpoint
+    post-run: *restartPostRun
+  - name: CheckpointCt8
+    description: Test that checkpointing works for 8 GPUs (ParticleCounter)
+    cmakeFlag: 1
+    cfgFile:   0008gpus.cfg
+    post-run: *cpPostRun
+  - name: RestartCt8
+    description: Restart from the last checkpoint of CheckpointCt8 (ParticleCounter)
+    cmakeFlag: 1
+    cfgFile:   0008gpusRestart.cfg
+    dependency: CheckpointCt
+    post-run: *restartPostRun
+  - name: HDF5FieldInterpolator8
+    description: Uses HDF5 files to interpolate the field between timesteps on 8 GPUs
+    cmakeFlag: 2
+    cfgFile:   0008gpusHDF5Field.cfg
+    pre-run: *createHDF5Field
+    post-run: *validateHDF5Field
 

examples/hdf5Test/submit/0008gpus.cfg.in

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# variables which are created by tbg
+# TBG_jobName
+# TBG_jobNameShort
+# TBG_cfgPath
+# TBG_cfgFile
+# TBG_projectPath
+# TBG_dstPath
+
+#################################
+## Section: Required Variables ##
+#################################
+
+TBG_wallTime="0:10:00"
+
+TBG_gpu_x=2
+TBG_gpu_y=2
+TBG_gpu_z=2
+
+TBG_gridSize="-g 64 128 128"
+TBG_steps="-s 100"
+
+TBG_program="@PROJECT_NAME@"
+
+#################################
+## Section: Optional Variables ##
+#################################
+
+TBG_plugins="--detSize 1400 900                           \
+             !TBG_globalSeed                              \
+             --p_position.period 0                        \
+             --DensityField_printSlice.period 0           \
+             --PhotonDetector_print.period 0              \
+             --checkpoints 50                             \
+             "
+
+#################################
+## Section: Program Parameters ##
+#################################
+
+TBG_devices="-d !TBG_gpu_x !TBG_gpu_y !TBG_gpu_z"
+
+TBG_programParams="!TBG_devices      \
+                   !TBG_gridSize     \
+                   !TBG_steps        \
+                   !TBG_plugins | tee output"
+
+# TOTAL number of GPUs
+TBG_tasks="$(( TBG_gpu_x * TBG_gpu_y * TBG_gpu_z ))"
+
+"$TBG_cfgPath"/submitAction.sh

examples/hdf5Test/submit/0008gpusHDF5Field.cfg.in

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# variables which are created by tbg
+# TBG_jobName
+# TBG_jobNameShort
+# TBG_cfgPath
+# TBG_cfgFile
+# TBG_projectPath
+# TBG_dstPath
+
+#################################
+## Section: Required Variables ##
+#################################
+
+TBG_wallTime="0:10:00"
+
+TBG_gpu_x=2
+TBG_gpu_y=2
+TBG_gpu_z=2
+
+TBG_gridSize="-g 64 128 128"
+TBG_steps="-s 100"
+
+TBG_program="@PROJECT_NAME@"
+
+#################################
+## Section: Optional Variables ##
+#################################
+
+TBG_HDF5_FILE="${TEST_INSTALL_PATH}/testData/$TEST_NAME/field"
+
+TBG_plugins="--detSize 1400 900                           \
+             !TBG_globalSeed                              \
+             --p_position.period 0                        \
+             --DensityField_printSlice.period 0           \
+             --DensityField.hdf5File \"!TBG_HDF5_FILE\"   \
+             --PhotonDetector_print.period 0              \
+             --checkpoints 50                             \
+             "
+
+#################################
+## Section: Program Parameters ##
+#################################
+
+TBG_devices="-d !TBG_gpu_x !TBG_gpu_y !TBG_gpu_z"
+
+TBG_programParams="!TBG_devices      \
+                   !TBG_gridSize     \
+                   !TBG_steps        \
+                   !TBG_plugins | tee output"
+
+# TOTAL number of GPUs
+TBG_tasks="$(( TBG_gpu_x * TBG_gpu_y * TBG_gpu_z ))"
+
+"$TBG_cfgPath"/submitAction.sh

examples/hdf5Test/submit/0008gpusRestart.cfg.in

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# variables which are created by tbg
+# TBG_jobName
+# TBG_jobNameShort
+# TBG_cfgPath
+# TBG_cfgFile
+# TBG_projectPath
+# TBG_dstPath
+
+#################################
+## Section: Required Variables ##
+#################################
+
+TBG_wallTime="0:10:00"
+
+TBG_gpu_x=2
+TBG_gpu_y=2
+TBG_gpu_z=2
+
+TBG_gridSize="-g 64 128 128"
+TBG_steps="-s 150"
+
+TBG_program="@PROJECT_NAME@"
+
+#################################
+## Section: Optional Variables ##
+#################################
+
+TBG_plugins="--detSize 1400 900                           \
+             !TBG_globalSeed                              \
+             --p_position.period 0                        \
+             --DensityField_printSlice.period 0           \
+             --PhotonDetector_print.period 0              \
+             --checkpoints 50                             \
+             --restart                                    \
+             --restart-step 50                            \
+             --restart-directory \"${TEST_SIMOUTPUT_PATH/_Restart/_Checkpoint}\"/checkpoints \
+             "
+
+#################################
+## Section: Program Parameters ##
+#################################
+
+TBG_devices="-d !TBG_gpu_x !TBG_gpu_y !TBG_gpu_z"
+
+TBG_programParams="!TBG_devices      \
+                   !TBG_gridSize     \
+                   !TBG_steps        \
+                   !TBG_plugins | tee output"
+
+# TOTAL number of GPUs
+TBG_tasks="$(( TBG_gpu_x * TBG_gpu_y * TBG_gpu_z ))"
+
+"$TBG_cfgPath"/submitAction.sh