diff --git a/.github/workflows/checkpoint-restart.yml b/.github/workflows/checkpoint-restart.yml new file mode 100644 index 000000000..8cb451613 --- /dev/null +++ b/.github/workflows/checkpoint-restart.yml @@ -0,0 +1,64 @@ +name: CheckpointRestart + +on: + push: + branches: [ development ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ development ] + merge_group: + branches: [ development ] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-final-checkpoint-restart + cancel-in-progress: true + +env: + # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) + BUILD_TYPE: Release + +jobs: + test: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Create Build Environment + run: cmake -E make_directory ${{runner.workspace}}/build + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install gcc-11 g++-11 python3-dev python3-numpy python3-matplotlib python3-pip libopenmpi-dev libhdf5-mpi-dev + + - name: Build PlotfileTools + shell: bash + working-directory: ${{github.workspace}}/extern/amrex/Tools/Plotfile + run: make -j4 + + - name: Configure CMake + shell: bash + working-directory: ${{runner.workspace}}/build + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_C_COMPILER=gcc-11 -DCMAKE_CXX_COMPILER=g++-11 -DAMReX_SPACEDIM=3 + + - name: Build + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute the build. You can specify a specific target with "--target " + run: cmake --build . --config $BUILD_TYPE --parallel 4 --target test_hydro3d_blast + + - name: Checkpoint/Restart Test + working-directory: ${{github.workspace}}/tests + shell: bash + env: + BUILD_DIR: ${{runner.workspace}}/build + PLOTFILETOOLS_DIR: ${{github.workspace}}/extern/amrex/Tools/Plotfile + run: ./checkpoint_restart_test.sh + + - name: Upload output + if: always() + uses: actions/upload-artifact@v4 + with: + name: checkpoint-restart-results + path: ${{github.workspace}}/tests diff --git a/src/simulation.hpp b/src/simulation.hpp index 0e4e61d6b..6d0d530a5 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -908,11 +908,15 @@ template void AMRSimulation::evolve() WritePlotFile(); } + // IMPORTANT: this MUST be written *after* the plotfile to avoid corruption: + // https://github.com/quokka-astro/quokka/issues/554 if (checkpointTimeInterval_ > 0 && next_chk_file_time <= cur_time) { next_chk_file_time += checkpointTimeInterval_; WriteCheckpointFile(); } + // IMPORTANT: this MUST be written *after* the plotfile to avoid corruption: + // https://github.com/quokka-astro/quokka/issues/554 if (checkpointInterval_ > 0 && (step + 1) % checkpointInterval_ == 0) { last_chk_file_step = step + 1; WriteCheckpointFile(); @@ -958,11 +962,6 @@ template void AMRSimulation::evolve() } amrex::Print() << '\n'; - // write final checkpoint - if (checkpointInterval_ > 0 && istep[0] > last_chk_file_step) { - WriteCheckpointFile(); - } - // write final plotfile if (plotfileInterval_ > 0 && istep[0] > last_plot_file_step) { WritePlotFile(); @@ -978,6 +977,13 @@ template void AMRSimulation::evolve() WriteStatisticsFile(); } + // write final checkpoint + // IMPORTANT: this MUST be written *after* the plotfile to avoid corruption: + // https://github.com/quokka-astro/quokka/issues/554 + if (checkpointInterval_ > 0 && istep[0] > last_chk_file_step) { + WriteCheckpointFile(); + } + #ifdef AMREX_USE_ASCENT // close Ascent ascent_.close(); diff --git a/tests/blast_32.in b/tests/blast_32.in new file mode 100644 index 000000000..ee3f702f7 --- /dev/null +++ b/tests/blast_32.in @@ -0,0 +1,24 @@ +# ***************************************************************** +# Problem size and geometry +# ***************************************************************** +geometry.prob_lo = 0.0 0.0 0.0 +geometry.prob_hi = 1.2 1.2 1.2 +geometry.is_periodic = 0 0 0 + +# ***************************************************************** +# VERBOSITY +# ***************************************************************** +amr.v = 0 # verbosity in Amr + +# ***************************************************************** +# Resolution and refinement +# ***************************************************************** +amr.n_cell = 32 32 32 +amr.max_level = 0 # number of levels = max_level + 1 +amr.max_grid_size = 16 # at least 128 for GPUs +amr.blocking_factor = 64 # grid size must be divisible by this +amr.n_error_buf = 3 # minimum 3 cell buffer around tagged cells +amr.grid_eff = 0.7 # default + +do_reflux = 0 +do_subcycle = 0 diff --git a/tests/checkpoint_restart_test.sh b/tests/checkpoint_restart_test.sh new file mode 100755 index 000000000..d83f61942 --- /dev/null +++ b/tests/checkpoint_restart_test.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -x + +$BUILD_DIR/src/HydroBlast3D/test_hydro3d_blast blast_32.in max_walltime=0:00:10 plotfile_interval=100 checkpoint_interval=100 +$BUILD_DIR/src/HydroBlast3D/test_hydro3d_blast blast_32.in restartfile=last_chk max_timesteps=1 plotfile_interval=100 checkpoint_interval=100 + +old_plotfile=`ls -1drt plt*.old.* | head -1` +plotfile=${old_plotfile%.old.*} + +$PLOTFILETOOLS_DIR/fcompare.gnu.ex $plotfile $old_plotfile