From 1eb292cfc1e1ecce3e5ee471c7e7c52002c67036 Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Thu, 28 Mar 2024 15:38:29 -0500 Subject: [PATCH 01/23] Trick prints warning for checkpoints outside of freeze mode. --- .../CheckPointRestart/CheckPointRestart.cpp | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index 002f12896..675c12d29 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -22,6 +22,18 @@ Trick::CheckPointRestart * the_cpr ; +char * sim_mode_to_string(SIM_MODE mode) { + switch (mode) + { + case Initialization: return "INIT"; + case Run: return "RUN"; + case Step: return "STEP"; + case Freeze: return "FREEZE"; + case ExitMode: return "EXIT"; + default: return "INVALID"; + } +} + Trick::CheckPointRestart::CheckPointRestart() { int num_classes = 0 ; @@ -171,6 +183,18 @@ int Trick::CheckPointRestart::do_checkpoint(std::string file_name, bool print_st JobData * curr_job ; pid_t pid; + SIM_MODE mode; + + mode = the_exec->get_mode(); + + if (mode != SIM_MODE::Freeze) { + std::string msg_format = "Do not dump a checkpoint outside of Freeze Mode. "; + msg_format += "Current Mode: %s (%d)\n"; + message_publish(MSG_WARNING, msg_format.c_str(), + sim_mode_to_string(mode), mode); + return(1); + } + if ( ! file_name.compare("") ) { std::stringstream file_name_stream ; @@ -293,7 +317,19 @@ int Trick::CheckPointRestart::safestore_checkpoint() { } void Trick::CheckPointRestart::load_checkpoint(std::string file_name) { - load_checkpoint_file_name = file_name ; + SIM_MODE mode = the_exec->get_mode(); + + if (mode != SIM_MODE::Freeze) { + std::string msg_format = "Do not load checkpoint outside of Freeze Mode. "; + msg_format += "Current Mode: %s (%d)\n"; + + message_publish(MSG_WARNING, msg_format.c_str(), + file_name.c_str(), sim_mode_to_string(mode), mode); + + load_checkpoint_file_name = ""; + } else { + load_checkpoint_file_name = file_name ; + } } void Trick::CheckPointRestart::load_checkpoint(std::string file_name, bool stls_on) { From b7a65bb89b20332b9dd25222b5fa3415ea511a4d Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Thu, 28 Mar 2024 16:28:15 -0500 Subject: [PATCH 02/23] Fixed small syntax error --- .../sim_services/CheckPointRestart/CheckPointRestart.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index 675c12d29..18d7bc8ac 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -319,7 +319,7 @@ int Trick::CheckPointRestart::safestore_checkpoint() { void Trick::CheckPointRestart::load_checkpoint(std::string file_name) { SIM_MODE mode = the_exec->get_mode(); - if (mode != SIM_MODE::Freeze) { + if (mode != Freeze) { std::string msg_format = "Do not load checkpoint outside of Freeze Mode. "; msg_format += "Current Mode: %s (%d)\n"; From 9b9c8307183440bffca2c03444d67144b32894ef Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Thu, 28 Mar 2024 16:40:49 -0500 Subject: [PATCH 03/23] Fixed small syntax error --- .../sim_services/CheckPointRestart/CheckPointRestart.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index 18d7bc8ac..0d2eaddd7 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -187,7 +187,7 @@ int Trick::CheckPointRestart::do_checkpoint(std::string file_name, bool print_st mode = the_exec->get_mode(); - if (mode != SIM_MODE::Freeze) { + if (mode != Freeze) { std::string msg_format = "Do not dump a checkpoint outside of Freeze Mode. "; msg_format += "Current Mode: %s (%d)\n"; message_publish(MSG_WARNING, msg_format.c_str(), From 7cb2e5f0f1b64311b2f54c15b572b90b3bf4ea96 Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 29 Mar 2024 14:34:46 -0500 Subject: [PATCH 04/23] Warnings no longer abort checkpoint actions due to test conflict --- .../CheckPointRestart/CheckPointRestart.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index 0d2eaddd7..e690cd2db 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -188,11 +188,10 @@ int Trick::CheckPointRestart::do_checkpoint(std::string file_name, bool print_st mode = the_exec->get_mode(); if (mode != Freeze) { - std::string msg_format = "Do not dump a checkpoint outside of Freeze Mode. "; + std::string msg_format = "WARNING: Saving a checkpoint outside of 'Freeze Mode' causes undefined behavior. "; msg_format += "Current Mode: %s (%d)\n"; message_publish(MSG_WARNING, msg_format.c_str(), sim_mode_to_string(mode), mode); - return(1); } @@ -320,16 +319,14 @@ void Trick::CheckPointRestart::load_checkpoint(std::string file_name) { SIM_MODE mode = the_exec->get_mode(); if (mode != Freeze) { - std::string msg_format = "Do not load checkpoint outside of Freeze Mode. "; + std::string msg_format = "WARNING: Loading a checkpoint outside of 'Freeze Mode' causes undefined behavior. "; msg_format += "Current Mode: %s (%d)\n"; message_publish(MSG_WARNING, msg_format.c_str(), file_name.c_str(), sim_mode_to_string(mode), mode); - - load_checkpoint_file_name = ""; - } else { - load_checkpoint_file_name = file_name ; - } + } + + load_checkpoint_file_name = file_name ; } void Trick::CheckPointRestart::load_checkpoint(std::string file_name, bool stls_on) { From f71ce442407c8d882621730946b9ca1d4b47c202 Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Tue, 2 Apr 2024 11:48:36 -0500 Subject: [PATCH 05/23] Moved sim_mode_to_string function to a more generic place. --- include/trick/sim_mode.h | 12 ++++++++++++ .../CheckPointRestart/CheckPointRestart.cpp | 12 ------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/trick/sim_mode.h b/include/trick/sim_mode.h index 1bdbd2ace..79fa56682 100644 --- a/include/trick/sim_mode.h +++ b/include/trick/sim_mode.h @@ -38,4 +38,16 @@ typedef enum { } SIM_MODE ; +char * sim_mode_to_string(SIM_MODE mode) { + switch (mode) + { + case Initialization: return "Initialization"; + case Run: return "Run"; + case Step: return "Step"; + case Freeze: return "Freeze"; + case ExitMode: return "ExitMode"; + default: return "InvalidMode"; + } +} + #endif diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index e690cd2db..46b6000f6 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -22,18 +22,6 @@ Trick::CheckPointRestart * the_cpr ; -char * sim_mode_to_string(SIM_MODE mode) { - switch (mode) - { - case Initialization: return "INIT"; - case Run: return "RUN"; - case Step: return "STEP"; - case Freeze: return "FREEZE"; - case ExitMode: return "EXIT"; - default: return "INVALID"; - } -} - Trick::CheckPointRestart::CheckPointRestart() { int num_classes = 0 ; From 0218314b8279a486527b8e6af83bfa1abbb549ce Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 5 Apr 2024 10:29:46 -0500 Subject: [PATCH 06/23] Fixing simModeCharString --- include/trick/sim_mode.h | 12 +----------- .../VariableServer/simModeCharString.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 trick_source/sim_services/VariableServer/simModeCharString.c diff --git a/include/trick/sim_mode.h b/include/trick/sim_mode.h index 79fa56682..31b7edb2b 100644 --- a/include/trick/sim_mode.h +++ b/include/trick/sim_mode.h @@ -38,16 +38,6 @@ typedef enum { } SIM_MODE ; -char * sim_mode_to_string(SIM_MODE mode) { - switch (mode) - { - case Initialization: return "Initialization"; - case Run: return "Run"; - case Step: return "Step"; - case Freeze: return "Freeze"; - case ExitMode: return "ExitMode"; - default: return "InvalidMode"; - } -} +const char * simModeCharString(SIM_MODE mode); #endif diff --git a/trick_source/sim_services/VariableServer/simModeCharString.c b/trick_source/sim_services/VariableServer/simModeCharString.c new file mode 100644 index 000000000..0b193d1f9 --- /dev/null +++ b/trick_source/sim_services/VariableServer/simModeCharString.c @@ -0,0 +1,14 @@ + +#include "trick/sim_mode.h" + +const char * simModeCharString(SIM_MODE mode) { + switch (mode) + { + case Initialization: return "Initialization"; + case Run: return "Run"; + case Step: return "Step"; + case Freeze: return "Freeze"; + case ExitMode: return "ExitMode"; + default: return "InvalidMode"; + } +} \ No newline at end of file From 360431259f1c4cda8e88f424dae315bb26c803d0 Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 5 Apr 2024 10:30:19 -0500 Subject: [PATCH 07/23] Cancel checkpoint if sim is running --- .../CheckPointRestart/CheckPointRestart.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index 46b6000f6..a5384775c 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -175,11 +175,13 @@ int Trick::CheckPointRestart::do_checkpoint(std::string file_name, bool print_st mode = the_exec->get_mode(); - if (mode != Freeze) { + if (mode == Run) { std::string msg_format = "WARNING: Saving a checkpoint outside of 'Freeze Mode' causes undefined behavior. "; msg_format += "Current Mode: %s (%d)\n"; message_publish(MSG_WARNING, msg_format.c_str(), - sim_mode_to_string(mode), mode); + simModeCharString(mode), mode); + + return 0; } @@ -306,12 +308,14 @@ int Trick::CheckPointRestart::safestore_checkpoint() { void Trick::CheckPointRestart::load_checkpoint(std::string file_name) { SIM_MODE mode = the_exec->get_mode(); - if (mode != Freeze) { + if (mode == Run) { std::string msg_format = "WARNING: Loading a checkpoint outside of 'Freeze Mode' causes undefined behavior. "; msg_format += "Current Mode: %s (%d)\n"; message_publish(MSG_WARNING, msg_format.c_str(), - file_name.c_str(), sim_mode_to_string(mode), mode); + file_name.c_str(), simModeCharString(mode), mode); + + return; } load_checkpoint_file_name = file_name ; From ae586c1d3c042053170b2fed62976a0983a055d9 Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 5 Apr 2024 10:31:04 -0500 Subject: [PATCH 08/23] Define dependencies for simModeCharString --- trick_source/sim_services/VariableServer/Makefile_deps | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trick_source/sim_services/VariableServer/Makefile_deps b/trick_source/sim_services/VariableServer/Makefile_deps index 8d83bab4e..d585865ae 100644 --- a/trick_source/sim_services/VariableServer/Makefile_deps +++ b/trick_source/sim_services/VariableServer/Makefile_deps @@ -1,3 +1,6 @@ + +object_${TRICK_HOST_CPU}/simModeCharString.o: simModeCharString.c \ + ${TRICK_HOME}/include/trick/sim_mode.h object_${TRICK_HOST_CPU}/VariableServerSessionThread_loop.o: VariableServerSessionThread_loop.cpp \ ${TRICK_HOME}/include/trick/VariableServer.hh \ ${TRICK_HOME}/include/trick/tc.h \ From 92fbacb8cc2be9f6d6c08decfed9691af3c268af Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 17 May 2024 16:49:49 -0500 Subject: [PATCH 09/23] Fixing simModeCharString(SIM_MODE) --- include/trick/sim_mode.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/trick/sim_mode.h b/include/trick/sim_mode.h index 31b7edb2b..51ff2bd76 100644 --- a/include/trick/sim_mode.h +++ b/include/trick/sim_mode.h @@ -14,6 +14,10 @@ #ifndef SIMMODE_HH #define SIMMODE_HH +#ifdef __cplusplus +extern "C" { +#endif + typedef enum { NoCmd = 0 , /* NoCmd */ @@ -40,4 +44,8 @@ typedef enum { const char * simModeCharString(SIM_MODE mode); +#ifdef __cplusplus +} +#endif + #endif From 5b4e82c229872e6f94795289f80bcff163f976ac Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 17 May 2024 16:51:39 -0500 Subject: [PATCH 10/23] Update checkpointing and the freeze loop --- .../CheckPointRestart/CheckPointRestart.cpp | 19 +++++++++++++++++-- .../Executive/Executive_freeze_loop.cpp | 7 +++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index a5384775c..d05e4ea67 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -19,6 +19,7 @@ #include "trick/message_proto.h" #include "trick/message_type.h" #include "trick/TrickConstant.hh" +#include "trick/sim_mode.h" Trick::CheckPointRestart * the_cpr ; @@ -109,6 +110,11 @@ int Trick::CheckPointRestart::find_write_checkpoint_jobs(std::string sim_object_ return(0) ; } +/** + * @brief Schedule a checkpoint to be written at a given time. + * @param in_time The time the checkpoint should be dumped + * @see write_checkpoint() + */ int Trick::CheckPointRestart::checkpoint(double in_time) { long long curr_time = exec_get_time_tics() ; @@ -121,6 +127,8 @@ int Trick::CheckPointRestart::checkpoint(double in_time) { if ( new_time < write_checkpoint_job->next_tics ) { write_checkpoint_job->next_tics = new_time ; } + + the_exec->freeze(in_time); //std::cout << "\033[33mSET CHECKPOINT TIME " << in_time << " " << new_time << "\033[0m" << std::endl ; } else { message_publish(MSG_ERROR, "Checkpoint time specified in the past. specified %f, current_time %f\n", @@ -240,6 +248,10 @@ int Trick::CheckPointRestart::do_checkpoint(std::string file_name, bool print_st return 0 ; } +/** + * @brief Writes a scheduled checkpoint if it is the correct time. + * @see checkpoint(double in_time) + */ int Trick::CheckPointRestart::write_checkpoint() { long long curr_time = exec_get_time_tics() ; @@ -265,6 +277,7 @@ int Trick::CheckPointRestart::write_checkpoint() { checkpoint( chk_name_stream.str() ); + the_exec->run(); } return(0) ; @@ -315,10 +328,11 @@ void Trick::CheckPointRestart::load_checkpoint(std::string file_name) { message_publish(MSG_WARNING, msg_format.c_str(), file_name.c_str(), simModeCharString(mode), mode); - return; + // return; } load_checkpoint_file_name = file_name ; + the_exec->freeze(); } void Trick::CheckPointRestart::load_checkpoint(std::string file_name, bool stls_on) { @@ -331,7 +345,7 @@ int Trick::CheckPointRestart::load_checkpoint_job() { JobData * curr_job ; struct stat temp_buf ; - if ( ! load_checkpoint_file_name.empty() ) { + if ( ! load_checkpoint_file_name.empty() && the_exec->get_mode() != Run) { if ( stat( load_checkpoint_file_name.c_str() , &temp_buf) == 0 ) { preload_checkpoint_queue.reset_curr_index() ; @@ -363,6 +377,7 @@ int Trick::CheckPointRestart::load_checkpoint_job() { message_publish(MSG_INFO, "Could not find checkpoint file %s.\n", load_checkpoint_file_name.c_str()) ; } load_checkpoint_file_name.clear() ; + the_exec->run(); } return(0) ; diff --git a/trick_source/sim_services/Executive/Executive_freeze_loop.cpp b/trick_source/sim_services/Executive/Executive_freeze_loop.cpp index 19d5ef4bd..77ead7416 100644 --- a/trick_source/sim_services/Executive/Executive_freeze_loop.cpp +++ b/trick_source/sim_services/Executive/Executive_freeze_loop.cpp @@ -4,10 +4,13 @@ #include "trick/Executive.hh" #include "trick/ExecutiveException.hh" +#include "trick/CheckPointRestart.hh" #include "trick/exec_proto.h" #include "trick/message_proto.h" #include "trick/message_type.h" +extern Trick::CheckPointRestart * the_cpr ; + /** @details -# Set the mode to Freeze. Requirement [@ref r_exec_mode_2] @@ -32,6 +35,10 @@ int Trick::Executive::freeze_loop() { } message_publish(MSG_INFO, "Freeze ON. Simulation time holding at %f seconds.\n" , get_sim_time()) ; + + if (!the_cpr->checkpoint_times.empty()) { + the_cpr->write_checkpoint(); + } while (mode == Freeze) { From 2638f88041d5c6392247415953e9e6171c8d8e2d Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 17 May 2024 16:55:55 -0500 Subject: [PATCH 11/23] Correct the ref_logs to properly reflect the expected outcomes. --- test/SIM_checkpoint_data_recording/RUN_test1/ref_log_foo.csv | 1 + test/SIM_checkpoint_data_recording/RUN_test3/ref_log_foo.csv | 1 + test/SIM_checkpoint_data_recording/RUN_test4/ref_log_foo.csv | 1 + test/SIM_checkpoint_data_recording/RUN_test5/ref_log_foo.csv | 1 + test/SIM_checkpoint_data_recording/RUN_test6/ref_log_foo2.csv | 1 + .../RUN_test7/ref_log_fooChange.csv | 1 + .../RUN_test8/ref_log_fooChange2.csv | 1 + 7 files changed, 7 insertions(+) diff --git a/test/SIM_checkpoint_data_recording/RUN_test1/ref_log_foo.csv b/test/SIM_checkpoint_data_recording/RUN_test1/ref_log_foo.csv index 37c97556b..75a5c16e7 100644 --- a/test/SIM_checkpoint_data_recording/RUN_test1/ref_log_foo.csv +++ b/test/SIM_checkpoint_data_recording/RUN_test1/ref_log_foo.csv @@ -1,4 +1,5 @@ sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1} + 5,6,12 5.1,6,12 5.2,6,12 5.3,6,12 diff --git a/test/SIM_checkpoint_data_recording/RUN_test3/ref_log_foo.csv b/test/SIM_checkpoint_data_recording/RUN_test3/ref_log_foo.csv index 37c97556b..75a5c16e7 100644 --- a/test/SIM_checkpoint_data_recording/RUN_test3/ref_log_foo.csv +++ b/test/SIM_checkpoint_data_recording/RUN_test3/ref_log_foo.csv @@ -1,4 +1,5 @@ sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1} + 5,6,12 5.1,6,12 5.2,6,12 5.3,6,12 diff --git a/test/SIM_checkpoint_data_recording/RUN_test4/ref_log_foo.csv b/test/SIM_checkpoint_data_recording/RUN_test4/ref_log_foo.csv index 13327df84..b70298737 100644 --- a/test/SIM_checkpoint_data_recording/RUN_test4/ref_log_foo.csv +++ b/test/SIM_checkpoint_data_recording/RUN_test4/ref_log_foo.csv @@ -1,4 +1,5 @@ sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1} + 2,3,6 2.1,3,6 2.2,3,6 2.3,3,6 diff --git a/test/SIM_checkpoint_data_recording/RUN_test5/ref_log_foo.csv b/test/SIM_checkpoint_data_recording/RUN_test5/ref_log_foo.csv index 306de1b40..490689651 100644 --- a/test/SIM_checkpoint_data_recording/RUN_test5/ref_log_foo.csv +++ b/test/SIM_checkpoint_data_recording/RUN_test5/ref_log_foo.csv @@ -1,4 +1,5 @@ sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1} + 7,8,16 7.1,8,16 7.2,8,16 7.3,8,16 diff --git a/test/SIM_checkpoint_data_recording/RUN_test6/ref_log_foo2.csv b/test/SIM_checkpoint_data_recording/RUN_test6/ref_log_foo2.csv index f7b15a2b7..2ee326def 100644 --- a/test/SIM_checkpoint_data_recording/RUN_test6/ref_log_foo2.csv +++ b/test/SIM_checkpoint_data_recording/RUN_test6/ref_log_foo2.csv @@ -1,4 +1,5 @@ sys.exec.out.time {s},testSimObject.my_foo.b {1} + 7,16 8,18 9,20 10,22 diff --git a/test/SIM_checkpoint_data_recording/RUN_test7/ref_log_fooChange.csv b/test/SIM_checkpoint_data_recording/RUN_test7/ref_log_fooChange.csv index 31352b3bf..6abcd599d 100644 --- a/test/SIM_checkpoint_data_recording/RUN_test7/ref_log_fooChange.csv +++ b/test/SIM_checkpoint_data_recording/RUN_test7/ref_log_fooChange.csv @@ -1,4 +1,5 @@ sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1},testSimObject.my_foo.q {1} + 5,6,12,2 8,9,18,3 11,12,24,4 14,15,30,5 diff --git a/test/SIM_checkpoint_data_recording/RUN_test8/ref_log_fooChange2.csv b/test/SIM_checkpoint_data_recording/RUN_test8/ref_log_fooChange2.csv index 5692f5342..96eba9af7 100644 --- a/test/SIM_checkpoint_data_recording/RUN_test8/ref_log_fooChange2.csv +++ b/test/SIM_checkpoint_data_recording/RUN_test8/ref_log_fooChange2.csv @@ -1,4 +1,5 @@ sys.exec.out.time {s},testSimObject.my_foo.q {1} + 5,2 8,3 11,4 14,5 From ac8244421fe89655dce04d76c2f77e0aa769be1b Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 17 May 2024 16:56:57 -0500 Subject: [PATCH 12/23] Updated SIM_stls test to pass with new checkpoint restrictions --- test/SIM_stls/RUN_test/setup.py | 2 +- test/SIM_stls/RUN_test/unit_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/SIM_stls/RUN_test/setup.py b/test/SIM_stls/RUN_test/setup.py index 208d871f9..03bc6ddcf 100644 --- a/test/SIM_stls/RUN_test/setup.py +++ b/test/SIM_stls/RUN_test/setup.py @@ -5,7 +5,7 @@ def main(): trick.exec_set_job_onoff("the_object.stlc.test", 1, False) trick.exec_set_job_onoff("the_object.stlc.print", 1, False) - trick.add_read( 0.5, 'trick.checkpoint("chkpnt_in")') + trick.checkpoint(0.1) trick.exec_set_freeze_frame(0.10) trick.stop(1.0) diff --git a/test/SIM_stls/RUN_test/unit_test.py b/test/SIM_stls/RUN_test/unit_test.py index 16c31069f..05ce98843 100644 --- a/test/SIM_stls/RUN_test/unit_test.py +++ b/test/SIM_stls/RUN_test/unit_test.py @@ -3,7 +3,7 @@ def main(): - trick.load_checkpoint("RUN_test/chkpnt_in") + trick.load_checkpoint("RUN_test/chkpnt_0.100000") trick.load_checkpoint_job() trick.exec_set_job_onoff("the_object.stlc.addData", 1, False) From 08ed1e2008595a0936c982472ccf39301e4fb1ad Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 31 May 2024 13:49:08 -0500 Subject: [PATCH 13/23] Limit automatic freeze and re-run to when simulation is in run mode --- include/trick/CheckPointRestart.hh | 4 ++++ .../sim_services/CheckPointRestart/CheckPointRestart.cpp | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/include/trick/CheckPointRestart.hh b/include/trick/CheckPointRestart.hh index 4201d7464..8c0a597c6 100644 --- a/include/trick/CheckPointRestart.hh +++ b/include/trick/CheckPointRestart.hh @@ -22,6 +22,10 @@ namespace Trick { */ class CheckPointRestart : public Trick::Scheduler { + private: + /** Flag to track if an automatic freeze has been triggered */ + bool auto_freeze = false; /* ** */ + protected: /** queue to hold jobs to be called before a checkpoint is dumped. */ Trick::ScheduledJobQueue checkpoint_queue ; /* ** */ diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index d05e4ea67..4c60d15e4 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -328,11 +328,11 @@ void Trick::CheckPointRestart::load_checkpoint(std::string file_name) { message_publish(MSG_WARNING, msg_format.c_str(), file_name.c_str(), simModeCharString(mode), mode); - // return; + the_exec->freeze(); + auto_freeze = true; } load_checkpoint_file_name = file_name ; - the_exec->freeze(); } void Trick::CheckPointRestart::load_checkpoint(std::string file_name, bool stls_on) { @@ -377,7 +377,7 @@ int Trick::CheckPointRestart::load_checkpoint_job() { message_publish(MSG_INFO, "Could not find checkpoint file %s.\n", load_checkpoint_file_name.c_str()) ; } load_checkpoint_file_name.clear() ; - the_exec->run(); + if(auto_freeze) the_exec->run(); } return(0) ; From 3520b1efc6ff21a9174ed538049c97f29f7afada Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Thu, 20 Jun 2024 10:13:53 -0500 Subject: [PATCH 14/23] Restored specific checkpoint time in SIM_stls input file --- test/SIM_stls/RUN_test/setup.py | 2 +- test/SIM_stls/RUN_test/unit_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/SIM_stls/RUN_test/setup.py b/test/SIM_stls/RUN_test/setup.py index 03bc6ddcf..21164ad1b 100644 --- a/test/SIM_stls/RUN_test/setup.py +++ b/test/SIM_stls/RUN_test/setup.py @@ -5,7 +5,7 @@ def main(): trick.exec_set_job_onoff("the_object.stlc.test", 1, False) trick.exec_set_job_onoff("the_object.stlc.print", 1, False) - trick.checkpoint(0.1) + trick.checkpoint(0.5) trick.exec_set_freeze_frame(0.10) trick.stop(1.0) diff --git a/test/SIM_stls/RUN_test/unit_test.py b/test/SIM_stls/RUN_test/unit_test.py index 05ce98843..fefddf4f6 100644 --- a/test/SIM_stls/RUN_test/unit_test.py +++ b/test/SIM_stls/RUN_test/unit_test.py @@ -3,7 +3,7 @@ def main(): - trick.load_checkpoint("RUN_test/chkpnt_0.100000") + trick.load_checkpoint("RUN_test/chkpnt_0.500000") trick.load_checkpoint_job() trick.exec_set_job_onoff("the_object.stlc.addData", 1, False) From 61e92940dfed4445e3a5a355f205cec4a74d65a7 Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Thu, 20 Jun 2024 13:45:30 -0500 Subject: [PATCH 15/23] Added ability to checkpoint to a named file at a specific time. --- include/trick/CheckPointRestart.hh | 4 +++- .../CheckPointRestart/CheckPointRestart.cpp | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/include/trick/CheckPointRestart.hh b/include/trick/CheckPointRestart.hh index 8c0a597c6..f4c17d849 100644 --- a/include/trick/CheckPointRestart.hh +++ b/include/trick/CheckPointRestart.hh @@ -9,6 +9,7 @@ #include #include #include +#include #include "trick/Scheduler.hh" @@ -25,6 +26,7 @@ namespace Trick { private: /** Flag to track if an automatic freeze has been triggered */ bool auto_freeze = false; /* ** */ + std::map chkpnt_names; protected: /** queue to hold jobs to be called before a checkpoint is dumped. */ @@ -204,7 +206,7 @@ namespace Trick { @param in_time - desired checkpoint time in seconds. @return always 0 */ - virtual int checkpoint(double in_time) ; + virtual int checkpoint(double in_time, std::string file_name = "") ; /** * Executes the pre_init_checkpoint diff --git a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp index 4c60d15e4..2d7bfc405 100644 --- a/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp +++ b/trick_source/sim_services/CheckPointRestart/CheckPointRestart.cpp @@ -115,7 +115,7 @@ int Trick::CheckPointRestart::find_write_checkpoint_jobs(std::string sim_object_ * @param in_time The time the checkpoint should be dumped * @see write_checkpoint() */ -int Trick::CheckPointRestart::checkpoint(double in_time) { +int Trick::CheckPointRestart::checkpoint(double in_time, std::string file_name) { long long curr_time = exec_get_time_tics() ; long long new_time ; @@ -128,6 +128,8 @@ int Trick::CheckPointRestart::checkpoint(double in_time) { write_checkpoint_job->next_tics = new_time ; } + if (!file_name.empty()) chkpnt_names[new_time] = file_name; + the_exec->freeze(in_time); //std::cout << "\033[33mSET CHECKPOINT TIME " << in_time << " " << new_time << "\033[0m" << std::endl ; } else { @@ -271,11 +273,18 @@ int Trick::CheckPointRestart::write_checkpoint() { } double sim_time = exec_get_sim_time() ; - std::stringstream chk_name_stream ; + std::string file_name = ""; - chk_name_stream << "chkpnt_" << std::fixed << std::setprecision(6) << sim_time ; + if (chkpnt_names.find(curr_time) == chkpnt_names.end()) { + std::stringstream chk_name_stream ; + chk_name_stream << "chkpnt_" << std::fixed << std::setprecision(6) << sim_time ; + file_name = chk_name_stream.str(); + } else { + file_name = chkpnt_names[curr_time]; + chkpnt_names.erase(curr_time); + } - checkpoint( chk_name_stream.str() ); + checkpoint( file_name ); the_exec->run(); } From 7d7fd632b93af424d06f5363ce7222042bcf7d29 Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Thu, 27 Jun 2024 15:08:24 -0500 Subject: [PATCH 16/23] Updated tests to increase coverage --- test/SIM_checkpoint_data_recording/RUN_test8/dump.py | 2 ++ test/SIM_stls/RUN_test/setup.py | 2 +- test/SIM_stls/RUN_test/unit_test.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/SIM_checkpoint_data_recording/RUN_test8/dump.py b/test/SIM_checkpoint_data_recording/RUN_test8/dump.py index a63985c11..ced70e3c6 100644 --- a/test/SIM_checkpoint_data_recording/RUN_test8/dump.py +++ b/test/SIM_checkpoint_data_recording/RUN_test8/dump.py @@ -6,6 +6,8 @@ def main(): exec(open("Modified_data/fooChange2.dr").read()) + # Testing that separately scheduling a freeze and checkpoint at the same time still results in a checkpoint + trick.freeze(5.0) trick.checkpoint(5.0) trick.stop(20.0) diff --git a/test/SIM_stls/RUN_test/setup.py b/test/SIM_stls/RUN_test/setup.py index 21164ad1b..498f3e311 100644 --- a/test/SIM_stls/RUN_test/setup.py +++ b/test/SIM_stls/RUN_test/setup.py @@ -5,7 +5,7 @@ def main(): trick.exec_set_job_onoff("the_object.stlc.test", 1, False) trick.exec_set_job_onoff("the_object.stlc.print", 1, False) - trick.checkpoint(0.5) + trick.checkpoint(0.5, "chkpnt_in") trick.exec_set_freeze_frame(0.10) trick.stop(1.0) diff --git a/test/SIM_stls/RUN_test/unit_test.py b/test/SIM_stls/RUN_test/unit_test.py index fefddf4f6..16c31069f 100644 --- a/test/SIM_stls/RUN_test/unit_test.py +++ b/test/SIM_stls/RUN_test/unit_test.py @@ -3,7 +3,7 @@ def main(): - trick.load_checkpoint("RUN_test/chkpnt_0.500000") + trick.load_checkpoint("RUN_test/chkpnt_in") trick.load_checkpoint_job() trick.exec_set_job_onoff("the_object.stlc.addData", 1, False) From 50dc60efe87de9c5ed67ad8e45ac588249c6cb11 Mon Sep 17 00:00:00 2001 From: Marcus Rockwell Date: Fri, 28 Jun 2024 10:27:48 -0500 Subject: [PATCH 17/23] Updated Checkpointing Documentation - Also marked a section for a more detailed revision in a few days --- docs/documentation/running_a_simulation/Input-File.md | 6 ++++++ docs/documentation/simulation_capabilities/Checkpoints.md | 4 ++++ docs/howto_guides/Checkpointing-Best-Practices.md | 2 +- docs/not_referenced/Input-File-Quick-Reference.md | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/documentation/running_a_simulation/Input-File.md b/docs/documentation/running_a_simulation/Input-File.md index 4410b9db2..67207e040 100644 --- a/docs/documentation/running_a_simulation/Input-File.md +++ b/docs/documentation/running_a_simulation/Input-File.md @@ -383,9 +383,15 @@ in the future. # Checkpoints immediately trick.checkpoint() +# Checkpoints immediately, saving to 'checkpoint_save' +trick.checkpoint("checkpoint_save") + # Checkpoints at an absolute time trick.checkpoint(100.0) +# Checkpoints at an absolute time, saving to 'late_checkpoint' +trick.checkpoint(100.0, "late_checkpoint") + # Checkpoints 5 seconds relative from the current sim_time trick.checkpoint(trick.exec_get_sim_time() + 5.0) ``` diff --git a/docs/documentation/simulation_capabilities/Checkpoints.md b/docs/documentation/simulation_capabilities/Checkpoints.md index 0f9aedb3f..eaa2e9fcd 100644 --- a/docs/documentation/simulation_capabilities/Checkpoints.md +++ b/docs/documentation/simulation_capabilities/Checkpoints.md @@ -13,8 +13,12 @@ trick.checkpoint_post_init(True|False) trick.checkpoint_end(True|False) # Save a checkpoint at a time in the future trick.checkpoint(