From 0be18b3ddba7e66ed1ee5db0c14b8d3ec89ce06b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Ma=C5=82ysa?= Date: Sat, 27 Jul 2024 16:31:35 +0200 Subject: [PATCH] sim: backups: add restore-backup program --- subprojects/sim/README.md | 12 +++ subprojects/sim/meson.build | 14 ++++ subprojects/sim/src/backup.cc | 116 ++++++++++++-------------- subprojects/sim/src/restore_backup.cc | 115 +++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 62 deletions(-) create mode 100644 subprojects/sim/src/restore_backup.cc diff --git a/subprojects/sim/README.md b/subprojects/sim/README.md index 3efe999d..cb237c3b 100644 --- a/subprojects/sim/README.md +++ b/subprojects/sim/README.md @@ -160,6 +160,18 @@ For more options: sim/manage help ``` +## Backups +A sim backup can be created with: +```sh +sim/bin/backup +``` +Multiple backups may be created. + +To restore a specific backup use the `sim/bin/restore-backup` program. You may want to edit the `.db.config` file if migrating sim from another machine. To learn more run: +```sh +sim/bin/restore-backup +``` + ## Running tests ```sh ninja -C build/ test # or other build directory diff --git a/subprojects/sim/meson.build b/subprojects/sim/meson.build index 087cea3f..7c0072b5 100644 --- a/subprojects/sim/meson.build +++ b/subprojects/sim/meson.build @@ -192,6 +192,19 @@ backup = executable('backup', install_rpath : get_option('prefix') / get_option('libdir'), ) +restore_backup = executable('restore-backup', + implicit_include_directories : false, + sources : [ + 'src/restore_backup.cc', + ], + dependencies : [ + libsim_dep, + static_dep, + ], + install : _install, + install_rpath : get_option('prefix') / get_option('libdir'), +) + sim_server = executable('sim-server', implicit_include_directories : false, sources : [ @@ -262,6 +275,7 @@ base_targets = [ job_server, libsim, manage, + restore_backup, setup_installation, sim_merger, sim_server, diff --git a/subprojects/sim/src/backup.cc b/subprojects/sim/src/backup.cc index ce3dad78..c686b107 100644 --- a/subprojects/sim/src/backup.cc +++ b/subprojects/sim/src/backup.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -33,48 +34,21 @@ int main2(int argc, char** argv) { chdir_relative_to_executable_dirpath(".."); -#define MYSQL_CNF ".mysql.cnf" - FileRemover mysql_cnf_guard; - - // Get connection - auto mysql = sim::mysql::Connection::from_credential_file(".db.config"); - - FileDescriptor fd{MYSQL_CNF, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, S_0600}; - if (fd == -1) { - errlog("Failed to open file `" MYSQL_CNF "`: open()", errmsg()); - return 1; - } - - mysql_cnf_guard = FileRemover{MYSQL_CNF}; - { - auto old_mysql = old_mysql::ConnectionView{mysql}; - write_all_throw( - fd, - from_unsafe{concat( - "[client]\nuser=\"", - old_mysql.impl()->user, - "\"\npassword=\"", - old_mysql.impl()->passwd, - "\"\n" - )} - ); - } - auto run_command = [](vector args) { auto es = Spawner::run(args[0], args); - if (es.si.code != CLD_EXITED or es.si.status != 0) { + if (es.si.code != CLD_EXITED || es.si.status != 0) { errlog(args[0], " failed: ", es.message); _exit(1); } }; - // Remove temporary internal files that were not removed (e.g. a problem - // adding job was cancelled between the first and second stage while it was - // pending) + // Remove temporary internal files that were not removed e.g. because of web/job server crash + stdlog("Removing orphaned temporary internal files..."); { using std::chrono::system_clock; using namespace std::chrono_literals; + auto mysql = sim::mysql::Connection::from_credential_file(".db.config"); auto transaction = mysql.start_repeatable_read_transaction(); auto old_mysql = old_mysql::ConnectionView{mysql}; @@ -115,56 +89,74 @@ int main2(int argc, char** argv) { transaction.commit(); } + stdlog("... done."); - auto old_mysql = old_mysql::ConnectionView{mysql}; - run_command({ - "mysqldump", - concat_tostr("--defaults-file=", MYSQL_CNF), - "--result-file=dump.sql", - "--single-transaction", - old_mysql.impl()->db, - }); + stdlog("Dumping the database..."); + { + ConfigFile cf; + cf.add_vars("user", "password", "db", "host"); + cf.load_config_from_file(".db.config"); + const auto& user = cf.get_var("user"); + throw_assert(user.is_set()); + const auto& password = cf.get_var("password"); + throw_assert(password.is_set()); + const auto& db = cf.get_var("db"); + throw_assert(db.is_set()); + const auto& host = cf.get_var("host"); + throw_assert(host.is_set()); + + run_command({ + "/usr/bin/mariadb-dump", + concat_tostr("--user=", user.as_string()), + concat_tostr("--password=", password.as_string()), + concat_tostr("--host=", host.as_string()), + "--single-transaction", + "--result-file=dump.sql", + db.as_string(), + }); + } + stdlog("... done."); if (chmod("dump.sql", S_0600)) { THROW("chmod()", errmsg()); } - run_command({"git", "init", "--initial-branch", "main"}); - run_command({"git", "config", "user.name", "bin/backup"}); - run_command({"git", "config", "user.email", ""}); + run_command({"/usr/bin/git", "init", "--initial-branch", "main"}); + run_command({"/usr/bin/git", "config", "user.name", "bin/backup"}); + run_command({"/usr/bin/git", "config", "user.email", ""}); // Optimize git for worktree with many files - run_command({"git", "config", "feature.manyFiles", "true"}); + run_command({"/usr/bin/git", "config", "feature.manyFiles", "true"}); // Tell git not to delta compress too big files - run_command({"git", "config", "core.bigFileThreshold", "16m"}); + run_command({"/usr/bin/git", "config", "core.bigFileThreshold", "16m"}); // Tell git not to make internal files too large - run_command({"git", "config", "pack.packSizeLimit", "1g"}); + run_command({"/usr/bin/git", "config", "pack.packSizeLimit", "1g"}); // Tell git not to repack big packs - run_command({"git", "config", "gc.bigPackThreshold", "500m"}); + run_command({"/usr/bin/git", "config", "gc.bigPackThreshold", "500m"}); // Tell git not to repack large number of big packs - run_command({"git", "config", "gc.autoPackLimit", "0"}); + run_command({"/usr/bin/git", "config", "gc.autoPackLimit", "0"}); // Makes fetching from this repository faster - run_command({"git", "config", "pack.window", "0"}); + run_command({"/usr/bin/git", "config", "pack.window", "0"}); // Prevent git from too excessive memory usage during git fetch (on remote) and git gc - run_command({"git", "config", "pack.deltaCacheSize", "128m"}); - run_command({"git", "config", "pack.windowMemory", "128m"}); - run_command({"git", "config", "pack.threads", "1"}); - // Run automatic git gc more fequently - run_command({"git", "config", "gc.auto", "500"}); - - run_command({"git", "add", "--verbose", "dump.sql"}); - run_command({"git", "add", "--verbose", "bin/", "lib/", "manage"}); - run_command({"git", "add", "--verbose", "internal_files/"}); - run_command({"git", "add", "--verbose", "logs/"}); - run_command({"git", "add", "--verbose", "sim.conf", ".db.config"}); - run_command({"git", "add", "--verbose", "static/"}); + run_command({"/usr/bin/git", "config", "pack.deltaCacheSize", "128m"}); + run_command({"/usr/bin/git", "config", "pack.windowMemory", "128m"}); + run_command({"/usr/bin/git", "config", "pack.threads", "1"}); + // Disable automatic git gc + run_command({"/usr/bin/git", "config", "gc.auto", "0"}); + + run_command({"/usr/bin/git", "add", "--verbose", "dump.sql"}); + run_command({"/usr/bin/git", "add", "--verbose", "bin/", "lib/", "manage"}); + run_command({"/usr/bin/git", "add", "--verbose", "internal_files/"}); + run_command({"/usr/bin/git", "add", "--verbose", "logs/"}); + run_command({"/usr/bin/git", "add", "--verbose", "sim.conf", ".db.config"}); + run_command({"/usr/bin/git", "add", "--verbose", "static/"}); run_command( - {"git", + {"/usr/bin/git", "commit", "-q", "-m", concat_tostr("Backup ", local_mysql_datetime(), " (", utc_mysql_datetime(), " UTC)")} ); - run_command({"git", "--no-pager", "show", "--stat"}); + run_command({"/usr/bin/git", "--no-pager", "show", "--stat"}); return 0; } diff --git a/subprojects/sim/src/restore_backup.cc b/subprojects/sim/src/restore_backup.cc new file mode 100644 index 00000000..322d6aff --- /dev/null +++ b/subprojects/sim/src/restore_backup.cc @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void help(const char* program_name) { + if (!program_name) { + program_name = "restore_backup"; + } + + printf("Usage: %s \n", program_name); + puts("Restore sim from backup revision . To list available revisions use:"); + puts("git log --all"); + puts("E.g. if you want to restore the revision identified by line \"commit " + "987f1889ed38a9ebc01f031ddb988f0f8d9a0c6a\", use the commit hash as a revision:"); + printf(" %s 987f1889ed38a9ebc01f031ddb988f0f8d9a0c6a\n", program_name); +} + +int main(int argc, char** argv) { + if (argc != 2) { + help(argc > 0 ? argv[0] : nullptr); + return 1; + } + + chdir_relative_to_executable_dirpath(".."); + + auto revision = argv[1]; + + auto run_command = [](std::vector args, + std::optional stdin_fd = std::nullopt) { + auto es = Spawner::run( + args[0], + args, + Spawner::Options(stdin_fd.value_or(STDIN_FILENO), STDOUT_FILENO, STDERR_FILENO) + ); + if (es.si.code != CLD_EXITED || es.si.status != 0) { + errlog(args[0], " failed: ", es.message); + _exit(1); + } + }; + + // Stops Sim so it doesn't run on the messed up (during restoration) database and files + run_command({"./manage", "stop"}); + + // Restore the database + stdlog("Restoring database dump file..."); + run_command({"/usr/bin/git", "checkout", revision, "--", "dump.sql"}); + stdlog("... done."); + { + ConfigFile cf; + cf.add_vars("user", "password", "db", "host"); + cf.load_config_from_file(".db.config"); + const auto& user = cf.get_var("user"); + throw_assert(user.is_set()); + const auto& password = cf.get_var("password"); + throw_assert(password.is_set()); + const auto& db = cf.get_var("db"); + throw_assert(db.is_set()); + const auto& host = cf.get_var("host"); + throw_assert(host.is_set()); + + auto dump_sql_fd = FileDescriptor{"dump.sql", O_RDONLY}; + throw_assert(dump_sql_fd.is_open()); + + stdlog("Restoring database..."); + run_command( + { + "/usr/bin/mariadb", + concat_tostr("--user=", user.as_string()), + concat_tostr("--password=", password.as_string()), + concat_tostr("--database=", db.as_string()), + concat_tostr("--host=", host.as_string()), + }, + dump_sql_fd + ); + stdlog("... done."); + } + + // Restore all files except modifications to .db.config + auto db_config = get_file_contents(".db.config"); + stdlog("Restoring files..."); + run_command({"/usr/bin/git", "checkout", revision, "--", "."}); + stdlog("... done."); + put_file_contents(".db.config", db_config); + // Remove extra files e.g. internal files (e.g. ones created after the backup was made) + stdlog("Removing extra files files..."); + run_command({ + "/usr/bin/sh", + "-c", + "/usr/bin/git ls-files --others | /usr/bin/xargs rm -f", + }); + stdlog("... done."); + + stdlog("\033[1;32mRestoring sim done.\033[m"); + stdlog( + "\033[1;33mSim is not running. If needed, you have to start it manually with: ", + StringView{argv[0]}.without_trailing([](char c) { return c != '/'; } + ).without_suffix(strlen("bin/")), + "manage start -b\033[m" + ); + + return 0; +}