Merge remote-tracking branch 'develop'

README.md

@@ -34,9 +34,9 @@ By default no subproject is marked to be installed on `meson install -C build`.
 ```sh
 meson setup build --wipe -Dbuild=sim,sip -Dinstall=sip
 ```
-This command will build `sim` and `sip` subprojects, but will mark only `sip` to be installed.
+This command will build `sim` and `sip` subprojects, but will only mark `sip` to be installed.
 
-Specifying subpojects to install is also adds them to the list of subprojects to build.
+Specifying subpojects to install also adds them to the list of subprojects to build.
 
 For detailed option description you can inspect [meson_options.txt](meson_options.txt) file.
 

subprojects/sim/README.md

@@ -161,30 +161,32 @@ sim/manage help
 ```
 
 ## Backups
+Backups are made using [restic](https://restic.net/). Debin/Ubuntu package: `restic`, Arch Linux package: `restic`.
+
 A sim backup can be created with:
 ```sh
-sim/bin/backup
+sim/bin/backup save
 ```
 Multiple backups may be created.
 
-To restore a specific backup use the `sim/bin/restore-backup` program. You may want to edit the `.db.config` file if migrating sim from another machine. To learn more run:
+To restore a specific backup use the `sim/bin/backup restore` program. You may want to edit the `.db.config` file if migrating sim from another machine. To learn more run:
 ```sh
-sim/bin/restore-backup
+sim/bin/backup help
 ```
 
 ## Merging one Sim instance into the other
 First, naming. Let's call the Sim instance that we want to merge into the main Sim instance and the one we want to merge from the other Sim instance.
 
 In case the merging fails for some reason, make a backup of the main sim instance:
 ```sh
-main-sim-instance/bin/backup
+main-sim-instance/bin/backup save
 ```
 
 The other sim instance has to be on the same machine as the main sim instance. If this is not the case, follow the steps below:
 
 1. Make a backup of the other instance if not already done:
 ```sh
-other-sim-instance/bin/backup
+other-sim-instance/bin/backup save
 ```
 
 2. Copy the other sim instance to the machine containing the main sim instance, e.g. using `rsync`.
@@ -196,9 +198,9 @@ sudo mariadb -e "CREATE USER other_sim@localhost IDENTIFIED BY 'other_sim'; CREA
 
 4. Edit file `other-sim-instance/.db.config` to use the new user and database.
 
-5. Restore the other sim instance from backup:
+5. Restore the other sim instance from latest backup:
 ```sh
-other-sim-instance/bin/restore-backup @
+other-sim-instance/bin/backup restore latest
 ```
 
 Now we have both instances on the same machine we can merge them. To do so, run:

subprojects/sim/meson.build

@@ -32,6 +32,7 @@ if get_option('warning_level') > '0'
         '-Wnull-dereference',
         '-Wformat=2',
         '-Wlifetime',
+        '-Wunused-result',
     ]
     foreach warning : warnings
         if cc.has_argument(warning)
@@ -212,19 +213,6 @@ backup = executable('backup',
     install_rpath : get_option('prefix') / get_option('libdir'),
 )
 
-restore_backup = executable('restore-backup',
-    implicit_include_directories : false,
-    sources : [
-        'src/restore_backup.cc',
-    ],
-    dependencies : [
-        libsim_dep,
-        static_dep,
-    ],
-    install : _install,
-    install_rpath : get_option('prefix') / get_option('libdir'),
-)
-
 sim_server = executable('sim-server',
     implicit_include_directories : false,
     sources : [
@@ -295,7 +283,6 @@ base_targets = [
     job_server,
     libsim,
     manage,
-    restore_backup,
     setup_installation,
     sim_merger,
     sim_server,

subprojects/sim/scripts/setup_sim_on_debian_12.py

@@ -95,27 +95,27 @@ def setup_sim():
 
     print('\033[1;32m==>\033[0;1m Build sim\033[m')
     apt_install('meson pkgconf g++ libmariadb-dev libseccomp-dev libcap-dev libzip-dev libssl-dev rustc fpc clang')
-    user_cmd(f"cd '{args.sim_path}/subprojects/sim' && meson setup build/ -Dbuildtype=release")
-    user_cmd(f"cd '{args.sim_path}/subprojects/sim' && meson configure build/ --prefix \"$(pwd)/sim\"")
-    user_cmd(f"cd '{args.sim_path}/subprojects/sim' && meson compile -C build/")
-    user_cmd(f"cd '{args.sim_path}/subprojects/sim' && meson test -C build/ --print-errorlogs")
+    user_cmd(f"cd '{args.sim_path}' && meson setup build/ -Dbuildtype=release -Dinstall=sim")
+    user_cmd(f"cd '{args.sim_path}' && meson configure build/ --prefix \"$(pwd)/sim\"")
+    user_cmd(f"cd '{args.sim_path}' && meson compile -C build/")
+    user_cmd(f"cd '{args.sim_path}' && meson test -C build/ --print-errorlogs")
 
     print('\033[1;32m==>\033[0;1m Configure sim\033[m')
-    user_cmd(f"cd '{args.sim_path}/subprojects/sim' && mkdir --parents sim/")
-    user_cmd(f"""cd '{args.sim_path}/subprojects/sim' && cat > sim/.db.config << HEREDOCEND
+    user_cmd(f"cd '{args.sim_path}' && mkdir --parents sim/")
+    user_cmd(f"""cd '{args.sim_path}' && cat > sim/.db.config << HEREDOCEND
 user: '{args.db_user}'
 password: '{args.db_user_password}'
 db: '{args.db}'
 host: 'localhost'
 HEREDOCEND""")
-    user_cmd(f"cd '{args.sim_path}/subprojects/sim' && meson install -C build/")
+    user_cmd(f"cd '{args.sim_path}' && meson install -C build/")
     # Change sim address
-    user_cmd(f"grep -P '^web_server_address:.*$' '{args.sim_path}/subprojects/sim/sim/sim.conf' -q || (echo \"\033[1;31merror: couldn't find field 'address' in the {args.sim_path}/subprojects/sim/sim/sim.conf\033[m\" && false)")
-    user_cmd(f"sed 's/^web_server_address:.*$/web_server_address: {args.sim_local_address}/' -i '{args.sim_path}/subprojects/sim/sim/sim.conf'")
+    user_cmd(f"grep -P '^web_server_address:.*$' '{args.sim_path}/sim/sim.conf' -q || (echo \"\033[1;31merror: couldn't find field 'address' in the {args.sim_path}/sim/sim.conf\033[m\" && false)")
+    user_cmd(f"sed 's/^web_server_address:.*$/web_server_address: {args.sim_local_address}/' -i '{args.sim_path}/sim/sim.conf'")
 
     print('\033[1;32m==>\033[0;1m Run sim\033[m')
     apt_install('curl')
-    user_cmd(f"'{args.sim_path}/subprojects/sim/sim/manage' stop")
+    user_cmd(f"'{args.sim_path}/sim/manage' stop")
     user_cmd(f"curl --silent '{args.sim_local_address}' > /dev/null; test $? -eq 7 || (echo '\033[1;31merror: address {args.sim_local_address} is already in use\033[m' && false)")
     # user_cmd does not work and we want the job to run in the background
     subprocess.check_call([
@@ -126,7 +126,7 @@ def setup_sim():
         '--quiet',
         'sh',
         '-c',
-        f"'{args.sim_path}/subprojects/sim/sim/manage' restart",
+        f"'{args.sim_path}/sim/manage' restart",
     ])
 
 if args.sim_path is not None:
@@ -148,7 +148,7 @@ def start_sim_at_boot():
 Description=Start Sim instance {args.sim_path}
 
 [Service]
-ExecStart=sh -c "until test -e /var/run/mysqld/mysqld.sock; do sleep 0.4; done; '{args.sim_path}/subprojects/sim/sim/manage' start"
+ExecStart=sh -c "until test -e /var/run/mysqld/mysqld.sock; do sleep 0.4; done; '{args.sim_path}/sim/manage' start"
 
 [Install]
 WantedBy=default.target
@@ -494,9 +494,16 @@ def setup_daily_backup():
 
     setup_sim()
 
+    full_sim_path = args.sim_path if args.sim_path.startswith("/") else os.path.join(f"/home/{args.user}", args.sim_path)
+
+    print('\033[1;32m==>\033[0;1m Install restic\033[m')
+    apt_install('restic')
+
     print('\033[1;32m==>\033[0;1m Set up daily backup\033[m')
-    root_cmd("mkdir -p /opt/sim_backups")
-    root_cmd("chmod 700 /opt/sim_backups")
+    if args.daily_backup_to_partition is None:
+        root_cmd("mkdir -p /opt/sim_backups")
+        root_cmd("chmod 700 /opt/sim_backups")
+
     root_cmd(f"""cat > "/etc/systemd/system/sim-backup:$(systemd-escape '{args.user}'):$(systemd-escape --path '{args.sim_path}').service" << HEREDOCEND
 [Unit]
 Description=Back up Sim and copy backup of {args.user}:{args.sim_path}
@@ -508,7 +515,7 @@ def setup_daily_backup():
 NoNewPrivileges=true
 PrivateTmp=true
 BindReadOnlyPaths=/:/:rbind
-ReadWritePaths="{os.path.join(f"/home/{args.user}", args.sim_path)}"
+ReadWritePaths="{full_sim_path}"
 
 {f"MountImages=/dev/disk/by-uuid/{args.daily_backup_to_partition}:/tmp/mnt/disk/" if args.daily_backup_to_partition is not None else ''}
 {"ReadWritePaths=/opt/sim_backups/" if args.daily_backup_to_partition is None else ''}
@@ -517,27 +524,21 @@ def setup_daily_backup():
 # Hide backups from unprivileged users
 ExecStartPre=!chmod 0700 /tmp/mnt/
 ExecStartPre=!mkdir --parents --mode=0700 /tmp/mnt/disk/sim_backups/
-# Run bin/backup as the unprivileged user
-ExecStart=nice "{os.path.join(f"/home/{args.user}", args.sim_path)}/subprojects/sim/sim/bin/backup"
+
+# Save backup as the unprivileged user
+ExecStart=nice '{full_sim_path}/sim/bin/backup' save
+# Remove old backups as the unprivileged user
+ExecStart=nice restic --repo '{full_sim_path}/sim/backup.restic' --password-command="echo sim" forget --group-by= --prune --keep-daily 30 --keep-weekly 52
+
+# Create protected backup repository if absent
+ExecStart=!sh -c 'test -e /tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.restic || nice restic --repo /tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.restic --password-command="echo sim" init'
 # Copy backup to the protected backup location i.e accessible for privileged users only
-ExecStart=!sh -c '\\\\
-    git init --bare /tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git --initial-branch main && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config feature.manyFiles true && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config core.bigFileThreshold 16m && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config pack.packSizeLimit 1g && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config gc.bigPackThreshold 500m && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config gc.autoPackLimit 0 && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config pack.window 0 && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config pack.deltaCacheSize 128m && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config pack.windowMemory 128m && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config pack.threads 1 && \\\\
-    git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git config gc.auto 500 && \\\\
-    nice git --git-dir=/tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.git fetch "{os.path.join(f"/home/{args.user}", args.sim_path)}/subprojects/sim/sim" HEAD:HEAD --progress'
+ExecStart=!nice restic --repo /tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.restic --password-command="echo sim" copy --from-repo '{full_sim_path}/sim/backup.restic' --from-password-command="echo sim"
+# Remove old backups
+ExecStart=!nice restic --repo /tmp/mnt/disk/sim_backups/{args.daily_backup_filename}.restic --password-command="echo sim" forget --group-by= --prune --keep-daily 30 --keep-weekly 52
 
 # To restore from backup use:
-# git --git-dir=/path/to/sim_backups/{args.daily_backup_filename}.git --work-tree=/path/to/where/to/restore/ checkout HEAD -- .
-# To check working tree against backup:
-# git --git-dir=/path/to/sim_backups/{args.daily_backup_filename}.git --work-tree=/path/to/where/to/restore/ status
+# '{full_sim_path}/sim/bin/backup' restore --from /path/to/sim/backups/{args.daily_backup_filename}.restic
 
 HEREDOCEND""")
     root_cmd(f"""cat > "/etc/systemd/system/sim-backup:$(systemd-escape '{args.user}'):$(systemd-escape --path '{args.sim_path}').timer" << HEREDOCEND