feat(ci): run and update service (#579)

This adds some files that can be installed into a linux system to run
sig as a long running service, and periodically have it self-update
based on the latest code and restart.

This also adds an option to the logger to output to a file, and makes
stderr logging optional.

See the
[readme](https://github.com/Syndica/sig/blob/dnut/ci/runupdate/ci/run-and-update-service/README.md)
for more info.

I added a folder in the repo root called "ci" for this since no existing
folders seemed appropriate.

ci/run-and-update-service/Makefile

@@ -0,0 +1,47 @@
+prefix := /usr/local
+bin := bin
+systemd := lib/systemd
+branch := main
+
+install: create_user install_config
+	install -Dm755 sig-update         "$(prefix)/$(bin)/sig-update"
+	install -Dm644 sig.service        "$(prefix)/$(systemd)/system/sig.service"
+	install -Dm644 sig-update.service "$(prefix)/$(systemd)/system/sig-update.service"
+	install -Dm644 sig-update.timer   "$(prefix)/$(systemd)/system/sig-update.timer"
+	sudo -u sig git clone https://github.com/Syndica/sig.git /home/sig/sig
+	sudo -u sig mkdir /home/sig/sig/logs
+	systemctl daemon-reload
+
+create_user:
+	@if ! id sig >/dev/null 2>&1; then \
+		echo "Creating user sig"; \
+		useradd -m sig; \
+	fi
+
+install_config:
+	@if ! [ -f /etc/sig.conf ]; then \
+		echo "Installing /etc/sig.conf"; \
+		install -Dm644 sig.conf /etc/sig.conf; \
+		sed -i 's#BRANCH=main#BRANCH=$(branch)#g' /etc/sig.conf; \
+	fi
+
+uninstall: stop
+	-rm "$(prefix)/$(bin)/sig-update"
+	-rm "$(prefix)/$(systemd)/system/sig.service"
+	-rm "$(prefix)/$(systemd)/system/sig-update.service"
+	-rm "$(prefix)/$(systemd)/system/sig-update.timer"
+	-systemctl daemon-reload
+	-userdel sig
+	-rm -rf /home/sig/sig
+
+start:
+	systemctl enable sig.service
+	systemctl enable sig-update.timer
+	systemctl start sig-update.timer
+
+stop:
+	-systemctl stop sig.service
+	-systemctl stop sig-update.timer
+	-systemctl stop sig-update.service
+	-systemctl disable sig.service
+	-systemctl disable sig-update.timer

ci/run-and-update-service/README.md

@@ -0,0 +1,46 @@
+Install sig as a long running service on a linux system, and periodically have it self-update based on the latest code in the configured branch and restart. This exists to maintain CI test environments.
+
+## Install
+
+```bash
+git clone https://github.com/Syndica/sig.git
+cd sig/ci/run-and-update-service
+
+sudo make install      # install update script and systemd units to the system
+sudo vim /etc/sig.conf # optionally edit file to specify custom configuration
+sudo make start        # start sig, metrics, and the timer to periodically update sig
+```
+
+## Uninstall
+
+```bash
+sudo make uninstall
+```
+
+## Configuration
+
+These options may be configured in /etc/sig.conf
+
+- `CLI_ARGS`: The command line options that will be passed to sig. Default: '--log-file /home/sig/sig/logs/sig.log'
+- `BRANCH`: The branch that will be checked periodically and rebuilt when it changes. Default: 'main'
+- `SLACK_WEBHOOK_URL` (optional): Enable slack web hooks. Only used if non-empty. Default: ''
+
+## Design
+
+The service is installed to the system in `/usr/local` using make. A new user called `sig` is added to the system and is used to build and run the sig binary.
+
+The service is orchestrated by systemd, using two services and one timer.
+
+> *Note:* **Systemd** is the most common init system for linux. It can run processes, daemons, and schedules. A *service* is a short- or long-running process that's run by systemd, and a *timer* is used to start services on a schedule, similar to a cron job.
+
+Call graph: `sig-update.timer` -> `sig-update.service` -> `sig-update` -> `sig.service` -> `sig`
+
+`sig-update.timer` periodically runs the `sig-update` binary as root on a schedule. This script de-escalates to the sig user to check if there are new commits on `BRANCH`, and if so, it builds a new sig binary. Then as root, it restarts `sig.service` and starts sig's metrics with docker-compose. As the sig user, `sig.service` runs the sig binary that was built in the sig user's home folder, passing the configured `CLI_ARGS`.
+
+### Config
+
+The configuration file `/etc/sig.conf` is used by both systemd and the `sig-update` binary. systemd reads the file when running `sig.service` in order to pass the correct `CLI_ARGS` to sig. `sig-update` reads the file when it's checking the `BRANCH` for new commits, and when setting the `SLACK_WEBHOOK_URL` for sig's metrics.
+
+## Root privileges
+
+All of the systemd units are system-level units, with some root privileges, because we want the service to start reliably when the system boots. If these three units were instead user-level units, they would only run after the user logs in. Automating startup on boot would require additional system-level units to log in the user, which would increase the complexity. Minimal root privileges were integrated into the existing units to keep this both simple and reliable.

ci/run-and-update-service/sig-update

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# check if there is an update. if so rebuild sig. otherwise exit
+sudo -iu sig bash <<'EOF' | tee /dev/null
+    set -euxo pipefail
+    . /etc/sig.conf
+    cd /home/sig/sig
+    git fetch
+
+    if [[ $(git rev-parse HEAD) == $(git rev-parse remotes/origin/$BRANCH) ]] && \
+        [ -f /home/sig/sig/zig-out/bin/sig ]; then
+        echo sig is unchanged, skipping update
+        exit 58
+    fi
+
+    git checkout remotes/origin/$BRANCH
+    echo building sig
+    zig build -Doptimize=ReleaseSafe -Dno-run sig
+
+    if ! [ -z "$SLACK_WEBHOOK_URL" ]; then
+        echo "SLACK_WEBHOOK_URL=$SLACK_WEBHOOK_URL" > /home/sig/sig/metrics/.env
+    fi
+EOF
+code=$?
+
+set -euxo pipefail
+
+if [[ $code == 58 ]]; then
+    # 58 means there were no changes. just make sure sig is running
+    systemctl start sig
+    cd /home/sig/sig/metrics
+    docker-compose up -d
+elif [[ $code == 0 ]]; then
+    # 0 means there was a change and rebuild, so restart sig
+    systemctl restart sig
+    cd /home/sig/sig/metrics
+    docker-compose down
+    docker-compose pull
+    docker-compose up -d
+else
+    # other codes indicate error and should be propagated
+    exit $code
+fi

ci/run-and-update-service/sig-update.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Runs sig-update command.
+
+[Service]
+ExecStart=/usr/local/bin/sig-update
+Restart=on-failure

ci/run-and-update-service/sig-update.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Update sig weekly and on boot
+After=network.target
+
+[Timer]
+OnBootSec=0
+OnActiveSec=0
+OnUnitActiveSec=1w
+
+[Install]
+WantedBy=timers.target

ci/run-and-update-service/sig.conf

@@ -0,0 +1,3 @@
+CLI_ARGS='--log-file /home/sig/sig/logs/sig.log'
+BRANCH=main
+SLACK_WEBHOOK_URL=

ci/run-and-update-service/sig.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Run user-installed Sig as sig user with the provided arguments.
+After=network.target
+
+[Service]
+EnvironmentFile=/etc/sig.conf
+WorkingDirectory=/home/sig/sig
+ExecStart=/home/sig/sig/zig-out/bin/sig $CLI_ARGS
+Restart=always
+User=sig
+Group=sig
+LimitNOFILE=infinity
+
+[Install]
+WantedBy=multi-user.target

docs/docusaurus/docs/code/run-and-update-service.md

@@ -0,0 +1,46 @@
+Install sig as a long running service on a linux system, and periodically have it self-update based on the latest code in the configured branch and restart. This exists to maintain CI test environments.
+
+## Install
+
+```bash
+git clone https://github.com/Syndica/sig.git
+cd sig/ci/run-and-update-service
+
+sudo make install      # install update script and systemd units to the system
+sudo vim /etc/sig.conf # optionally edit file to specify custom configuration
+sudo make start        # start sig, metrics, and the timer to periodically update sig
+```
+
+## Uninstall
+
+```bash
+sudo make uninstall
+```
+
+## Configuration
+
+These options may be configured in /etc/sig.conf
+
+- `CLI_ARGS`: The command line options that will be passed to sig. Default: '--log-file /home/sig/sig/logs/sig.log'
+- `BRANCH`: The branch that will be checked periodically and rebuilt when it changes. Default: 'main'
+- `SLACK_WEBHOOK_URL` (optional): Enable slack web hooks. Only used if non-empty. Default: ''
+
+## Design
+
+The service is installed to the system in `/usr/local` using make. A new user called `sig` is added to the system and is used to build and run the sig binary.
+
+The service is orchestrated by systemd, using two services and one timer.
+
+> *Note:* **Systemd** is the most common init system for linux. It can run processes, daemons, and schedules. A *service* is a short- or long-running process that's run by systemd, and a *timer* is used to start services on a schedule, similar to a cron job.
+
+Call graph: `sig-update.timer` -> `sig-update.service` -> `sig-update` -> `sig.service` -> `sig`
+
+`sig-update.timer` periodically runs the `sig-update` binary as root on a schedule. This script de-escalates to the sig user to check if there are new commits on `BRANCH`, and if so, it builds a new sig binary. Then as root, it restarts `sig.service` and starts sig's metrics with docker-compose. As the sig user, `sig.service` runs the sig binary that was built in the sig user's home folder, passing the configured `CLI_ARGS`.
+
+### Config
+
+The configuration file `/etc/sig.conf` is used by both systemd and the `sig-update` binary. systemd reads the file when running `sig.service` in order to pass the correct `CLI_ARGS` to sig. `sig-update` reads the file when it's checking the `BRANCH` for new commits, and when setting the `SLACK_WEBHOOK_URL` for sig's metrics.
+
+## Root privileges
+
+All of the systemd units are system-level units, with some root privileges, because we want the service to start reliably when the system boots. If these three units were instead user-level units, they would only run after the user logs in. Automating startup on boot would require additional system-level units to log in the user, which would increase the complexity. Minimal root privileges were integrated into the existing units to keep this both simple and reliable.

metrics/docker-compose.yml

@@ -26,6 +26,7 @@ services:
       - ./loki:/etc/loki:ro
     command:
       - "--config.file=/etc/loki/loki.yml"
+    restart: unless-stopped
 
   alloy:
     image: grafana/alloy:v1.3.1
@@ -48,6 +49,7 @@ services:
       ]
     environment:
       - SIG_PID=${SIG_PID}
+    restart: unless-stopped
 
   grafana:
     image: grafana/grafana:11.4.0
@@ -79,6 +81,7 @@ services:
       - "4040:4040"
     environment:
       - STORAGE_DRIVER=in-memory
+    restart: unless-stopped
 
 volumes:
   prom_data:

src/accountsdb/fuzz.zig

@@ -3,7 +3,7 @@ const sig = @import("../sig.zig");
 const zstd = @import("zstd");
 
 const AccountsDB = sig.accounts_db.AccountsDB;
-const StandardErrLogger = sig.trace.ChannelPrintLogger;
+const ChannelPrintLogger = sig.trace.ChannelPrintLogger;
 const Account = sig.core.Account;
 const Slot = sig.core.time.Slot;
 const Pubkey = sig.core.pubkey.Pubkey;
@@ -59,11 +59,11 @@ pub fn run(seed: u64, args: *std.process.ArgIterator) !void {
     defer _ = gpa_state.deinit();
     const allocator = gpa_state.allocator();
 
-    var std_logger = try StandardErrLogger.init(.{
+    var std_logger = try ChannelPrintLogger.init(.{
         .allocator = allocator,
         .max_level = .debug,
         .max_buffer = 1 << 20,
-    });
+    }, null);
     defer std_logger.deinit();
     const logger = std_logger.logger();