Skip to content

Commit

Permalink
use lxd images for postgresql and electricsql nodes
Browse files Browse the repository at this point in the history
  • Loading branch information
@nurturenature
nurturenature committed Feb 15, 2024
1 parent 769ee9e commit 07e4402
Show file tree
Hide file tree
Showing 13 changed files with 213 additions and 708 deletions.
90 changes: 90 additions & 0 deletions lxd/build-electricsql-image.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
#! /usr/bin/bash
set -e

echo "install deps..."
apt update -qy
apt install -qy lsb-release gpg wget

echo "install postgresql-client..."
wget --quiet -O - "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo apt-key add -
echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list
apt update -qy

apt install -qy postgresql-client

echo "install erlang..."
apt install -qy build-essential autoconf m4 libncurses-dev libwxgtk3.2-dev libwxgtk-webview3.2-dev libgl1-mesa-dev libglu1-mesa-dev libpng-dev libssh-dev unixodbc-dev xsltproc fop libxml2-utils wget
mkdir /root/erlang
cd /root/erlang
wget "https://github.com/erlang/otp/releases/download/OTP-25.3.2.8/otp_src_25.3.2.8.tar.gz"
tar -xvzf "./otp_src_25.3.2.8.tar.gz"
cd otp_src_25.3.2.8
ERL_TOP=$(pwd)
export ERL_TOP=$ERL_TOP
./configure
make
make install
echo "erlang version:"
erl -eval "erlang:display (erlang:system_info (otp_release)), halt () ." -noshell

echo "install elixir..."
apt install -qy git
mkdir /root/elixir
cd /root/elixir
wget "https://github.com/elixir-lang/elixir/archive/refs/tags/v1.15.7.tar.gz"
tar -xvzf ./v1.15.7.tar.gz
cd elixir-1.15.7
make
ln -s "/root/elixir/elixir-1.15.7/bin/elixir" "/usr/local/bin/elixir"
ln -s "/root/elixir/elixir-1.15.7/bin/elixirc" "/usr/local/bin/elixirc"
ln -s "/root/elixir/elixir-1.15.7/bin/iex" "/usr/local/bin/iex"
ln -s "/root/elixir/elixir-1.15.7/bin/mix" "/usr/local/bin/mix"
echo "elixir version:"
elixir -v

echo "install electricsql"
apt install -qy git
mkdir /root/electricsql
cd /root/electricsql
wget "https://github.com/electric-sql/electric/archive/refs/tags/[email protected]"
tar -xvzf ./[email protected]
cd electric-electric-sql-0.9.0
cd components/electric
mix deps.get
mix compile
export MIX_ENV=prod
export ELECTRIC_VERSION=0.9.0
mix release

echo "electricsql systemd..."
cat <<EOF > /etc/systemd/system/electricsql.service
[Unit]
Description=ElectricSQL.
Requires=network-online.target
[Service]
Type=simple
Restart=on-failure
Environment=HOME=/root
Environment=MIX_ENV=prod
Environment=DATABASE_URL=postgresql://postgres:postgres@postgresql
Environment=ELECTRIC_WRITE_TO_PG_MODE=direct_writes
Environment=PG_PROXY_PORT=65432
Environment=PG_PROXY_PASSWORD=postgres
Environment=AUTH_MODE=insecure
Environment=ELECTRIC_USE_IPV6=false
WorkingDirectory=/root/electricsql/electric-electric-sql-0.9.0/components/electric
ExecStart=/root/electricsql/electric-electric-sql-0.9.0/components/electric/_build/prod/rel/electric/bin/electric start
ExecStop=/root/electricsql/electric-electric-sql-0.9.0/components/electric/_build/prod/rel/electric/bin/electric stop
[Install]
WantedBy=multi-user.target
EOF
chmod 644 /etc/systemd/system/electricsql.service

systemctl enable electricsql
32 changes: 32 additions & 0 deletions lxd/build-postgresql-image.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#! /usr/bin/bash
set -e

echo "install deps..."
apt update -qy
apt install -qy lsb-release gpg wget

echo "install postgresql..."
wget --quiet -O - "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo apt-key add -
echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list
apt update -qy

apt install -qy postgresql
pg_isready

echo "enable logical replication..."
su - postgres -c "psql -U postgres -c 'ALTER SYSTEM SET wal_level = logical;'"
systemctl restart postgresql
pg_isready
su - postgres -c "psql -U postgres -c 'show wal_level;'"

echo "configure access"
echo "listen_addresses = '*'" >> "/etc/postgresql/16/main/postgresql.conf"
echo "host all postgres all scram-sha-256" >> "/etc/postgresql/16/main/pg_hba.conf"
systemctl restart postgresql
pg_isready

echo "set electricsql password"
su - postgres -c "psql -U postgres -c \"ALTER USER postgres WITH PASSWORD 'postgres';\""

echo "PostgreSQL tables:"
su - postgres -c "psql -U postgres -c '\\dt';"
157 changes: 6 additions & 151 deletions lxd/lxd.md
View file
Original file line number Diff line number Diff line change
@@ -1,158 +1,13 @@
# Setting Up a Jepsen Environment with LXD/LXC
### Make Images

## Debian 12 - Bookworm

For further information, [LXD - Debian Wiki](https://wiki.debian.org/LXD).

### Install host packages:
```bash
sudo apt install lxd lxd-tools dnsmasq-base btrfs-progs
```

### Initialize LXD:
```bash
# defaults are good
sudo lxd init

# add yourself to the LXD group
sudo usermod -aG lxd <username>

# will need to logout/login for new group to be active

# try creating a sample container if you want
lxc launch images:debian/12 scratch
lxc list
lxc shell scratch
lxc stop scratch
lxc delete scratch
```

### Create and start Jepsen's node containers:

```bash
for i in {1..10}; do lxc launch images:debian/12 n${i}; done
```

### Configure LXD bridge network:

`lxd` automatically creates the bridge network, and `lxc launch` automatically configures the containers for it:
```bash
lxc network list
+--------+----------+---------+----------------+---+-------------+---------+---------+
| NAME | TYPE | MANAGED | IPV4 |...| DESCRIPTION | USED BY | STATE |
+--------+----------+---------+----------------+---+-------------+---------+---------+
| lxdbr0 | bridge | YES | 10.82.244.1/24 |...| | 11 | CREATED |
+--------+----------+---------+----------------+---+-------------+---------+---------+
```

Assuming you are using `systemd-resolved`:

```bash
# confirm your settings
lxc network get lxdbr0 ipv4.address
lxc network get lxdbr0 ipv6.address
lxc network get lxdbr0 dns.domain # will be blank if default lxd is used

# create a systemd unit file
sudo nano /etc/systemd/system/lxd-dns-lxdbr0.service
# with the contents:
[Unit]
Description=LXD per-link DNS configuration for lxdbr0
BindsTo=sys-subsystem-net-devices-lxdbr0.device
After=sys-subsystem-net-devices-lxdbr0.device

[Service]
Type=oneshot
ExecStart=/usr/bin/resolvectl dns lxdbr0 10.82.244.1
ExecStart=/usr/bin/resolvectl domain lxdbr0 ~lxd
ExecStopPost=/usr/bin/resolvectl revert lxdbr0
RemainAfterExit=yes

[Install]
WantedBy=sys-subsystem-net-devices-lxdbr0.device

# bring up and confirm status
sudo systemctl daemon-reload
sudo systemctl enable --now lxd-dns-lxdbr0
sudo systemctl status lxd-dns-lxdbr0.service
sudo resolvectl status lxdbr0

ping n1
PING n1 (10.82.244.166) 56(84) bytes of data.
64 bytes from n1.lxd (10.82.244.166): icmp_seq=1 ttl=64 time=0.598 ms
./make-image.sh postgresql
./make-image.sh electricsql # takes a long time, erlang, elixir, electricsql from source
```

### Add required packages to node containers:
### Running Tests

```bash
for i in {1..10}; do
lxc exec n${i} -- sh -c "apt-get -qy update && apt-get -qy install openssh-server sudo";
done
./restore-cluster.sh # must be run before each test
lein run test ...
```

### Configure SSH:

Slip your preferred SSH key into each node's `.ssh/.authorized-keys`:
```bash
for i in {1..10}; do
lxc exec n${i} -- sh -c "mkdir -p /root/.ssh && chmod 700 /root/.ssh/";
lxc file push ~/.ssh/id_rsa.pub n${i}/root/.ssh/authorized_keys --uid 0 --gid 0 --mode 644;
done
```

Reset the root password to root, and allow root logins with passwords on each container.
If you've got an SSH agent set up, Jepsen can use that instead.
```bash
for i in {1..10}; do
lxc exec n${i} -- bash -c 'echo -e "root\nroot\n" | passwd root';
lxc exec n${i} -- sed -i 's,^#\?PermitRootLogin .*,PermitRootLogin yes,g' /etc/ssh/sshd_config;
lxc exec n${i} -- systemctl restart sshd;
done
```

Store the node keys unencrypted so that jsch can use them.
If you already have the node keys, they may be unreadable to Jepsen -- remove them from `~/.ssh/known_hosts` and rescan:
```bash
for n in {1..10}; do
ssh-keyscan -t rsa n${n} >> ~/.ssh/known_hosts;
done
```

### Confirm that your host can ssh in:

```bash
ssh root@n1
```

### Stopping and deleting containers:

```bash
for i in {1..10}; do
lxc stop n${i};
lxc delete n${i};
done
```

----

### Real VMs w/Real Clocks

```bash
sudo apt install qemu-system

lxc launch images:debian/12 n1 --vm
```

Allows the clock nemesis to bump, skew, and scramble time in the Jepsen node as it's a real vm with a real clock.

----

## Misc

The `lxc` command's \<Tab\> completion works well, even autocompletes container names.

### LXD/LXC and Docker

There are issues with running LXD and Docker simultaneously, Docker grabs port forwarding.
Running Docker in an LXC container resolves the issue:
[Prevent connectivity issues with LXD and Docker](https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker).
25 changes: 25 additions & 0 deletions lxd/make-image.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#! /usr/bin/bash
set -e

if [ -z "$1" ]; then
echo "Usage: ./make-image.sh image-name";
exit 1;
fi

lxc stop "$1" --force || true
lxc delete "$1" || true

lxc launch images:debian/12 "$1"
sleep 3

lxc file push "./build-$1-image.sh" "$1/" --uid 0 --gid 0 --mode 744

echo "building $1..."
lxc exec "$1" -- bash -c "/build-$1-image.sh"

echo "publishing $1..."
lxc stop "$1"
lxc image delete "$1" || true
lxc publish "$1" --alias "$1"

lxc delete "$1"
13 changes: 13 additions & 0 deletions lxd/restore-cluster.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#! /usr/bin/bash
set -e

lxc delete postgresql --force || true
lxc launch postgresql postgresql

sleep 3

lxc delete electricsql --force || true
lxc launch electricsql electricsql

sleep 3

12 changes: 2 additions & 10 deletions src/causal/checker/without_noops.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,11 @@

(defn without-noops
"A checker that filters:
- noop-nodes from nodes
- noop? true from history"
[checker]
(reify checker/Checker
(check [_this {:keys [nodes noop-nodes] :as test} history opts]
(let [nodes (into #{} nodes)
nodes (->> noop-nodes
(set/difference nodes)
seq)
test (-> test
(assoc :nodes nodes)
(dissoc :noop-nodes))
history (->> history
(check [_this test history opts]
(let [history (->> history
(h/remove (fn [{:keys [noop?] :as op}]
(if (h/invoke? op)
(:noop? (h/completion history op))
Expand Down
Loading

0 comments on commit 07e4402

Please sign in to comment.