Test multiple distros

.github/workflows/tests.yml

@@ -11,9 +11,15 @@ permissions:
 jobs:
   end-to-end:
     name: End to end testing
+    timeout-minutes: 45
     strategy:
       fail-fast: false
-    timeout-minutes: 45
+      matrix:
+        os:
+          - debian/12
+          - ubuntu/20.04
+          - ubuntu/22.04
+          - ubuntu/24.04
     runs-on:
       - self-hosted
       - cpu-16
@@ -50,7 +56,7 @@ jobs:
         run: |
           cd terraform
           tofu init
-          tofu apply -auto-approve -target=module.baremetal
+          tofu apply -auto-approve -target=module.baremetal -var incus_image=${{ matrix.os }}
 
       - name: Waiting for VMs to boot up
         run: |
@@ -60,6 +66,12 @@ jobs:
         run: |
           cd ansible
           cp hosts.yaml.example hosts.yaml
+
+          if [ "${{ matrix.os }}" = "ubuntu/20.04" ]; then
+            # Ubuntu 20.04's OVN is too old.
+            sed -i "s/ovn_release:.*/ovn_release: \"ppa\"/g" hosts.yaml
+          fi
+
           ansible-playbook deploy.yaml
 
       - name: Post deployment validation

ansible/books/ceph.yaml

@@ -48,7 +48,11 @@
 - name: Ceph - Install packages
   hosts: all
   order: shuffle
+  gather_facts: yes
+  gather_subset:
+    - "distribution_release"
   vars:
+    task_release: "{{ ceph_release | default('squid') }}"
     task_roles: "{{ ceph_roles | default([]) }}"
   any_errors_fatal: true
   tasks:
@@ -68,12 +72,19 @@
         state: present
       when: '"mon" in task_roles'
 
+    - name: Install ceph-mgr dependencies
+      apt:
+        name:
+          - python3-distutils
+        install_recommends: no
+        state: present
+      when: '"mgr" in task_roles and task_release != "distro"'
+
     - name: Install ceph-mgr
       apt:
         name:
           - ceph-mgr
           - python3-bcrypt
-          - python3-distutils
         install_recommends: no
         state: present
       when: '"mgr" in task_roles'
@@ -90,12 +101,19 @@
       apt:
         name:
           - ceph-osd
-          - ceph-volume
           - python3-packaging
         install_recommends: no
         state: present
       when: '"osd" in task_roles'
 
+    - name: Install ceph-volume
+      apt:
+        name:
+          - ceph-volume
+        install_recommends: no
+        state: present
+      when: '"osd" in task_roles and (task_release != "distro" or ansible_distribution_release not in ("bookworm", "focal"))'
+
     - name: Install ceph-rbd-mirror
       apt:
         name:

ansible/books/incus.yaml

@@ -52,6 +52,7 @@
   gather_subset:
     - "default_ipv4"
     - "default_ipv6"
+    - "distribution_release"
   vars:
     task_init: "{{ incus_init | default('{}') }}"
     task_ip_address: "{{ incus_ip_address | default(ansible_default_ipv6['address'] | default(ansible_default_ipv4['address'])) }}"
@@ -103,6 +104,14 @@
         state: present
       when: "task_roles | length > 0 and 'lvm' in task_init['storage'] | dict2items | json_query('[].value.driver')"
 
+    - name: Install ZFS dependencies
+      apt:
+        name:
+          - zfs-dkms
+        install_recommends: yes
+        state: present
+      when: "task_roles | length > 0 and 'zfs' in task_init['storage'] | dict2items | json_query('[].value.driver') and ansible_distribution == 'Debian'"
+
     - name: Install ZFS tools
       apt:
         name:
@@ -175,6 +184,12 @@
       register: cluster_add
       when: 'install.changed and "cluster" in task_roles and task_servers[0] != inventory_hostname'
 
+    - name: Wait 5s to avoid token use before valid
+      ansible.builtin.wait_for:
+        timeout: 5
+      delegate_to: localhost
+      when: 'cluster_add.changed'
+
     - name: Join the cluster
       throttle: 1
       shell:

ansible/hosts.yaml.example

@@ -4,6 +4,7 @@ all:
     ceph_rbd_cache: "2048Mi"
     ceph_rbd_cache_max: "1792Mi"
     ceph_rbd_cache_target: "1536Mi"
+    ceph_release: "distro"
 
     incus_name: "baremetal"
     incus_release: "stable"
@@ -12,7 +13,7 @@ all:
 
     ovn_name: "baremetal"
     ovn_az_name: "zone1"
-    ovn_release: "ppa"
+    ovn_release: "distro"
   children:
     baremetal:
       vars:

terraform/baremetal-incus/main.tf

@@ -30,8 +30,9 @@ resource "incus_profile" "this" {
   description = "Profile to be used by the cluster VMs"
 
   config = {
-    "limits.cpu"    = "4"
-    "limits.memory" = var.memory
+    "limits.cpu"          = "4"
+    "limits.memory"       = var.memory
+    "security.secureboot" = "false"
   }
 
   device {

terraform/main.tf

@@ -3,7 +3,7 @@ module "baremetal" {
 
   project_name   = "dev-incus-deploy"
   instance_names = ["server01", "server02", "server03", "server04", "server05"]
-  image          = "images:ubuntu/22.04"
+  image          = "images:${var.incus_image}"
   memory         = "4GiB"
 
   storage_pool = var.incus_storage_pool
@@ -18,7 +18,7 @@ module "services" {
 
   project_name   = "dev-incus-deploy-services"
   instance_names = ["ceph-mds01", "ceph-mds02", "ceph-mds03", "ceph-mgr01", "ceph-mgr02", "ceph-mgr03", "ceph-rgw01", "ceph-rgw02", "ceph-rgw03"]
-  image          = "images:ubuntu/24.04"
+  image          = "images:${var.incus_image}"
 
   storage_pool = var.incus_storage_pool
   network      = var.incus_network

terraform/terraform.tfvars.example

@@ -9,9 +9,10 @@
 # https://opentofu.org/docs/language/values/variables/#variable-definition-precedence
 
 # Incus variables
-incus_remote         = "local"    # Name of the Incus remote to deploy on (see `incus remote list`)
-incus_storage_pool   = "default"  # Name of the storage pool to use for the VMs and volumes
-incus_network        = "incusbr0" # Name of the network to use for the VMs
+incus_remote         = "local"          # Name of the Incus remote to deploy on (see `incus remote list`)
+incus_image          = "ubuntu/22.04"   # Incus image to use for the VMs
+incus_storage_pool   = "default"        # Name of the storage pool to use for the VMs and volumes
+incus_network        = "incusbr0"       # Name of the network to use for the VMs
 
 # OVN uplink configuration
 ovn_uplink_ipv4_address = "172.31.254.1/24"

terraform/variables.tf

@@ -3,6 +3,11 @@ variable "incus_remote" {
   default = "local"
 }
 
+variable "incus_image" {
+  type    = string
+  default = "ubuntu/22.04"
+}
+
 variable "incus_storage_pool" {
   type    = string
   default = "default"