Merge pull request #718 from mrc0mmand/move-everything-to-c9s

Move (almost) all C8S jobs to C9S

agent-control.py

@@ -109,7 +109,7 @@ def execute_local_command(self, command):
 
         return proc.wait()
 
-    def execute_remote_command(self, command, expected_rc=0, artifacts_dir=None, ignore_rc=False):
+    def execute_remote_command(self, command, expected_rcode=0, artifacts_dir=None, ignore_rc=False):
         """Execute a command on a remote host
 
         Technically the function wraps the command in an ssh command and executes
@@ -119,7 +119,7 @@ def execute_remote_command(self, command, expected_rc=0, artifacts_dir=None, ign
         -------
         command : str
             Command to execute on the remote host
-        expected_rc : int (default: 0)
+        expected_rcode : int or list(int) (default: 0)
             Expected return code
         artifacts_dir : str (default: None)
             If not None and `self.artifacts_storage` is set, all files from the
@@ -136,12 +136,17 @@ def execute_remote_command(self, command, expected_rc=0, artifacts_dir=None, ign
 
         Throws:
         -------
-        An exception if the `ignore_rc` is False and the return code != `expected_rc`
+        An exception if the `ignore_rc` is False and the return code != `expected_rcode`
         """
         assert self.node, "Can't continue without a valid node"
 
         timeout = False
 
+        if isinstance(expected_rcode, list):
+            expected_rcodes = expected_rcode
+        else:
+            expected_rcodes = [expected_rcode]
+
         command_wrapper = [
             "/usr/bin/ssh", "-t",
             "-o", "UserKnownHostsFile=/dev/null",
@@ -156,7 +161,9 @@ def execute_remote_command(self, command, expected_rc=0, artifacts_dir=None, ign
         logging.info("Executing a REMOTE command on node '%s': %s", self.node, command)
         try:
             rc = self.execute_local_command(command_wrapper)
+            logging.info(f"Remote command exited with {rc}")
         except AlarmException:
+            logging.info("Remote command was interrupted by timeout")
             # Delay the timeout exception, so we can fetch artifacts first
             timeout = True
 
@@ -169,9 +176,9 @@ def execute_remote_command(self, command, expected_rc=0, artifacts_dir=None, ign
         if timeout:
             raise AlarmException()
 
-        if not ignore_rc and rc != expected_rc:
+        if not ignore_rc and rc not in expected_rcodes:
             raise RuntimeError("Remote command exited with an unexpected return code "
-                               f"(got: {rc}, expected: {expected_rc}), bailing out")
+                               f"(got: {rc}, expected: {expected_rcode}), bailing out")
         return rc
 
     def fetch_artifacts(self, remote_dir, local_dir):
@@ -272,7 +279,7 @@ def kexec_to_latest(self, args=""):
         assert self.node, "Can't continue without a valid node"
 
         logging.info("Rebooting node %s using kexec", self.node)
-        self.execute_remote_command(f"{GITHUB_CI_REPO}/utils/kexec.sh {args}", 255, ignore_rc=True)
+        self.execute_remote_command(f"{GITHUB_CI_REPO}/utils/kexec.sh {args}", [0, 255])
 
         self.wait_for_node(ping_attempts=10, ssh_attempts=10)
 
@@ -495,6 +502,14 @@ def main():
                     token_file.flush()
                     ac.upload_file(token_file.name, "/.coveralls.token")
 
+            # Upgrade the hypervisor, so we get the latest QEMU & kernel
+            logging.info("PHASE 1.5: Upgrading the hypervisor")
+            # FIXME: drop the custom kernel once RHEL-32384 is resolved
+            kernel_repo = "https://jenkins-systemd.apps.ocp.cloud.ci.centos.org/job/reposync/lastSuccessfulBuild/artifact/repos/mrc0mmand-c9s-kernel-debug-stream9/mrc0mmand-c9s-kernel-debug-stream9.repo"
+            ac.execute_remote_command(f"dnf -y config-manager --add-repo {kernel_repo}")
+            ac.execute_remote_command("dnf -y --refresh upgrade")
+            ac.kexec_to_latest()
+
             # Setup Vagrant and run the tests inside VM
             logging.info("PHASE 2: Run tests in Vagrant VMs")
             command = f"{GITHUB_CI_REPO}/vagrant/vagrant-ci-wrapper.sh -d '{args.vagrant}' -r '{remote_ref}' {args.bootstrap_args}"

jenkins/runners/upstream-coverage.sh

@@ -24,4 +24,4 @@ fi
 git clone https://github.com/systemd/systemd-centos-ci
 cd systemd-centos-ci
 
-./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 --vagrant arch-coverage ${ARGS:+"${ARGS[@]}"}
+./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --vagrant arch-coverage ${ARGS:+"${ARGS[@]}"}

jenkins/runners/upstream-cron-build.sh

@@ -41,12 +41,12 @@ cd systemd-centos-ci
 run_arch_remaining_sanitizer_job() {
     # Run the "leftover" ASan/UBSan job (i.e. the one which is not run by
     # the `upstream-vagrant-archlinux-sanitizers` job for each PR)
-    ./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 \
+    ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \
                        --vagrant arch-sanitizers-gcc \
                        --no-index \
                        ${ARGS:+"${ARGS[@]}"}
 
-    #./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 \
+    #./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \
     #                   --vagrant arch-sanitizers-clang \
     #                   --no-index \
     #                   ${ARGS:+"${ARGS[@]}"}

jenkins/runners/upstream-vagrant-archlinux-sanitizers-stable.sh

@@ -48,13 +48,13 @@ git clone https://github.com/systemd/systemd-centos-ci
 cd systemd-centos-ci
 
 # Run both jobs, since we don't run the other-half-of-this-job in cron in this case
-./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 \
+./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \
                    --bootstrap-args='-s https://github.com/systemd/systemd-stable.git' \
                    --no-index \
                    --vagrant arch-sanitizers-gcc \
                    ${ARGS:+"${ARGS[@]}"}
 
-./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 \
+./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \
                    --bootstrap-args='-s https://github.com/systemd/systemd-stable.git' \
                    --no-index \
                    --vagrant arch-sanitizers-clang \

jenkins/runners/upstream-vagrant-archlinux-sanitizers.sh

@@ -47,12 +47,12 @@ fi
 git clone https://github.com/systemd/systemd-centos-ci
 cd systemd-centos-ci
 
-#./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 \
+#./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \
 #                   --vagrant arch-sanitizers-gcc \
 #                   --no-index \
 #                   ${ARGS:+"${ARGS[@]}"}
 
-./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 \
+./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \
                    --vagrant arch-sanitizers-clang \
                    --no-index \
                    ${ARGS:+"${ARGS[@]}"}

jenkins/runners/upstream-vagrant-archlinux-stable.sh

@@ -33,7 +33,7 @@ fi
 git clone https://github.com/systemd/systemd-centos-ci
 cd systemd-centos-ci
 
-./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 \
+./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \
                    --bootstrap-args='-s https://github.com/systemd/systemd-stable.git' \
                    --vagrant arch \
                    ${ARGS:+"${ARGS[@]}"}

jenkins/runners/upstream-vagrant-archlinux.sh

@@ -33,4 +33,4 @@ fi
 git clone https://github.com/systemd/systemd-centos-ci
 cd systemd-centos-ci
 
-./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 --vagrant arch ${ARGS:+"${ARGS[@]}"}
+./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --vagrant arch ${ARGS:+"${ARGS[@]}"}

jenkins/runners/upstream-vagrant-rawhide-selinux.sh

@@ -20,4 +20,4 @@ ARGS=()
 git clone https://github.com/systemd/systemd-centos-ci
 cd systemd-centos-ci
 
-./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 --vagrant rawhide-selinux ${ARGS:+"${ARGS[@]}"}
+./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --vagrant rawhide-selinux ${ARGS:+"${ARGS[@]}"}

jenkins/runners/vagrant-make-cache.sh

@@ -45,11 +45,11 @@ set +e
     echo "Updating the systemd Arch Linux image"
 
     # Generate a new image with '-new' suffix
-    ./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 --no-index --vagrant-sync Vagrantfile_archlinux_systemd ${ARGS:+"${ARGS[@]}"}
+    ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --no-index --vagrant-sync Vagrantfile_archlinux_systemd ${ARGS:+"${ARGS[@]}"}
     # Check if it doesn't break anything
-    ./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 --no-index --vagrant arch-new ${ARGS:+"${ARGS[@]}"}
-    ./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 --no-index --vagrant arch-sanitizers-clang-new ${ARGS:+"${ARGS[@]}"}
-    ./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 --no-index --vagrant arch-sanitizers-gcc-new ${ARGS:+"${ARGS[@]}"}
+    ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --no-index --vagrant arch-new ${ARGS:+"${ARGS[@]}"}
+    ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --no-index --vagrant arch-sanitizers-clang-new ${ARGS:+"${ARGS[@]}"}
+    ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --no-index --vagrant arch-sanitizers-gcc-new ${ARGS:+"${ARGS[@]}"}
     # Overwrite the production image with the just tested one. Since the CentOS CI
     # artifact server supports only rsync protocol, use a single-purpose script
     # to do that
@@ -67,7 +67,7 @@ fi
     echo "Updating the systemd/selinux Fedora Rawhide image"
 
     # Generate a new image with '-new' suffix
-    ./agent-control.py --pool metal-ec2-c5n-centos-8s-x86_64 --no-index --vagrant-sync Vagrantfile_rawhide_selinux ${ARGS:+"${ARGS[@]}"}
+    ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --no-index --vagrant-sync Vagrantfile_rawhide_selinux ${ARGS:+"${ARGS[@]}"}
     # Overwrite the production image with the just tested one. Since the CentOS CI
     # artifact server supports only rsync protocol, use a single-purpose script
     # to do that

utils/kexec.sh

@@ -9,7 +9,7 @@ if [[ "${1:-}" == "-u" ]]; then
 fi
 
 RUNNING_KERNEL="$(uname -r)"
-LATEST_KERNEL="$(rpm -q kernel --qf '%{EVR}.%{ARCH}\n' | sort -Vr | head -n1)"
+LATEST_KERNEL="$(rpm -q kernel --qf '%{VERSION}-%{RELEASE}.%{ARCH}\n' | sort -Vr | head -n1)"
 
 echo "Running kernel: $RUNNING_KERNEL"
 echo "Latest installed kernel: $LATEST_KERNEL"

vagrant/boxes/Vagrantfile_archlinux_systemd

@@ -23,7 +23,7 @@ Vagrant.configure("2") do |config|
         libvirt.machine_type = "q35"
 
         # Emulate UEFI using OVMF
-        libvirt.loader = "/usr/share/OVMF/OVMF_CODE.fd"
+        libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
         libvirt.nvram = ovmf_nvram
 
         # Emulate TPM 2.0 using swtpm
@@ -33,7 +33,7 @@ Vagrant.configure("2") do |config|
     end
 
     config.trigger.before [:up, :provision] do |trigger|
-        trigger.run = {inline: "cp -u /usr/share/OVMF/OVMF_VARS.fd #{ovmf_nvram}"}
+        trigger.run = {inline: "cp -u /usr/share/edk2/ovmf/OVMF_VARS.fd #{ovmf_nvram}"}
     end
 
     config.trigger.after [:destroy] do |trigger|

vagrant/boxes/Vagrantfile_rawhide_selinux

@@ -24,7 +24,7 @@ Vagrant.configure("2") do |config|
         libvirt.machine_type = "q35"
 
         # Emulate UEFI using OVMF
-        libvirt.loader = "/usr/share/OVMF/OVMF_CODE.fd"
+        libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
         libvirt.nvram = ovmf_nvram
 
         # Emulate TPM 2.0 using swtpm
@@ -34,7 +34,7 @@ Vagrant.configure("2") do |config|
     end
 
     config.trigger.before [:up, :provision] do |trigger|
-        trigger.run = {inline: "cp -u /usr/share/OVMF/OVMF_VARS.fd #{ovmf_nvram}"}
+        trigger.run = {inline: "cp -u /usr/share/edk2/ovmf/OVMF_VARS.fd #{ovmf_nvram}"}
     end
 
     config.trigger.after [:destroy] do |trigger|

vagrant/test_scripts/test-arch.sh

@@ -43,6 +43,7 @@ fi
 
 # Disable swap, since it seems to cause CPU soft lock-ups in some cases
 swapoff -av
+swapon --show
 
 pushd /build || { echo >&2 "Can't pushd to /build"; exit 1; }
 

vagrant/vagrant-make-cache.sh

@@ -152,13 +152,13 @@ Vagrant.configure("2") do |config|
     config.vm.provider :libvirt do |libvirt|
         libvirt.random :model => 'random'
         libvirt.machine_type = "q35"
-        libvirt.loader = "/usr/share/OVMF/OVMF_CODE.fd"
+        libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
         libvirt.nvram = "/tmp/OVMF_VARS.fd"
     end
 
 end
 EOF
-cp -fvL /usr/share/OVMF/OVMF_VARS.fd /tmp
+cp -fvL /usr/share/edk2/ovmf/OVMF_VARS.fd /tmp
 chmod o+rw /tmp/OVMF_VARS.fd
 vagrant up --no-tty --provider=libvirt
 # shellcheck disable=SC2016

vagrant/vagrant-setup.sh

@@ -69,7 +69,7 @@ fi
 
 if ! vagrant plugin list | grep vagrant-libvirt; then
     # Install vagrant-libvirt dependencies
-    cmd_retry dnf -y install gcc libguestfs-tools-c libvirt libvirt-devel libgcrypt make qemu-kvm ruby-devel
+    cmd_retry dnf -y --enablerepo crb install gcc libguestfs-tools-c libvirt libvirt-devel libgcrypt make qemu-kvm ruby-devel
     # Start libvirt daemon
     systemctl start libvirtd
     systemctl status libvirtd
@@ -88,9 +88,6 @@ fi
 vagrant --version
 vagrant plugin list
 
-# Install OVMF so we can enable UEFI in the Vagrant VMs
-# Local mirror of https://copr.fedorainfracloud.org/coprs/mrc0mmand/systemd-centos-ci-centos8/
-cmd_retry dnf -y config-manager --add-repo "https://jenkins-systemd.apps.ocp.cloud.ci.centos.org/job/reposync/lastSuccessfulBuild/artifact/repos/mrc0mmand-systemd-centos-ci-centos8-stream8/mrc0mmand-systemd-centos-ci-centos8-stream8.repo"
 cmd_retry dnf -y install edk2-ovmf
 
 # Configure NFS for Vagrant's shared folders

vagrant/vagrantfiles/Vagrantfile_arch

@@ -53,9 +53,8 @@ Vagrant.configure("2") do |config|
         # Collect output from a serial console into a file to make debugging easier
         # The -nographic option allows us to collect BIOS messages as well
         libvirt.qemuargs :value => "-nographic"
-        libvirt.qemuargs :value => "-serial"
         # This file needs to be collected later by vagrant-ci-wrapper.sh
-        libvirt.qemuargs :value => "file:/tmp/vagrant-arch-serial-console.log"
+        libvirt.serial :type => "file", :source => {:path => "/tmp/vagrant-arch-serial-console.log"}
 
         # Pass through /dev/random from the host to the VM
         libvirt.random :model => 'random'
@@ -67,7 +66,7 @@ Vagrant.configure("2") do |config|
         libvirt.machine_type = "q35"
 
         # Emulate UEFI using OVMF
-        libvirt.loader = "/usr/share/OVMF/OVMF_CODE.fd"
+        libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
         libvirt.nvram = ovmf_nvram
 
         # Emulate TPM 2.0 using swtpm
@@ -84,7 +83,7 @@ Vagrant.configure("2") do |config|
     ]
 
     config.trigger.before [:up, :provision] do |trigger|
-        trigger.run = {inline: "cp -u /usr/share/OVMF/OVMF_VARS.fd #{ovmf_nvram}"}
+        trigger.run = {inline: "cp -u /usr/share/edk2/ovmf/OVMF_VARS.fd #{ovmf_nvram}"}
     end
 
     config.trigger.after [:destroy] do |trigger|

vagrant/vagrantfiles/Vagrantfile_rawhide

@@ -49,9 +49,8 @@ Vagrant.configure("2") do |config|
         # Collect output from a serial console into a file to make debugging easier
         # The -nographic option allows us to collect BIOS messages as well
         libvirt.qemuargs :value => "-nographic"
-        libvirt.qemuargs :value => "-serial"
         # This file needs to be collected later by vagrant-ci-wrapper.sh
-        libvirt.qemuargs :value => "file:/tmp/vagrant-rawhide-serial-console.log"
+        libvirt.serial :type => "file", :source => {:path => "/tmp/vagrant-arch-serial-console.log"}
 
         # Pass through /dev/random from the host to the VM
         libvirt.random :model => 'random'
@@ -63,7 +62,7 @@ Vagrant.configure("2") do |config|
         libvirt.machine_type = "q35"
 
         # Emulate UEFI using OVMF
-        libvirt.loader = "/usr/share/OVMF/OVMF_CODE.fd"
+        libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
         libvirt.nvram = ovmf_nvram
 
         # Emulate TPM 2.0 using swtpm
@@ -73,7 +72,7 @@ Vagrant.configure("2") do |config|
     end
 
     config.trigger.before [:up, :provision] do |trigger|
-        trigger.run = {inline: "cp -u /usr/share/OVMF/OVMF_VARS.fd #{ovmf_nvram}"}
+        trigger.run = {inline: "cp -u /usr/share/edk2/ovmf/OVMF_VARS.fd #{ovmf_nvram}"}
     end
 
     config.trigger.after [:destroy] do |trigger|