From bf71f74d9b4b2294fe847bdc4a37eb38d9ecb227 Mon Sep 17 00:00:00 2001
From: Sebastian Chlad <sebastianchlad@gmail.com>
Date: Mon, 11 Mar 2024 21:48:18 +0100
Subject: [PATCH 1/2] HPC: pdsh over mrsh using slurm

The patch enables test for pdsh using configures slurm cluster, so it
runs pdsh over mrsh against available slurm partition
---
 tests/hpc/slurm_db.pm               |  5 +++++
 tests/hpc/slurm_master.pm           | 27 +++++++++++++++++++++++++++
 tests/hpc/slurm_master_backup.pm    |  4 ++++
 tests/hpc/slurm_master_backup_db.pm |  5 +++++
 tests/hpc/slurm_slave.pm            |  4 ++++
 5 files changed, 45 insertions(+)

diff --git a/tests/hpc/slurm_db.pm b/tests/hpc/slurm_db.pm
index 98abfd11f908..5ef19528532c 100644
--- a/tests/hpc/slurm_db.pm
+++ b/tests/hpc/slurm_db.pm
@@ -62,6 +62,11 @@ EOF
     ## munge must start before other slurm daemons
     $self->enable_and_start('munge');
     systemctl('is-active munge');
+
+    # Install mrsh and mrsh-server to allow t10 basic
+    zypper_call('in mrsh mrsh-server');
+    $self->enable_and_start('mrlogind.socket mrshd.socket');
+
     $self->prepare_slurmdb_conf();
     record_info("slurmdbd conf", script_output("cat /etc/slurm/slurmdbd.conf"));
     $self->enable_and_start("slurmdbd");
diff --git a/tests/hpc/slurm_master.pm b/tests/hpc/slurm_master.pm
index 4487fdcf5c87..622f3bfe7879 100644
--- a/tests/hpc/slurm_master.pm
+++ b/tests/hpc/slurm_master.pm
@@ -82,6 +82,9 @@ sub run_basic_tests() {
     my %test09 = t09_basic();
     push(@all_results, \%test09);
 
+    my %test10 = t10_basic();
+    push(@all_results, \%test10);
+
     return @all_results;
 }
 
@@ -225,6 +228,26 @@ sub t09_basic() {
     return %results;
 }
 
+sub t10_basic() {
+    my $name = 'pdsh-slurm over mrsh';
+    my $description = 'Basic check of pdsh-slurm over mrsh';
+    my $result = 0;
+
+    my $sinfo_nodeaddr = script_output('sinfo -a --Format=nodeaddr -h');
+    my $pdsh_nodes = script_output("runuser -l nobody -c 'pdsh -R mrsh -P minor /usr/bin/hostname'");
+    my @sinfo_nodeaddr = (split ' ', $sinfo_nodeaddr);
+
+    foreach my $i (@sinfo_nodeaddr) {
+        if (index($pdsh_nodes, $i) == -1) {
+            $result = 1;
+            last;
+        }
+    }
+
+    my %results = generate_results($name, $description, $result);
+    return %results;
+}
+
 #############################################
 ## Accounting tests: for HPC/slurm cluster ##
 #############################################
@@ -466,6 +489,10 @@ sub run ($self) {
 
     $self->enable_and_start('munge');
     systemctl('is-active munge');
+
+    zypper_call('in mrsh mrsh-server');
+    $self->enable_and_start('mrlogind.socket mrshd.socket');
+
     barrier_wait('SLURM_SETUP_DBD');
 
     $self->enable_and_start('slurmctld');
diff --git a/tests/hpc/slurm_master_backup.pm b/tests/hpc/slurm_master_backup.pm
index ce82a4bd9b78..e181f3d9cb1b 100644
--- a/tests/hpc/slurm_master_backup.pm
+++ b/tests/hpc/slurm_master_backup.pm
@@ -31,6 +31,10 @@ sub run ($self) {
     # enable and start munge
     $self->enable_and_start('munge');
 
+    # Install mrsh and mrsh-server to allow t10 basic
+    zypper_call('in mrsh mrsh-server');
+    $self->enable_and_start('mrlogind.socket mrshd.socket');
+
     # enable and start slurmctld
     $self->enable_and_start('slurmctld');
     systemctl 'status slurmctld';
diff --git a/tests/hpc/slurm_master_backup_db.pm b/tests/hpc/slurm_master_backup_db.pm
index 90e202da7931..ffc03866eb97 100644
--- a/tests/hpc/slurm_master_backup_db.pm
+++ b/tests/hpc/slurm_master_backup_db.pm
@@ -36,6 +36,11 @@ sub run ($self) {
     barrier_wait("SLURM_MASTER_SERVICE_ENABLED");
     record_info('slurm conf', script_output('cat /etc/slurm/slurm.conf'));
     $self->enable_and_start('munge');
+
+    # Install mrsh and mrsh-server to allow t10 basic
+    zypper_call('in mrsh mrsh-server');
+    $self->enable_and_start('mrlogind.socket mrshd.socket');
+
     $self->enable_and_start('slurmctld');
     systemctl 'status slurmctld';
     $self->enable_and_start('slurmd');
diff --git a/tests/hpc/slurm_slave.pm b/tests/hpc/slurm_slave.pm
index 32a91fc05c80..dff90da9e832 100644
--- a/tests/hpc/slurm_slave.pm
+++ b/tests/hpc/slurm_slave.pm
@@ -47,6 +47,10 @@ sub run ($self) {
     # enable and start munge
     $self->enable_and_start('munge');
 
+    # Install mrsh and mrsh-server to allow t10 basic
+    zypper_call('in mrsh mrsh-server');
+    $self->enable_and_start('mrlogind.socket mrshd.socket');
+
     # enable and start slurmd
     $self->enable_and_start('slurmd');
     systemctl 'status slurmd';

From 28271f1ac3043155b0eada894f5520ab0007abd5 Mon Sep 17 00:00:00 2001
From: Sebastian Chlad <sebastianchlad@gmail.com>
Date: Tue, 12 Mar 2024 06:17:08 +0100
Subject: [PATCH 2/2] HPC: Make the test name more precise

This make sense since there are two tests now for that slurm plugin
which use ssh and mrsh
---
 tests/hpc/slurm_master.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/hpc/slurm_master.pm b/tests/hpc/slurm_master.pm
index 622f3bfe7879..062de86cba0f 100644
--- a/tests/hpc/slurm_master.pm
+++ b/tests/hpc/slurm_master.pm
@@ -196,7 +196,7 @@ sub t07_basic() {
 }
 
 sub t08_basic() {
-    my $name = 'pdsh-slurm';
+    my $name = 'pdsh-slurm over ssh';
     my $description = 'Basic check of pdsh-slurm over ssh';
     my $result = 0;
     # $slurm_pkg-munge is installed explicitly since slurm_23_02