node-control: add more ssh support

sni · Oct 14, 2024 · 4c84829 · 4c84829
1 parent f452ecd
commit 4c84829
Show file tree

Hide file tree

Showing 7 changed files with 217 additions and 65 deletions.
diff --git a/MANIFEST b/MANIFEST
@@ -414,6 +414,7 @@ plugins/plugins-available/node-control/preview.png
 plugins/plugins-available/node-control/root/node_control-3.18.js
 plugins/plugins-available/node-control/routes
 plugins/plugins-available/node-control/scripts/omd_update.sh
+plugins/plugins-available/node-control/scripts/runtime.sh
 plugins/plugins-available/node-control/t/controller_node-control.t
 plugins/plugins-available/node-control/templates/node_control.tt
 plugins/plugins-available/node-control/templates/node_control_facts.tt

diff --git a/plugins/plugins-available/node-control/lib/Thruk/Controller/node_control.pm b/plugins/plugins-available/node-control/lib/Thruk/Controller/node_control.pm
@@ -37,6 +37,7 @@ sub index {
     $c->stash->{template}              = 'node_control.tt';
     $c->stash->{infoBoxTitle}          = 'Node Control';
     $c->stash->{plugin_name}           = Thruk::Utils::get_plugin_name(__FILE__, __PACKAGE__);
+    $c->stash->{has_omd}               = $ENV{'OMD_SITE'} ? 1 : 0;
     $c->stash->{'has_jquery_ui'}       = 1;
 
     my $config               = Thruk::NodeControl::Utils::config($c);

diff --git a/plugins/plugins-available/node-control/lib/Thruk/NodeControl/Utils.pm b/plugins/plugins-available/node-control/lib/Thruk/NodeControl/Utils.pm
@@ -310,36 +310,72 @@ sub _ansible_get_facts {
 sub _runtime_data {
     my($c, $peer, $skip_cpu) = @_;
     my $runtime = {};
-    my(undef, $omd_version) = _remote_cmd($c, $peer, 'omd version -b');
-    chomp($omd_version);
-    $runtime->{'omd_version'} = $omd_version;
 
-    my(undef, $omd_status) = _remote_cmd($c, $peer, 'omd status -b');
-    my %services = ($omd_status =~ m/^(\S+?)\s+(\d+)/gmx);
+    my $script = abs_path(Thruk::Base::dirname(__FILE__)."/../../../scripts/runtime.sh");
+    if($skip_cpu) {
+        $script .= " 1";
+    }
+    my($rc, $out) = _remote_script($c, $peer, $script);
+    if($rc != 0) {
+        die("failed to gather runtime data: rc ".$rc." ".$out);
+    }
+    $out =~ s/\r\n/\n/gmx;
+    my %blocks = $out =~ m/<<<([^>]+)>>>\s*(.*?)\s*<<<>>>/sgmx;
+
+    $runtime->{'omd_version'} = $blocks{'OMD VERSION'};
+
+    my %services = ($blocks{'OMD STATUS'} =~ m/^(\S+?)\s+(\d+)/gmx);
     $runtime->{'omd_status'} = \%services;
 
-    my(undef, $omd_site) = _remote_cmd($c, $peer, 'id -un');
-    chomp($omd_site);
-    $runtime->{'omd_site'} = $omd_site;
+    $runtime->{'omd_site'} = $blocks{'ID'};
+
+    my @inst = split/\n/mx, $blocks{'OMD VERSIONS'};
+    my $default;
+    for my $i (@inst) {
+        if($i =~ m/\Q(default)\E/mx) {
+            $i =~ s/\s*\Q(default)\E//gmx;
+            $default = $i;
+        }
+    }
+    @inst = reverse sort @inst;
+    $runtime->{'omd_versions'} = \@inst;
+
+    my %omd_sites;
+    my %in_use;
+    my $sites = $blocks{'OMD SITES'};
+    my @sites = split/\n/mx, $sites;
+    for my $s (@sites) {
+        my($name, $version, $comment) = split/\s+/mx, $s;
+        next if $version eq 'VERSION';
+        $omd_sites{$name} = $version;
+        $in_use{$version} = 1;
+    }
+    $in_use{$default} = 1 if $default;
+
+    my @cleanable;
+    for my $v (@inst) {
+        next if $in_use{$v};
+        push @cleanable, $v;
+    }
+    @inst = reverse sort @inst;
+    $runtime->{'omd_cleanable'} = \@cleanable;
+    $runtime->{'omd_sites'}     = \%omd_sites;
 
-    my(undef, $omd_disk) = _remote_cmd($c, $peer, 'df -k version/.');
-    if($omd_disk =~ m/^.*\s+(\d+)\s+(\d+)\s+(\d+)\s+/gmx) {
+    if($blocks{'OMD DF'} =~ m/^.*\s+(\d+)\s+(\d+)\s+(\d+)\s+/gmx) {
         $runtime->{'omd_disk_total'} = $1;
         $runtime->{'omd_disk_free'}  = $3;
     }
 
-    my(undef, $has_tmux) = _remote_cmd($c, $peer, '/bin/sh -c \'command -v tmux\'');
-    if($has_tmux =~ m/tmux$/gmx) {
-        $runtime->{'has_tmux'} = $has_tmux;
+    if($blocks{'HAS TMUX'} =~ m/tmux$/gmx) {
+        $runtime->{'has_tmux'} = $blocks{'HAS TMUX'};
     }
 
-    if(!$skip_cpu) {
-        my(undef, $omd_cpu) = _remote_cmd($c, $peer, 'top -bn2 | grep Cpu | tail -n 1');
-        if($omd_cpu =~ m/Cpu/gmx) {
-            my @val = split/\s+/mx, $omd_cpu;
-            $runtime->{'omd_cpu_perc'}  = (100-$val[7])/100;
-        }
+    if($blocks{'CPUTOP'} && $blocks{'CPUTOP'} =~ m/Cpu/gmx) {
+        my @val = split/\s+/mx, $blocks{'CPUTOP'};
+        $runtime->{'omd_cpu_perc'}  = (100-$val[7])/100;
     }
+
+
     return($runtime);
 }
 
@@ -366,38 +402,7 @@ sub _ansible_available_packages {
     @pkgs = reverse sort @pkgs;
     @pkgs = map { my $pkg = $_; $pkg =~ s/^omd\-//gmx; $pkg; } @pkgs;
 
-    # get installed omd versions
-    my $installed;
-    (undef, $installed) = _remote_cmd($c, $peer, 'omd versions');
-    my @inst = split/\n/mx, $installed;
-    my $default;
-    for my $i (@inst) {
-        if($i =~ m/\Q(default)\E/mx) {
-            $i =~ s/\s*\Q(default)\E//gmx;
-            $default = $i;
-        }
-    }
-
-    my %omd_sites;
-    my %in_use;
-    my $sites;
-    (undef, $sites) = _remote_cmd($c, $peer, 'omd sites');
-    my @sites = split/\n/mx, $sites;
-    for my $s (@sites) {
-        my($name, $version, $comment) = split/\s+/mx, $s;
-        $omd_sites{$name} = $version;
-        $in_use{$version} = 1;
-    }
-    $in_use{$default} = 1;
-
-    my @cleanable;
-    for my $v (@inst) {
-        next if $in_use{$v};
-        push @cleanable, $v;
-    }
-    @inst = reverse sort @inst;
-
-    return({ omd_packages_available => \@pkgs, omd_versions => \@inst, omd_cleanable => \@cleanable, omd_sites => \%omd_sites });
+    return({ omd_packages_available => \@pkgs });
 }
 
 ##########################################################
@@ -571,10 +576,15 @@ sub _omd_update_step2 {
 
     if($config->{'hook_update_pre'}) {
         print "*** hook_update_pre:\n";
-        my $rc = _remote_run_hook($c, $peer, $config->{'hook_update_pre'}, $env);
-        print "*** hook_update_pre rc: $rc\n";
-        if($rc ne '0') {
-            return _set_job_errored($c, 'updating', $peer->{'key'}, sprintf("update canceled by pre hook (rc: %d)", $rc));
+        eval {
+            my $rc = _remote_run_hook($c, $peer, $config->{'hook_update_pre'}, $env);
+            print "*** hook_update_pre rc: $rc\n";
+            if($rc ne '0') {
+                return _set_job_errored($c, 'updating', $peer->{'key'}, sprintf("update canceled by pre hook (rc: %d)", $rc));
+            }
+        };
+        if($@) {
+            return _set_job_errored($c, 'updating', $peer->{'key'}, $@);
         }
     }
 
@@ -597,15 +607,27 @@ sub _omd_update_step2 {
     my $post_hooks_failed = 0;
     if($config->{'hook_update_post'}) {
         print "*** hook_update_post:\n";
-        my $rc = _remote_run_hook($c, $peer, $config->{'hook_update_post'}, $env);
-        print "*** hook_update_post rc: $rc\n";
+        my $rc = -1;
+        eval {
+            $rc = _remote_run_hook($c, $peer, $config->{'hook_update_post'}, $env);
+            print "*** hook_update_post rc: $rc\n";
+        };
+        if($@) {
+            _info("hook_update_post failed: ".$@);
+        }
         $post_hooks_failed = 1 if $rc ne '0';
     }
 
     if($config->{'hook_update_post_local'}) {
         print "*** hook_update_post_local:\n";
-        my($rc, $out) = _local_run_hook($c, $config->{'hook_update_post_local'}, $env);
-        print "*** hook_update_post_local rc: $rc\n";
+        my($rc, $out);
+        eval {
+            ($rc, $out) = _local_run_hook($c, $config->{'hook_update_post_local'}, $env);
+            print "*** hook_update_post_local rc: $rc\n";
+        };
+        if($@) {
+            _info("hook_update_post_local failed: ".$@);
+        }
         $post_hooks_failed = 1 if $rc ne '0';
     }
 
@@ -838,6 +860,7 @@ sub _omd_cleanup_step2 {
 }
 
 ##########################################################
+# run given command on remote peer
 sub _remote_cmd {
     my($c, $peer, $cmd, $background_options, $env) = @_;
     my($rc, $out, $err);
@@ -885,6 +908,79 @@ sub _remote_cmd {
     return($rc, $out);
 }
 
+##########################################################
+# upload and run a local script
+sub _remote_script {
+    my($c, $peer, $script, $background_options, $env) = @_;
+    my($rc, $out, $err);
+
+    my $args;
+    ($script, $args) = split(/\s+/mx, $script, 2);
+    if(!defined $args) { $args = ""; }
+    if($args ne "") { $args = " ".$args; }
+
+    if(!$peer->{'ssh_ok'} && ($peer->is_local() || $peer->is_peer_machine_reachable_by_http())) {
+        my $script_data = Thruk::Utils::IO::read($script);
+        my $remote_path = sprintf('var/tmp/%s', Thruk::Base::basename($script));
+
+        eval {
+            $peer->rpc($c, 'Thruk::Utils::IO::write', $remote_path, $script_data);
+            ($rc, $out) = _remote_cmd($c, $peer, 'bash '.$remote_path.$args, $background_options, undef, $env);
+        };
+        $err = $@;
+        if(!$err) {
+            return($rc, $out);
+        }
+    }
+
+    # fallback to ssh if possible
+    my $facts     = ansible_get_facts($c, $peer, 0);
+    my $config    = config($c);
+    if(!$config->{'ssh_fallback'}) {
+        die("http(s) connection failed\n".$err) if $err;
+        die("no http(s) control connection available\n");
+    }
+
+    my $server = get_server($c, $peer, $config);
+    my $host_name = $server->{'host_name'};
+    if(!$host_name) {
+        die("http(s) connection failed\n".$err);
+    }
+
+    _debug("remote cmd failed, trying ssh fallback: %s", $err) if $err;
+    my $env_vars = "";
+    for my $key (sort keys %{$env}) {
+        $env_vars .= " --extra-vars $key=\"".$env->{$key}."\"";
+    }
+    my $fullcmd = "ansible all -i ".$server->{'omd_site'}."\@$host_name, -m script -a \"".$script.$args."\"".$env_vars;
+    ($rc, $out) = Thruk::Utils::IO::cmd($fullcmd, { env => { 'ANSIBLE_PYTHON_INTERPRETER' => 'auto_silent' }});
+    if($out =~ m/^.*?\s+\|\s+UNREACHABLE.*?=>/mx) {
+        die("http(s) and ssh connection failed\nhttp(s):\n".$err."\n\nssh:\n".$out) if $err;
+        die("ssh connection failed\n".$out);
+    }
+    $out =~ s/^.*?\s+\|\s+.*?\s+\|\s+rc=\d\s+>>\s*//gmx;
+    if($out =~ m/usage:\ ansible/mx) {
+        confess("ansible command failed: cmd: ".$fullcmd."\n".$out);
+    }
+
+    if($out !~ m/\Q | CHANGED =>\E/gmx) {
+        die("ansible failed: rc $rc ".$out);
+    }
+    $out =~ s/\A.*?\Q | CHANGED =>\E//sgmx;
+    my $jsonreader = Cpanel::JSON::XS->new->utf8;
+       $jsonreader->relaxed();
+    my $f;
+    eval {
+        $f = $jsonreader->decode($out);
+    };
+    if($@) {
+        die("ansible failed to parse json: ".$@);
+    }
+
+    $peer->{'ssh_ok'} = 1;
+    return($rc, $f->{'stdout'});
+}
+
 ##########################################################
 sub _local_run_hook {
     my($c, $hook, $env) = @_;

diff --git a/plugins/plugins-available/node-control/lib/Thruk/Utils/CLI/Nodecontrol.pm b/plugins/plugins-available/node-control/lib/Thruk/Utils/CLI/Nodecontrol.pm
@@ -108,13 +108,13 @@ sub cmd {
                 $facts = Thruk::NodeControl::Utils::ansible_get_facts($c, $peer, 1);
             }
             if($mode eq 'runtime') {
-                $facts = Thruk::NodeControl::Utils::update_runtime_data($c, $peer);
+                $facts = Thruk::NodeControl::Utils::update_runtime_data($c, $peer, 1);
                 if(!$facts->{'ansible_facts'}) {
                     $facts = Thruk::NodeControl::Utils::ansible_get_facts($c, $peer, 1);
                 }
             }
             if(!$facts || $facts->{'last_error'} || $facts->{'last_facts_error'}) {
-                my $err = sprintf("%s updating %s failed: %s\n", $peer->{'name'}, $mode, ($facts->{'last_facts_error'}//$facts->{'last_error'}//'unknown error'));
+                my $err = sprintf("%s updating %s failed: %s\n", $peer->{'name'}, $mode, ($facts->{'last_facts_error'}||$facts->{'last_error'}//'unknown error'));
                 if($ENV{'THRUK_CRON'}) {
                     _warn($err); # don't fill the log with errors from cronjobs
                 } else {

diff --git a/plugins/plugins-available/node-control/scripts/omd_update.sh b/plugins/plugins-available/node-control/scripts/omd_update.sh
@@ -18,11 +18,16 @@ echo "*** updating site $(id -un) from $(omd version -b) to version $OMD_UPDATE.
 echo "*** Site will be stopped during the update, so no progress can be displayed."
 echo "*** this may take a couple of minutes..."; sleep 3 # wait 3 seconds, so this message can be transfered back via http
 
-omd stop
-# make sure it is stopped
+SITE_STARTED=0
 omd status -b > /dev/null 2>&1
 if [ $? -ne 1 ]; then
+    SITE_STARTED=1
     omd stop
+    # make sure it is stopped
+    omd status -b > /dev/null 2>&1
+    if [ $? -ne 1 ]; then
+        omd stop
+    fi
 fi
 
 CMD="omd -f -V $OMD_UPDATE update --conflict=ask"
@@ -69,8 +74,12 @@ if [ "$(omd version -b)" = "$OMD_UPDATE" ]; then
         tmux -f /dev/null send-keys -t $session:$window "exit" C-m
     fi
 
-    echo "%> omd start"
-    omd start
+    if [ $SITE_STARTED = 1 ]; then
+        echo "%> omd start"
+        omd start
+    else
+        echo "SITE was not started before the update, won't start it again"
+    fi
 
     echo "%> omd status"
     omd status

diff --git a/plugins/plugins-available/node-control/scripts/runtime.sh b/plugins/plugins-available/node-control/scripts/runtime.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+echo "<<<ID>>>"
+id -un
+echo "<<<>>>"
+
+echo "<<<OMD VERSION>>>"
+omd version -b
+echo "<<<>>>"
+
+echo "<<<OMD VERSIONS>>>"
+omd versions
+echo "<<<>>>"
+
+echo "<<<OMD SITES>>>"
+omd sites
+echo "<<<>>>"
+
+echo "<<<OMD STATUS>>>"
+omd status -b
+echo "<<<>>>"
+
+echo "<<<OMD DF>>>"
+df -k version/.
+echo "<<<>>>"
+
+echo "<<<HAS TMUX>>>"
+command -v tmux
+echo "<<<>>>"
+
+if [ "$1" = "" ]; then
+  echo "<<<CPUTOP>>>"
+  top -bn2 | grep Cpu | tail -n 1
+  echo "<<<>>>"
+fi
diff --git a/plugins/plugins-available/node-control/templates/node_control.tt b/plugins/plugins-available/node-control/templates/node_control.tt
@@ -1,6 +1,16 @@
 [% PROCESS _header.tt js="plugins/${plugin_name}/node_control-${fileversion}.js" %]
 [% PROCESS _message.tt %]
 
+[% IF !has_omd %]
+<div class="card red alert top-center w-fit text-left">
+  <div class="body textALERT">
+    <b>No OMD:</b> This plugin requires OMD to operate!<br>
+    It cannot be used with a standalone Thruk installation.<br>
+    Read more about OMD at <a href="https://omd.consol.de/" target="_blank"><i class="uil uil-external-link-alt text-sm"></i> omd.consol.de</a>.
+  </div>
+</div>
+[% END %]
+
 <script>
   var logs_menu = {};
   var ms_parallel = [% ms_parallel | html %];