From 845d7a7893e253d0f0ddbceea5f14ba97daae340 Mon Sep 17 00:00:00 2001
From: Aravinda Vishwanathapura <aravinda@kadalu.io>
Date: Mon, 16 Aug 2021 17:21:54 +0530
Subject: [PATCH] Exporter self metrics (#18)

* Exporter Health metric is added(0 - Offline, 1 - Up but errors,
  2 - Up and All OK)
* Exporter local metrics(CPU percentage, Memory percentage and Uptime)
  added.

Updates: #15
Fixes: #16
Signed-off-by: Aravinda Vishwanathapura <aravinda@kadalu.io>
---
 shard.lock             |  2 +-
 src/metrics/helpers.cr | 20 ++++++++++++++++----
 src/metrics/local.cr   | 18 ++++++++++++++++++
 3 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/shard.lock b/shard.lock
index 7353699..36b007e 100644
--- a/shard.lock
+++ b/shard.lock
@@ -10,7 +10,7 @@ shards:
 
   glustercli:
     git: https://github.com/aravindavk/glustercli-crystal.git
-    version: 0.1.0+git.commit.b213b433ff886cc063c7fa3a0214212a7fcee3f3
+    version: 0.1.0+git.commit.01d9db7a282bbb8cba661a3be8adf01fa96fc70e
 
   kemal:
     git: https://github.com/kemalcr/kemal.git
diff --git a/src/metrics/helpers.cr b/src/metrics/helpers.cr
index a6f4b8c..222919c 100644
--- a/src/metrics/helpers.cr
+++ b/src/metrics/helpers.cr
@@ -6,7 +6,8 @@ module GlusterMetricsExporter
 
     property volumes = [] of GlusterCLI::VolumeInfo,
       peers = [] of GlusterCLI::NodeInfo,
-      local_metrics = Hash(String, GlusterCLI::LocalMetrics).new
+      local_metrics = Hash(String, GlusterCLI::LocalMetrics).new,
+      exporter_health = Hash(String, Int32).new
 
     def self.collect
       data = MetricsData.new
@@ -34,9 +35,20 @@ module GlusterMetricsExporter
       data.peers.each do |peer|
         url = "http://#{peer.hostname}:#{GlusterMetricsExporter.config.port}/_api/local-metrics"
         # TODO: Handle HTTP error and Connection refused errors
-        response = HTTP::Client.get url
-
-        data.local_metrics[peer.hostname] = GlusterCLI::LocalMetrics.from_json(response.body)
+        begin
+          response = HTTP::Client.get url
+          if response.status_code == 200
+            data.local_metrics[peer.hostname] = GlusterCLI::LocalMetrics.from_json(response.body)
+            data.exporter_health[peer.hostname] = 2
+            next
+          else
+            # Exporter is Up but error
+            data.exporter_health[peer.hostname] = 1
+          end
+        rescue Socket::ConnectError
+          # Exporter is Offline
+          data.exporter_health[peer.hostname] = 0
+        end
       end
 
       data
diff --git a/src/metrics/local.cr b/src/metrics/local.cr
index ec7ffa7..3de25b7 100644
--- a/src/metrics/local.cr
+++ b/src/metrics/local.cr
@@ -16,6 +16,11 @@ module GlusterMetricsExporter
   @@shd_memory_percentage = ShdGauge.new(:shd_memory_percentage, "Self Heal Daemon Memory Percentage")
   @@shd_uptime_seconds = ShdGauge.new(:shd_uptime_seconds, "Self Heal Daemon Uptime in Seconds")
 
+  @@exporter_cpu_percentage = PeerGauge.new(:exporter_cpu_percentage, "Metrics Exporter CPU Percentage")
+  @@exporter_memory_percentage = PeerGauge.new(:exporter_memory_percentage, "Metrics Exporter Memory Percentage")
+  @@exporter_uptime_seconds = PeerGauge.new(:exporter_uptime_seconds, "Metrics Exporter Uptime in Seconds")
+  @@exporter_health = PeerGauge.new(:exporter_health, "Metrics Exporter Health")
+
   def self.clear_local_metrics
     @@brick_cpu_percentage.clear
     @@brick_memory_percentage.clear
@@ -28,6 +33,10 @@ module GlusterMetricsExporter
     @@shd_uptime_seconds.clear
     @@node_uptime_seconds.clear
     @@log_dir_size_bytes.clear
+    @@exporter_cpu_percentage.clear
+    @@exporter_memory_percentage.clear
+    @@exporter_uptime_seconds.clear
+    @@exporter_health.clear
   end
 
   handle_metrics(["local_metrics"]) do |metrics_data|
@@ -68,6 +77,8 @@ module GlusterMetricsExporter
         hostname: peer.hostname,
       }
 
+      # TODO: Handle the case when Metrics exporter is down or error
+
       @@node_uptime_seconds[**peer_labels].set(metrics_data.local_metrics[peer.hostname].node_uptime_seconds)
 
       log_labels = peer_labels.merge({path: "/var/log/glusterfs"})
@@ -86,6 +97,13 @@ module GlusterMetricsExporter
         @@shd_memory_percentage[**shd_labels].set(shd.memory_percentage)
         @@shd_uptime_seconds[**shd_labels].set(shd.uptime_seconds)
       end
+
+      exporter_metrics = metrics_data.local_metrics[peer.hostname].exporter
+      @@exporter_cpu_percentage[**peer_labels].set(exporter_metrics.cpu_percentage)
+      @@exporter_memory_percentage[**peer_labels].set(exporter_metrics.memory_percentage)
+      @@exporter_uptime_seconds[**peer_labels].set(exporter_metrics.uptime_seconds)
+
+      @@exporter_health[**peer_labels].set(metrics_data.exporter_health[peer.hostname])
     end
   end
 end