feat: kubernetes check add healthy flag (#1795)

* feat: kubernetes check add healthy flag * chore: change comment
flanksource · Apr 18, 2024 · 161e1be · 161e1be
1 parent d361b6e
commit 161e1be
Show file tree

Hide file tree

Showing 13 changed files with 48 additions and 17 deletions.
diff --git a/api/v1/checks.go b/api/v1/checks.go
@@ -970,7 +970,12 @@ type KubernetesCheck struct {
 	// Ignore the specified resources from the fetched resources. Can be a glob pattern.
 	Ignore []string `yaml:"ignore,omitempty" json:"ignore,omitempty"`
 	Kind   string   `yaml:"kind" json:"kind"`
-	Ready  *bool    `yaml:"ready,omitempty" json:"ready,omitempty"`
+
+	// Fail the check if any resources are unhealthy
+	Healthy bool `yaml:"healthy,omitempty" json:"healthy,omitempty"`
+
+	// Fail the check if any resources are not ready
+	Ready bool `yaml:"ready,omitempty" json:"ready,omitempty"`
 }
 
 func (c KubernetesCheck) GetType() string {

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/checks/kubernetes.go b/checks/kubernetes.go
@@ -92,17 +92,28 @@ func (c *KubernetesChecker) Check(ctx *context.Context, extConfig external.Check
 			}
 		}
 
-		ctx.Tracef("Found  %d %s in namespace %s with label=%s field=%s", len(resources), check.Kind, namespace, check.Resource.LabelSelector, check.Resource.FieldSelector)
+		ctx.Tracef("Found %d %s in namespace %s with label=%s field=%s", len(resources), check.Kind, namespace, check.Resource.LabelSelector, check.Resource.FieldSelector)
 		for _, resource := range resources {
 			_resource := resource
 			resourceHealth, err := health.GetResourceHealth(&_resource, nil)
-			if err == nil {
+			if err != nil {
+				results.Failf("error getting resource health (%s/%s/%s): %v",
+					resource.GetKind(), resource.GetNamespace(), resource.GetName(), err)
+			} else {
 				resource.Object["healthStatus"] = resourceHealth
-			}
-			if check.Ready != nil && *check.Ready && resourceHealth.Status != health.HealthStatusHealthy {
-				results.Failf("%s/%s/%s is %s: %s", resource.GetKind(), resource.GetNamespace(), resource.GetName(), resourceHealth.Status, resourceHealth.Message)
+
+				if check.Healthy && resourceHealth.Health != health.HealthHealthy {
+					results.Failf("%s/%s/%s is not healthy (health: %s, status: %s): %s",
+						resource.GetKind(), resource.GetNamespace(), resource.GetName(), resourceHealth.Health, resourceHealth.Status, resourceHealth.Message)
+				}
+
+				if check.Ready && !resourceHealth.Ready {
+					results.Failf("%s/%s/%s is not ready (status: %s): %s", resource.GetKind(),
+						resource.GetNamespace(), resource.GetName(), resourceHealth.Status, resourceHealth.Message)
+				}
 			}
 		}
+
 		allResources = append(allResources, resources...)
 	}
 

diff --git a/config/deploy/crd.yaml b/config/deploy/crd.yaml
@@ -4844,6 +4844,9 @@ spec:
                           template:
                             type: string
                         type: object
+                      healthy:
+                        description: Fail the check if any resources are unhealthy
+                        type: boolean
                       icon:
                         description: Icon for overwriting default icon on the dashboard
                         type: string
@@ -4942,6 +4945,7 @@ spec:
                             type: string
                         type: object
                       ready:
+                        description: Fail the check if any resources are not ready
                         type: boolean
                       resource:
                         properties:

diff --git a/config/deploy/manifests.yaml b/config/deploy/manifests.yaml
@@ -4843,6 +4843,9 @@ spec:
                           template:
                             type: string
                         type: object
+                      healthy:
+                        description: Fail the check if any resources are unhealthy
+                        type: boolean
                       icon:
                         description: Icon for overwriting default icon on the dashboard
                         type: string
@@ -4941,6 +4944,7 @@ spec:
                             type: string
                         type: object
                       ready:
+                        description: Fail the check if any resources are not ready
                         type: boolean
                       resource:
                         properties:

diff --git a/config/schemas/canary.schema.json b/config/schemas/canary.schema.json
@@ -2256,6 +2256,9 @@
         "kind": {
           "type": "string"
         },
+        "healthy": {
+          "type": "boolean"
+        },
         "ready": {
           "type": "boolean"
         }

diff --git a/config/schemas/component.schema.json b/config/schemas/component.schema.json
@@ -2513,6 +2513,9 @@
         "kind": {
           "type": "string"
         },
+        "healthy": {
+          "type": "boolean"
+        },
         "ready": {
           "type": "boolean"
         }

diff --git a/config/schemas/health_kubernetes.schema.json b/config/schemas/health_kubernetes.schema.json
@@ -119,6 +119,9 @@
         "kind": {
           "type": "string"
         },
+        "healthy": {
+          "type": "boolean"
+        },
         "ready": {
           "type": "boolean"
         }

diff --git a/config/schemas/topology.schema.json b/config/schemas/topology.schema.json
@@ -2483,6 +2483,9 @@
         "kind": {
           "type": "string"
         },
+        "healthy": {
+          "type": "boolean"
+        },
         "ready": {
           "type": "boolean"
         }

diff --git a/go.mod b/go.mod
@@ -24,7 +24,7 @@ require (
 	github.com/flanksource/commons v1.22.1
 	github.com/flanksource/duty v1.0.419
 	github.com/flanksource/gomplate/v3 v3.24.3
-	github.com/flanksource/is-healthy v1.0.2
+	github.com/flanksource/is-healthy v1.0.4
 	github.com/flanksource/kommons v0.31.4
 	github.com/friendsofgo/errors v0.9.2
 	github.com/go-git/go-git/v5 v5.11.0

diff --git a/go.sum b/go.sum
@@ -866,8 +866,8 @@ github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9
 github.com/flanksource/gomplate/v3 v3.24.3 h1:vBvwlBqLT+8u5xrIcAXEHCeQ5OZPNM89rGTmeMHTMvs=
 github.com/flanksource/gomplate/v3 v3.24.3/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo=
 github.com/flanksource/is-healthy v0.0.0-20230705092916-3b4cf510c5fc/go.mod h1:4pQhmF+TnVqJroQKY8wSnSp+T18oLson6YQ2M0qPHfQ=
-github.com/flanksource/is-healthy v1.0.2 h1:QJvtwIFoz4k8D2atHOciaXCsaFepUXz8laP4c0RBc4E=
-github.com/flanksource/is-healthy v1.0.2/go.mod h1:cFejm0MapnJzgeoG3iizMv+tCIPthe0XqO+3nrhM79c=
+github.com/flanksource/is-healthy v1.0.4 h1:r/dVsi7keR5NGLCTpgO/M6Hah2jRtYNfMh/XJNnxacc=
+github.com/flanksource/is-healthy v1.0.4/go.mod h1:ijdyDDpdRzDtIt1UVZv5WNpAnb8V4hGUz75rnr3Ubhk=
 github.com/flanksource/kommons v0.31.4 h1:zksAgYjZuwPgS8XTejDIWEYB0nPSU1i3Jxcavm/vovI=
 github.com/flanksource/kommons v0.31.4/go.mod h1:70BPMzjTvejsqRyVyAm/ZCeZ176toCvauaZjU03svnE=
 github.com/flanksource/kubectl-neat v1.0.4 h1:t5/9CqgE84oEtB0KitgJ2+WIeLfD+RhXSxYrqb4X8yI=

diff --git a/hack/generate-schemas/go.mod b/hack/generate-schemas/go.mod
@@ -51,7 +51,7 @@ require (
 	github.com/exaring/otelpgx v0.5.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/flanksource/duty v1.0.419 // indirect
-	github.com/flanksource/is-healthy v1.0.2 // indirect
+	github.com/flanksource/is-healthy v1.0.4 // indirect
 	github.com/flanksource/kommons v0.31.4 // indirect
 	github.com/flanksource/kubectl-neat v1.0.4 // indirect
 	github.com/flanksource/postq v1.0.0 // indirect

diff --git a/hack/generate-schemas/go.sum b/hack/generate-schemas/go.sum
@@ -739,8 +739,8 @@ github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9
 github.com/flanksource/gomplate/v3 v3.24.3 h1:vBvwlBqLT+8u5xrIcAXEHCeQ5OZPNM89rGTmeMHTMvs=
 github.com/flanksource/gomplate/v3 v3.24.3/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo=
 github.com/flanksource/is-healthy v0.0.0-20230705092916-3b4cf510c5fc/go.mod h1:4pQhmF+TnVqJroQKY8wSnSp+T18oLson6YQ2M0qPHfQ=
-github.com/flanksource/is-healthy v1.0.2 h1:QJvtwIFoz4k8D2atHOciaXCsaFepUXz8laP4c0RBc4E=
-github.com/flanksource/is-healthy v1.0.2/go.mod h1:cFejm0MapnJzgeoG3iizMv+tCIPthe0XqO+3nrhM79c=
+github.com/flanksource/is-healthy v1.0.4 h1:r/dVsi7keR5NGLCTpgO/M6Hah2jRtYNfMh/XJNnxacc=
+github.com/flanksource/is-healthy v1.0.4/go.mod h1:ijdyDDpdRzDtIt1UVZv5WNpAnb8V4hGUz75rnr3Ubhk=
 github.com/flanksource/kommons v0.31.4 h1:zksAgYjZuwPgS8XTejDIWEYB0nPSU1i3Jxcavm/vovI=
 github.com/flanksource/kommons v0.31.4/go.mod h1:70BPMzjTvejsqRyVyAm/ZCeZ176toCvauaZjU03svnE=
 github.com/flanksource/kubectl-neat v1.0.4 h1:t5/9CqgE84oEtB0KitgJ2+WIeLfD+RhXSxYrqb4X8yI=