feat: disabled checks

checks/ec2.go

@@ -389,7 +389,12 @@ func (c *EC2Checker) Check(ctx *context.Context, extConfig external.Check) pkg.R
 		if len(inner.Spec.EC2) > 0 {
 			return results.Failf("EC2 checks may not be nested to avoid potential recursion.  Skipping inner EC2")
 		}
-		innerResults := RunChecks(ctx.New(ec2Vars))
+
+		innerResults, err := RunChecks(ctx.New(ec2Vars))
+		if err != nil {
+			return results.ErrorMessage(err)
+		}
+
 		for _, result := range innerResults {
 			if !result.Pass {
 				innerFail = true

checks/runchecks.go

@@ -2,6 +2,7 @@ package checks
 
 import (
 	"database/sql"
+	"fmt"
 	"time"
 
 	"github.com/flanksource/canary-checker/api/context"
@@ -11,15 +12,52 @@ import (
 	"github.com/flanksource/commons/logger"
 )
 
-func RunChecks(ctx *context.Context) []*pkg.CheckResult {
+// A list of check types that are permanently disabled.
+var disabledChecks map[string]struct{}
+
+func getDisabledChecks() (map[string]struct{}, error) {
+	if disabledChecks != nil {
+		return disabledChecks, nil
+	}
+
+	rows, err := db.Gorm.Raw("SELECT name FROM disabled_checks").Rows()
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	result := make(map[string]struct{})
+	for rows.Next() {
+		var name string
+		if err := rows.Scan(&name); err != nil {
+			return nil, err
+		}
+
+		result[name] = struct{}{}
+	}
+
+	disabledChecks = result
+	return disabledChecks, nil
+}
+
+func RunChecks(ctx *context.Context) ([]*pkg.CheckResult, error) {
 	var results []*pkg.CheckResult
 
+	disabledChecks, err := getDisabledChecks()
+	if err != nil {
+		return nil, fmt.Errorf("error getting disabled checks: %v", err)
+	}
+
 	// Check if canary is not marked deleted in DB
 	if db.Gorm != nil && ctx.Canary.GetPersistedID() != "" {
 		var deletedAt sql.NullTime
 		err := db.Gorm.Table("canaries").Select("deleted_at").Where("id = ?", ctx.Canary.GetPersistedID()).Scan(&deletedAt).Error
-		if err == nil && deletedAt.Valid {
-			return results
+		if err != nil {
+			return nil, fmt.Errorf("error getting canary: %v", err)
+		}
+
+		if deletedAt.Valid {
+			return results, nil
 		}
 	}
 
@@ -30,13 +68,18 @@ func RunChecks(ctx *context.Context) []*pkg.CheckResult {
 		// t := GetDeadline(ctx.Canary)
 		// ctx, cancel := ctx.WithDeadline(t)
 		// defer cancel()
+
+		if _, ok := disabledChecks[c.Type()]; ok {
+			continue
+		}
+
 		if Checks(checks).Includes(c) {
 			result := c.Run(ctx)
 			results = append(results, transformResults(ctx, result)...)
 		}
 	}
 
-	return processResults(ctx, results)
+	return processResults(ctx, results), nil
 }
 
 func transformResults(ctx *context.Context, in []*pkg.CheckResult) (out []*pkg.CheckResult) {

cmd/run.go

@@ -65,8 +65,15 @@ var Run = &cobra.Command{
 				wg.Add(1)
 				_config := config
 				go func() {
-					queue <- checks.RunChecks(context.New(kommonsClient, k8s, db.Gorm, _config))
-					wg.Done()
+					defer wg.Done()
+
+					res, err := checks.RunChecks(context.New(kommonsClient, k8s, db.Gorm, _config))
+					if err != nil {
+						logger.Errorf("error running checks: %v", err)
+						return
+					}
+
+					queue <- res
 				}()
 			}
 		}

pkg/api/run_now.go

@@ -69,7 +69,10 @@ func RunCanaryHandler(c echo.Context) error {
 		logger.Warnf("failed to get kommons client, checks that read kubernetes configs will fail: %v", err)
 	}
 	ctx := context.New(kommonsClient, k8s, db.Gorm, *canary)
-	result := checks.RunChecks(ctx)
+	result, err := checks.RunChecks(ctx)
+	if err != nil {
+		return errorResonse(c, err, http.StatusInternalServerError)
+	}
 
 	var response RunCanaryResponse
 	response.FromCheckResults(result)

pkg/jobs/canary/canary_jobs.go

@@ -74,7 +74,7 @@ func (job CanaryJob) Run() {
 	}
 
 	if !lock.TryLock() {
-		logger.Debugf("config (id=%s) is already running. skipping this run ...", job.Canary.GetPersistedID())
+		logger.Debugf("canary (id=%s) is already running. skipping this run ...", job.Canary.GetPersistedID())
 		return
 	}
 	defer lock.Unlock()
@@ -96,7 +96,13 @@ func (job CanaryJob) Run() {
 	// Get transformed checks before and after, and then delete the olds ones that are not in new set
 	existingTransformedChecks, _ := db.GetTransformedCheckIDs(job.Canary.GetPersistedID())
 	var newChecksCreated []string
-	results := checks.RunChecks(job.NewContext())
+	results, err := checks.RunChecks(job.NewContext())
+	if err != nil {
+		logger.Errorf("error running checks for canary %s: %v", job.Canary.GetPersistedID(), err)
+		job.Errorf("error running checks for canary %s: %v", job.Canary.GetPersistedID(), err)
+		return
+	}
+
 	for _, result := range results {
 		if job.LogPass && result.Pass || job.LogFail && !result.Pass {
 			logger.Infof(result.String())

pkg/topology/run.go

@@ -201,7 +201,12 @@ func lookup(ctx *ComponentContext, name string, spec v1.CanarySpec) ([]interface
 	canaryCtx.Environment = ctx.Environment
 	canaryCtx.Logger = ctx.Logger
 
-	for _, result := range checks.RunChecks(canaryCtx) {
+	checkResults, err := checks.RunChecks(canaryCtx)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, result := range checkResults {
 		if result.Error != "" {
 			errMsg := fmt.Sprintf("Failed to lookup property: %s. Error in lookup: %s", name, result.Error)
 			logger.Errorf(errMsg)

test/run_test.go

@@ -77,7 +77,12 @@ func runFixture(t *testing.T, name string) {
 			}
 			context := context.New(kommonsClient, k8s, db.Gorm, canary)
 
-			checkResults := checks.RunChecks(context)
+			checkResults, err := checks.RunChecks(context)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
 			for _, res := range checkResults {
 				if res == nil {
 					t.Errorf("Result in %v returned nil:\n", name)