Merge pull request #1508 from flanksource/job-refactor

Job refactor
flanksource · Dec 20, 2023 · 83ff292 · 83ff292
2 parents ce6b2e3 + 379e581
commit 83ff292
Show file tree

Hide file tree

Showing 58 changed files with 2,333 additions and 838 deletions.
diff --git a/checks/azure_devops_test.go b/checks/azure_devops_test.go
@@ -3,8 +3,8 @@ package checks
 import (
 	"testing"
 
-	"github.com/flanksource/canary-checker/pkg/utils"
 	"github.com/microsoft/azure-devops-go-api/azuredevops/v7/pipelines"
+	"github.com/samber/lo"
 )
 
 func TestMatchPipelineVariables(t *testing.T) {
@@ -27,8 +27,8 @@ func TestMatchPipelineVariables(t *testing.T) {
 				"key2": "value2",
 			},
 			got: &map[string]pipelines.Variable{
-				"key1": {Value: utils.Ptr("value1")},
-				"key2": {Value: utils.Ptr("value2")},
+				"key1": {Value: lo.ToPtr("value1")},
+				"key2": {Value: lo.ToPtr("value2")},
 			},
 			wantResult: true,
 		},
@@ -39,7 +39,7 @@ func TestMatchPipelineVariables(t *testing.T) {
 				"key2": "value2",
 			},
 			got: &map[string]pipelines.Variable{
-				"key1": {Value: utils.Ptr("value1")},
+				"key1": {Value: lo.ToPtr("value1")},
 			},
 			wantResult: false,
 		},
@@ -50,8 +50,8 @@ func TestMatchPipelineVariables(t *testing.T) {
 				"key2": "value2",
 			},
 			got: &map[string]pipelines.Variable{
-				"key1": {Value: utils.Ptr("value1")},
-				"key2": {Value: utils.Ptr("value3")},
+				"key1": {Value: lo.ToPtr("value1")},
+				"key2": {Value: lo.ToPtr("value3")},
 			},
 			wantResult: false,
 		},

diff --git a/cmd/root.go b/cmd/root.go
@@ -47,12 +47,11 @@ var Root = &cobra.Command{
 }
 
 var (
-	httpPort                 = 8080
-	publicEndpoint           = "http://localhost:8080"
-	pushServers, pullServers []string
-	sharedLibrary            []string
-	exposeEnv                bool
-	logPass, logFail         bool
+	httpPort         = 8080
+	publicEndpoint   = "http://localhost:8080"
+	sharedLibrary    []string
+	exposeEnv        bool
+	logPass, logFail bool
 
 	otelcollectorURL string
 	otelServiceName  string
@@ -68,20 +67,20 @@ func ServerFlags(flags *pflag.FlagSet) {
 	_ = flags.MarkDeprecated("devGuiPort", "")
 	_ = flags.MarkDeprecated("metricsPort", "Extra metrics server removed")
 	_ = flags.MarkDeprecated("dev", "")
+	_ = flags.MarkDeprecated("push-servers", "")
+	_ = flags.MarkDeprecated("pull-servers", "")
 
 	flags.StringVar(&publicEndpoint, "public-endpoint", publicEndpoint, "Host on which the health dashboard is exposed. Could be used for generting-links, redirects etc.")
 	flags.StringSliceVar(&runner.IncludeCanaries, "include-check", []string{}, "Run matching canaries - useful for debugging")
 	flags.StringSliceVar(&runner.IncludeTypes, "include-type", []string{}, "Check type to disable")
 	flags.StringSliceVar(&runner.IncludeNamespaces, "include-namespace", []string{}, "Check type to disable")
 	flags.IntVar(&cache.DefaultCacheCount, "maxStatusCheckCount", 5, "Maximum number of past checks in the in memory cache")
-	flags.StringSliceVar(&pushServers, "push-servers", []string{}, "push check results to multiple canary servers")
-	flags.StringSliceVar(&pullServers, "pull-servers", []string{}, "push check results to multiple canary servers")
 	flags.StringVar(&runner.RunnerName, "name", "local", "Server name shown in aggregate dashboard")
 	flags.StringVar(&prometheus.PrometheusURL, "prometheus", "", "URL of the prometheus server that is scraping this instance")
 	flags.StringVar(&db.ConnectionString, "db", "DB_URL", "Connection string for the postgres database. Use embedded:///path/to/dir to use the embedded database")
 	flags.IntVar(&db.DefaultExpiryDays, "cache-timeout", 90, "Cache timeout in days")
 	flags.StringVarP(&cache.DefaultWindow, "default-window", "", "1h", "Default search window")
-	flags.IntVar(&db.CheckStatusRetentionDays, "check-status-retention-period", db.DefaultCheckStatusRetentionDays, "Check status retention period in days")
+	flags.IntVar(&db.CheckStatusRetention, "check-status-retention-period", db.CheckStatusRetention, "Check status retention period in days")
 	flags.IntVar(&db.CheckRetentionDays, "check-retention-period", db.DefaultCheckRetentionDays, "Check retention period in days")
 	flags.IntVar(&db.CanaryRetentionDays, "canary-retention-period", db.DefaultCanaryRetentionDays, "Canary retention period in days")
 	flags.StringVar(&checks.DefaultArtifactConnection, "artifact-connection", "", "Specify the default connection to use for artifacts")

diff --git a/cmd/serve.go b/cmd/serve.go
@@ -23,18 +23,16 @@ import (
 	"github.com/flanksource/canary-checker/pkg/db"
 	"github.com/flanksource/canary-checker/pkg/jobs"
 	canaryJobs "github.com/flanksource/canary-checker/pkg/jobs/canary"
+	"github.com/flanksource/duty"
 
 	"github.com/flanksource/canary-checker/pkg/runner"
 
-	"github.com/flanksource/canary-checker/pkg/push"
-
 	"github.com/flanksource/canary-checker/pkg/api"
 	"github.com/flanksource/canary-checker/pkg/cache"
 	"github.com/flanksource/canary-checker/pkg/prometheus"
 	commonsCtx "github.com/flanksource/commons/context"
 	"github.com/flanksource/commons/logger"
 	dutyContext "github.com/flanksource/duty/context"
-	"github.com/flanksource/duty/models"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
@@ -64,12 +62,6 @@ func setup() {
 	}
 	cache.PostgresCache = cache.NewPostgresCache(db.Pool)
 
-	push.AddServers(pushServers)
-
-	if err := models.SetPropertiesInDBFromFile(db.Gorm, propertiesFile); err != nil {
-		logger.Fatalf("Error setting properties in database: %v", err)
-	}
-
 	kommonsClient, k8s, err := pkg.NewKommonsClient()
 	if err != nil {
 		logger.Warnf("failed to get kommons client, checks that read kubernetes configs will fail: %v", err)
@@ -81,7 +73,9 @@ func setup() {
 		WithKommons(kommonsClient).
 		WithNamespace(runner.WatchNamespace)
 
-	go push.Start()
+	if err := duty.UpdatePropertiesFromFile(apicontext.DefaultContext, propertiesFile); err != nil {
+		logger.Fatalf("Error setting properties in database: %v", err)
+	}
 }
 
 func postgrestResponseModifier(r *http.Response) error {
@@ -123,9 +117,6 @@ func serve() {
 	// PostgREST needs to know how it is exposed to create the correct links
 	db.HTTPEndpoint = publicEndpoint + "/db"
 
-	push.AddServers(pushServers)
-	go push.Start()
-
 	runner.Prometheus, _ = prometheus.NewPrometheusAPI(prometheus.PrometheusURL)
 
 	if debug {
@@ -147,7 +138,6 @@ func serve() {
 		}
 	})
 
-	e.GET("/api", api.CheckSummary)
 	e.GET("/api/summary", api.HealthSummary) // Deprecated: Use Post request for filtering
 	e.POST("/api/summary", api.HealthSummary)
 	e.GET("/about", api.About)