fix: topology job history cleanup

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/fergusstrange/embedded-postgres v1.25.0
 	github.com/flanksource/artifacts v1.0.4
 	github.com/flanksource/commons v1.22.1
-	github.com/flanksource/duty v1.0.414
+	github.com/flanksource/duty v1.0.415
 	github.com/flanksource/gomplate/v3 v3.24.2
 	github.com/flanksource/is-healthy v1.0.2
 	github.com/flanksource/kommons v0.31.4

go.sum

@@ -848,8 +848,8 @@ github.com/flanksource/artifacts v1.0.4 h1:KjQTwsvQ73uHqTK7o4Jwt/RW8fyxJOTJ6JLgz
 github.com/flanksource/artifacts v1.0.4/go.mod h1:wkbdseaTkDo4Q6k6T86vXd4Uy47M6NPCmexgHvCTDl0=
 github.com/flanksource/commons v1.22.1 h1:Ycg8r26bx537UTdAEFgngDW1r2j5bX6Lr3NGxLICpiw=
 github.com/flanksource/commons v1.22.1/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY=
-github.com/flanksource/duty v1.0.414 h1:bcamelkxK3OkNLwmLcXWZ2I8zFr6M5OS9fsROcO/ODI=
-github.com/flanksource/duty v1.0.414/go.mod h1:qQm7wt7TlqV6TFn+PU98bQIYCCau5/3baepXGTQRtV4=
+github.com/flanksource/duty v1.0.415 h1:pwEfYQoPXNW4SGfev90lIIGaaU7wJKYFgWfI7tdc2ic=
+github.com/flanksource/duty v1.0.415/go.mod h1:qQm7wt7TlqV6TFn+PU98bQIYCCau5/3baepXGTQRtV4=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.24.2 h1:WZSriw1MaBhzrDV1IOP9eNsupIPxIHy0yTaMOVhCvsk=
 github.com/flanksource/gomplate/v3 v3.24.2/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo=

hack/generate-schemas/go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/exaring/otelpgx v0.5.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/flanksource/duty v1.0.414 // indirect
+	github.com/flanksource/duty v1.0.415 // indirect
 	github.com/flanksource/is-healthy v1.0.2 // indirect
 	github.com/flanksource/kommons v0.31.4 // indirect
 	github.com/flanksource/kubectl-neat v1.0.4 // indirect

hack/generate-schemas/go.sum

@@ -730,8 +730,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/flanksource/commons v1.22.1 h1:Ycg8r26bx537UTdAEFgngDW1r2j5bX6Lr3NGxLICpiw=
 github.com/flanksource/commons v1.22.1/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY=
-github.com/flanksource/duty v1.0.414 h1:bcamelkxK3OkNLwmLcXWZ2I8zFr6M5OS9fsROcO/ODI=
-github.com/flanksource/duty v1.0.414/go.mod h1:qQm7wt7TlqV6TFn+PU98bQIYCCau5/3baepXGTQRtV4=
+github.com/flanksource/duty v1.0.415 h1:pwEfYQoPXNW4SGfev90lIIGaaU7wJKYFgWfI7tdc2ic=
+github.com/flanksource/duty v1.0.415/go.mod h1:qQm7wt7TlqV6TFn+PU98bQIYCCau5/3baepXGTQRtV4=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.24.2 h1:WZSriw1MaBhzrDV1IOP9eNsupIPxIHy0yTaMOVhCvsk=
 github.com/flanksource/gomplate/v3 v3.24.2/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo=

pkg/db/check_statuses.go

@@ -15,10 +15,8 @@ var RefreshCheckStatusSummary = &job.Job{
 	Schedule:   "@every 1m",
 	JobHistory: true,
 	Retention: job.Retention{
-		Interval: 5 * time.Minute,
-		Success:  1,
-		Failed:   3,
-		Age:      time.Minute * 15,
+		Success: 1,
+		Failed:  3,
 	},
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = CheckStatuses
@@ -31,10 +29,8 @@ var RefreshCheckStatusSummaryAged = &job.Job{
 	Timeout:  60 * time.Minute,
 	Schedule: "@every 1h",
 	Retention: job.Retention{
-		Interval: 60 * time.Minute,
-		Success:  1,
-		Failed:   3,
-		Age:      time.Hour * 3,
+		Success: 1,
+		Failed:  3,
 	},
 	Singleton:  true,
 	JobHistory: true,
@@ -61,7 +57,7 @@ var DeleteOldCheckStatues1d = &job.Job{
 	Name:       "DeleteOldCheckStatuses1d",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.Retention3Day,
+	Retention:  job.RetentionBalanced,
 	Schedule:   "@every 24h",
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = CheckStatuses
@@ -75,7 +71,7 @@ var DeleteOldCheckStatues1h = &job.Job{
 	Name:       "DeleteOldCheckStatuses1h",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.Retention3Day,
+	Retention:  job.RetentionBalanced,
 	Schedule:   "@every 24h",
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = CheckStatuses
@@ -89,7 +85,7 @@ var AggregateCheckStatues1d = &job.Job{
 	Name:       "AggregateCheckStatuses1h",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.RetentionDay,
+	Retention:  job.RetentionBalanced,
 	Schedule:   "@every 1h",
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = CheckStatuses
@@ -103,7 +99,7 @@ var AggregateCheckStatues1h = &job.Job{
 	Name:       "AggregateCheckStatuses1d",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.Retention3Day,
+	Retention:  job.RetentionBalanced,
 	Schedule:   "@every 24h",
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = CheckStatuses

pkg/jobs/canary/canary_jobs.go

@@ -166,7 +166,7 @@ var CleanupDeletedCanaryChecks = &dutyjob.Job{
 	Schedule:   "@every 1h",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  dutyjob.RetentionDay,
+	Retention:  dutyjob.RetentionBalanced,
 	Fn: func(ctx dutyjob.JobRuntime) error {
 		var rows []struct {
 			ID string

pkg/jobs/canary/sync.go

@@ -129,7 +129,7 @@ func newCanaryJob(c CanaryJob) {
 		RunNow:       true,
 		Singleton:    true,
 		JobHistory:   true,
-		Retention:    job.RetentionDay,
+		Retention:    job.RetentionBalanced,
 		ResourceID:   c.DBCanary.ID.String(),
 		ResourceType: "canary",
 		ID:           fmt.Sprintf("%s/%s", c.Canary.Namespace, c.Canary.Name),
@@ -148,7 +148,7 @@ var SyncCanaryJobs = &job.Job{
 	Singleton:  true,
 	RunNow:     true,
 	Schedule:   "@every 5m",
-	Retention:  job.RetentionHour,
+	Retention:  job.RetentionFew,
 	Fn: func(ctx job.JobRuntime) error {
 		canaries, err := db.GetAllCanariesForSync(ctx.Context, runner.WatchNamespace)
 		if err != nil {

pkg/jobs/canary/sync_upstream.go

@@ -32,7 +32,7 @@ var UpstreamJobs = []*job.Job{
 var ReconcileCanaries = &job.Job{
 	Name:       "ReconcileCanaries",
 	Schedule:   "@every 1m",
-	Retention:  job.Retention3Day,
+	Retention:  job.RetentionBalanced,
 	Singleton:  true,
 	JobHistory: true,
 	RunNow:     true,
@@ -57,7 +57,7 @@ var PullUpstreamCanaries = &job.Job{
 	Singleton:  true,
 	RunNow:     true,
 	Schedule:   "@every 10m",
-	Retention:  job.RetentionHour,
+	Retention:  job.RetentionFew,
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = ResourceTypeUpstream
 		ctx.History.ResourceID = UpstreamConf.Host

pkg/jobs/topology/topology_jobs.go

@@ -41,7 +41,7 @@ func newTopologyJob(ctx context.Context, topology pkg.Topology) {
 		Schedule:     v1.Spec.Schedule,
 		Singleton:    true,
 		JobHistory:   true,
-		Retention:    job.RetentionHour,
+		Retention:    job.RetentionFew,
 		ResourceID:   id,
 		ResourceType: "topology",
 		RunNow:       ctx.Properties().On("topology.runNow"),
@@ -125,7 +125,7 @@ var CleanupDeletedTopologyComponents = &job.Job{
 	Schedule:   "@every 1h",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.RetentionDay,
+	Retention:  job.RetentionBalanced,
 	Fn: func(ctx job.JobRuntime) error {
 		var rows []struct {
 			ID string

pkg/topology/cleanup.go

@@ -18,7 +18,7 @@ var CleanupSoftDeletedComponents = &job.Job{
 	Schedule:   "@every 24h",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.Retention3Day,
+	Retention:  job.RetentionBalanced,
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = job.ResourceTypeComponent
 		retention := ctx.Properties().Duration("component.retention.period", DefaultRetention)
@@ -74,7 +74,7 @@ var CleanupChecks = &job.Job{
 	Schedule:   "@every 12h",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.Retention3Day,
+	Retention:  job.RetentionBalanced,
 	Fn: func(ctx job.JobRuntime) error {
 		retention := ctx.Properties().Duration("check.retention.age", DefaultRetention)
 		tx := ctx.DB().Exec(`
@@ -95,7 +95,7 @@ var CleanupCanaries = &job.Job{
 	Schedule:   "@every 12h",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.Retention3Day,
+	Retention:  job.RetentionBalanced,
 	RunNow:     true,
 	Fn: func(ctx job.JobRuntime) error {
 		retention := ctx.Properties().Duration("canary.retention.age", DefaultRetention)
@@ -117,7 +117,7 @@ var CleanupMetricsGauges = &job.Job{
 	Schedule:   "@every 1h",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.RetentionDay,
+	Retention:  job.RetentionBalanced,
 	RunNow:     true,
 	Fn: func(ctx job.JobRuntime) error {
 

pkg/topology/component_check.go

@@ -24,7 +24,7 @@ var ComponentCheckRun = &job.Job{
 	Schedule:   "@every 2m",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.RetentionHour,
+	Retention:  job.RetentionFew,
 	Fn: func(run job.JobRuntime) error {
 		var components = []pkg.Component{}
 		if err := run.DB().Table("components").

pkg/topology/component_config.go

@@ -24,7 +24,7 @@ var ComponentConfigRun = &job.Job{
 	Schedule:   "@every 2m",
 	Singleton:  true,
 	JobHistory: true,
-	Retention:  job.RetentionHour,
+	Retention:  job.RetentionFew,
 	Fn: func(run job.JobRuntime) error {
 		db := run.DB().Session(&gorm.Session{NewDB: true})
 		var components = []pkg.Component{}

pkg/topology/component_relationship.go

@@ -18,7 +18,7 @@ var ComponentRelationshipSync = &job.Job{
 	Name:       "ComponentRelationshipSync",
 	Schedule:   "@every 5m",
 	JobHistory: true,
-	Retention:  job.RetentionHour,
+	Retention:  job.RetentionFew,
 	Singleton:  true,
 	Fn: func(ctx job.JobRuntime) error {
 		var components []models.Component
@@ -73,7 +73,7 @@ var ComponentStatusSummarySync = &job.Job{
 	Name:       "ComponentStatusSummarySync",
 	Schedule:   "@every 2m",
 	JobHistory: true,
-	Retention:  job.RetentionHour,
+	Retention:  job.RetentionFew,
 	Singleton:  true,
 	Fn: func(ctx job.JobRuntime) error {
 		topology, err := Query(ctx.Context, duty.TopologyOptions{Depth: 3})
@@ -136,7 +136,7 @@ var ComponentCostRun = &job.Job{
 	Name:       "ComponentCostSync",
 	JobHistory: true,
 	Singleton:  true,
-	Retention:  job.RetentionDay,
+	Retention:  job.RetentionBalanced,
 	Schedule:   "@every 1h",
 	Fn: func(ctx job.JobRuntime) error {
 		return ctx.DB().Exec(`