diff --git a/.changelog/24112.txt b/.changelog/24112.txt new file mode 100644 index 00000000000..6383f63ed23 --- /dev/null +++ b/.changelog/24112.txt @@ -0,0 +1,3 @@ +```release-note:bug +state: Fixed setting GC threshold to more than 72hrs being ignored +``` diff --git a/api/csi.go b/api/csi.go index 65e1ca569f3..492b011aa27 100644 --- a/api/csi.go +++ b/api/csi.go @@ -351,6 +351,11 @@ type CSIVolume struct { CreateIndex uint64 ModifyIndex uint64 + // CreateTime stored as UnixNano + CreateTime int64 + // ModifyTime stored as UnixNano + ModifyTime int64 + // ExtraKeysHCL is used by the hcl parser to report unexpected keys ExtraKeysHCL []string `hcl1:",unusedKeys" json:"-"` } @@ -401,6 +406,11 @@ type CSIVolumeListStub struct { CreateIndex uint64 ModifyIndex uint64 + + // CreateTime stored as UnixNano + CreateTime int64 + // ModifyTime stored as UnixNano + ModifyTime int64 } type CSIVolumeListExternalResponse struct { @@ -543,6 +553,11 @@ type CSIPlugin struct { NodesExpected int CreateIndex uint64 ModifyIndex uint64 + + // CreateTime stored as UnixNano + CreateTime int64 + // ModifyTime stored as UnixNano + ModifyTime int64 } type CSIPluginListStub struct { diff --git a/api/deployments.go b/api/deployments.go index 6785b8f6d6a..94e2e97323f 100644 --- a/api/deployments.go +++ b/api/deployments.go @@ -193,6 +193,10 @@ type Deployment struct { CreateIndex uint64 ModifyIndex uint64 + + // Creation and modification times, stored as UnixNano + CreateTime int64 + ModifyTime int64 } // DeploymentState tracks the state of a deployment for a given task group. @@ -261,6 +265,9 @@ type DeploymentPromoteRequest struct { // Groups is used to set the promotion status per task group Groups []string + // PromotedAt is the timestamp stored as Unix nano + PromotedAt int64 + WriteRequest } diff --git a/nomad/blocked_evals.go b/nomad/blocked_evals.go index f227a3c7cfc..3889294bd7e 100644 --- a/nomad/blocked_evals.go +++ b/nomad/blocked_evals.go @@ -60,11 +60,11 @@ type BlockedEvals struct { // blocked eval exists for each job. The value is the blocked evaluation ID. jobs map[structs.NamespacedID]string - // unblockIndexes maps computed node classes or quota name to the index in - // which they were unblocked. This is used to check if an evaluation could - // have been unblocked between the time they were in the scheduler and the - // time they are being blocked. - unblockIndexes map[string]uint64 + // unblockIndexes maps computed node classes or quota name to the index and + // time at which they were unblocked. This is used to check if an + // evaluation could have been unblocked between the time they were in the + // scheduler and the time they are being blocked. + unblockIndexes map[string]unblockEvent // duplicates is the set of evaluations for jobs that had pre-existing // blocked evaluations. These should be marked as cancelled since only one @@ -76,14 +76,16 @@ type BlockedEvals struct { // duplicates. duplicateCh chan struct{} - // timetable is used to correlate indexes with their insertion time. This - // allows us to prune based on time. - timetable *TimeTable - // stopCh is used to stop any created goroutines. stopCh chan struct{} } +// unblockEvent keeps a record of the index and time of the unblock +type unblockEvent struct { + index uint64 + timestamp time.Time +} + // capacityUpdate stores unblock data. type capacityUpdate struct { computedClass string @@ -107,7 +109,7 @@ func NewBlockedEvals(evalBroker *EvalBroker, logger hclog.Logger) *BlockedEvals escaped: make(map[string]wrappedEval), system: newSystemEvals(), jobs: make(map[structs.NamespacedID]string), - unblockIndexes: make(map[string]uint64), + unblockIndexes: make(map[string]unblockEvent), capacityChangeCh: make(chan *capacityUpdate, unblockBuffer), duplicateCh: make(chan struct{}, 1), stopCh: make(chan struct{}), @@ -143,12 +145,6 @@ func (b *BlockedEvals) SetEnabled(enabled bool) { } } -func (b *BlockedEvals) SetTimetable(timetable *TimeTable) { - b.l.Lock() - b.timetable = timetable - b.l.Unlock() -} - // Block tracks the passed evaluation and enqueues it into the eval broker when // a suitable node calls unblock. func (b *BlockedEvals) Block(eval *structs.Evaluation) { @@ -303,10 +299,10 @@ func latestEvalIndex(eval *structs.Evaluation) uint64 { // the lock held. func (b *BlockedEvals) missedUnblock(eval *structs.Evaluation) bool { var max uint64 = 0 - for id, index := range b.unblockIndexes { + for id, u := range b.unblockIndexes { // Calculate the max unblock index - if max < index { - max = index + if max < u.index { + max = u.index } // The evaluation is blocked because it has hit a quota limit not class @@ -315,7 +311,7 @@ func (b *BlockedEvals) missedUnblock(eval *structs.Evaluation) bool { if eval.QuotaLimitReached != id { // Not a match continue - } else if eval.SnapshotIndex < index { + } else if eval.SnapshotIndex < u.index { // The evaluation was processed before the quota specification was // updated, so unblock the evaluation. return true @@ -326,7 +322,7 @@ func (b *BlockedEvals) missedUnblock(eval *structs.Evaluation) bool { } elig, ok := eval.ClassEligibility[id] - if !ok && eval.SnapshotIndex < index { + if !ok && eval.SnapshotIndex < u.index { // The evaluation was processed and did not encounter this class // because it was added after it was processed. Thus for correctness // we need to unblock it. @@ -335,7 +331,7 @@ func (b *BlockedEvals) missedUnblock(eval *structs.Evaluation) bool { // The evaluation could use the computed node class and the eval was // processed before the last unblock. - if elig && eval.SnapshotIndex < index { + if elig && eval.SnapshotIndex < u.index { return true } } @@ -415,7 +411,7 @@ func (b *BlockedEvals) Unblock(computedClass string, index uint64) { // Store the index in which the unblock happened. We use this on subsequent // block calls in case the evaluation was in the scheduler when a trigger // occurred. - b.unblockIndexes[computedClass] = index + b.unblockIndexes[computedClass] = unblockEvent{index, time.Now()} // Capture chan in lock as Flush overwrites it ch := b.capacityChangeCh @@ -450,7 +446,7 @@ func (b *BlockedEvals) UnblockQuota(quota string, index uint64) { // Store the index in which the unblock happened. We use this on subsequent // block calls in case the evaluation was in the scheduler when a trigger // occurred. - b.unblockIndexes[quota] = index + b.unblockIndexes[quota] = unblockEvent{index, time.Now()} ch := b.capacityChangeCh done := b.stopCh b.l.Unlock() @@ -479,10 +475,11 @@ func (b *BlockedEvals) UnblockClassAndQuota(class, quota string, index uint64) { // Store the index in which the unblock happened. We use this on subsequent // block calls in case the evaluation was in the scheduler when a trigger // occurred. + now := time.Now() if quota != "" { - b.unblockIndexes[quota] = index + b.unblockIndexes[quota] = unblockEvent{index, now} } - b.unblockIndexes[class] = index + b.unblockIndexes[class] = unblockEvent{index, now} // Capture chan inside the lock to prevent a race with it getting reset // in Flush. @@ -699,8 +696,7 @@ func (b *BlockedEvals) Flush() { b.captured = make(map[string]wrappedEval) b.escaped = make(map[string]wrappedEval) b.jobs = make(map[structs.NamespacedID]string) - b.unblockIndexes = make(map[string]uint64) - b.timetable = nil + b.unblockIndexes = make(map[string]unblockEvent) b.duplicates = nil b.capacityChangeCh = make(chan *capacityUpdate, unblockBuffer) b.stopCh = make(chan struct{}) @@ -781,18 +777,13 @@ func (b *BlockedEvals) prune(stopCh <-chan struct{}) { } // pruneUnblockIndexes is used to prune any tracked entry that is excessively -// old. This protects againsts unbounded growth of the map. +// old. This protects against unbounded growth of the map. func (b *BlockedEvals) pruneUnblockIndexes(cutoff time.Time) { b.l.Lock() defer b.l.Unlock() - if b.timetable == nil { - return - } - - oldThreshold := b.timetable.NearestIndex(cutoff) - for key, index := range b.unblockIndexes { - if index < oldThreshold { + for key, u := range b.unblockIndexes { + if u.timestamp.Before(cutoff) { delete(b.unblockIndexes, key) } } diff --git a/nomad/core_sched.go b/nomad/core_sched.go index 3433daac7cc..73431b6e81d 100644 --- a/nomad/core_sched.go +++ b/nomad/core_sched.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "fmt" - "math" "strings" "time" @@ -28,6 +27,18 @@ type CoreScheduler struct { srv *Server snap *state.StateSnapshot logger log.Logger + + // custom GC Threshold values can be used by unit tests to simulate time + // manipulation + customJobGCThreshold time.Duration + customEvalGCThreshold time.Duration + customBatchEvalGCThreshold time.Duration + customNodeGCThreshold time.Duration + customDeploymentGCThreshold time.Duration + customCSIVolumeClaimGCThreshold time.Duration + customCSIPluginGCThreshold time.Duration + customACLTokenExpirationGCThreshold time.Duration + customRootKeyGCThreshold time.Duration } // NewCoreScheduler is used to return a new system scheduler instance @@ -45,13 +56,13 @@ func (c *CoreScheduler) Process(eval *structs.Evaluation) error { job := strings.Split(eval.JobID, ":") // extra data can be smuggled in w/ JobID switch job[0] { case structs.CoreJobEvalGC: - return c.evalGC(eval) + return c.evalGC() case structs.CoreJobNodeGC: return c.nodeGC(eval) case structs.CoreJobJobGC: return c.jobGC(eval) case structs.CoreJobDeploymentGC: - return c.deploymentGC(eval) + return c.deploymentGC() case structs.CoreJobCSIVolumeClaimGC: return c.csiVolumeClaimGC(eval) case structs.CoreJobCSIPluginGC: @@ -78,10 +89,10 @@ func (c *CoreScheduler) forceGC(eval *structs.Evaluation) error { if err := c.jobGC(eval); err != nil { return err } - if err := c.evalGC(eval); err != nil { + if err := c.evalGC(); err != nil { return err } - if err := c.deploymentGC(eval); err != nil { + if err := c.deploymentGC(); err != nil { return err } if err := c.csiPluginGC(eval); err != nil { @@ -116,8 +127,15 @@ func (c *CoreScheduler) jobGC(eval *structs.Evaluation) error { return err } - oldThreshold := c.getThreshold(eval, "job", - "job_gc_threshold", c.srv.config.JobGCThreshold) + var threshold time.Duration + threshold = c.srv.config.JobGCThreshold + + // custom threshold override + if c.customJobGCThreshold != 0 { + threshold = c.customJobGCThreshold + } + + cutoffTime := c.getCutoffTime(threshold) // Collect the allocations, evaluations and jobs to GC var gcAlloc, gcEval []string @@ -128,7 +146,8 @@ OUTER: job := i.(*structs.Job) // Ignore new jobs. - if job.CreateIndex > oldThreshold { + st := time.Unix(0, job.SubmitTime) + if st.After(cutoffTime) { continue } @@ -142,7 +161,7 @@ OUTER: allEvalsGC := true var jobAlloc, jobEval []string for _, eval := range evals { - gc, allocs, err := c.gcEval(eval, oldThreshold, true) + gc, allocs, err := c.gcEval(eval, cutoffTime, true) if err != nil { continue OUTER } else if gc { @@ -244,7 +263,7 @@ func (c *CoreScheduler) partitionJobReap(jobs []*structs.Job, leaderACL string, } // evalGC is used to garbage collect old evaluations -func (c *CoreScheduler) evalGC(eval *structs.Evaluation) error { +func (c *CoreScheduler) evalGC() error { // Iterate over the evaluations ws := memdb.NewWatchSet() iter, err := c.snap.Evals(ws, false) @@ -252,22 +271,32 @@ func (c *CoreScheduler) evalGC(eval *structs.Evaluation) error { return err } - oldThreshold := c.getThreshold(eval, "eval", - "eval_gc_threshold", c.srv.config.EvalGCThreshold) - batchOldThreshold := c.getThreshold(eval, "eval", - "batch_eval_gc_threshold", c.srv.config.BatchEvalGCThreshold) + var threshold, batchThreshold time.Duration + threshold = c.srv.config.EvalGCThreshold + batchThreshold = c.srv.config.BatchEvalGCThreshold + + // custom threshold override + if c.customEvalGCThreshold != 0 { + threshold = c.customEvalGCThreshold + } + if c.customBatchEvalGCThreshold != 0 { + batchThreshold = c.customBatchEvalGCThreshold + } + + cutoffTime := c.getCutoffTime(threshold) + batchCutoffTime := c.getCutoffTime(batchThreshold) // Collect the allocations and evaluations to GC var gcAlloc, gcEval []string for raw := iter.Next(); raw != nil; raw = iter.Next() { eval := raw.(*structs.Evaluation) - gcThreshold := oldThreshold + gcCutoffTime := cutoffTime if eval.Type == structs.JobTypeBatch { - gcThreshold = batchOldThreshold + gcCutoffTime = batchCutoffTime } - gc, allocs, err := c.gcEval(eval, gcThreshold, false) + gc, allocs, err := c.gcEval(eval, gcCutoffTime, false) if err != nil { return err } @@ -288,15 +317,16 @@ func (c *CoreScheduler) evalGC(eval *structs.Evaluation) error { return c.evalReap(gcEval, gcAlloc) } -// gcEval returns whether the eval should be garbage collected given a raft -// threshold index. The eval disqualifies for garbage collection if it or its -// allocs are not older than the threshold. If the eval should be garbage -// collected, the associated alloc ids that should also be removed are also -// returned -func (c *CoreScheduler) gcEval(eval *structs.Evaluation, thresholdIndex uint64, allowBatch bool) ( +// gcEval returns whether the eval should be garbage collected given the cutoff +// time. The eval disqualifies for garbage collection if it or its allocs are not +// older than the cutoff. If the eval should be garbage collected, the associated +// alloc ids that should also be removed are also returned +func (c *CoreScheduler) gcEval(eval *structs.Evaluation, cutoffTime time.Time, allowBatch bool) ( bool, []string, error) { + // Ignore non-terminal and new evaluations - if !eval.TerminalStatus() || eval.ModifyIndex > thresholdIndex { + mt := time.Unix(0, eval.ModifyTime).UTC() + if !eval.TerminalStatus() || mt.After(cutoffTime) { return false, nil, nil } @@ -335,7 +365,7 @@ func (c *CoreScheduler) gcEval(eval *structs.Evaluation, thresholdIndex uint64, // If we cannot collect outright, check if a partial GC may occur collect := job == nil || job.Status == structs.JobStatusDead && (job.Stop || allowBatch) if !collect { - oldAllocs := olderVersionTerminalAllocs(allocs, job, thresholdIndex) + oldAllocs := olderVersionTerminalAllocs(allocs, job, cutoffTime) gcEval := (len(oldAllocs) == len(allocs)) return gcEval, oldAllocs, nil } @@ -345,7 +375,7 @@ func (c *CoreScheduler) gcEval(eval *structs.Evaluation, thresholdIndex uint64, gcEval := true var gcAllocIDs []string for _, alloc := range allocs { - if !allocGCEligible(alloc, job, time.Now(), thresholdIndex) { + if !allocGCEligible(alloc, job, time.Now(), cutoffTime) { // Can't GC the evaluation since not all of the allocations are // terminal gcEval = false @@ -358,12 +388,13 @@ func (c *CoreScheduler) gcEval(eval *structs.Evaluation, thresholdIndex uint64, return gcEval, gcAllocIDs, nil } -// olderVersionTerminalAllocs returns a list of terminal allocations that belong to the evaluation and may be -// GCed. -func olderVersionTerminalAllocs(allocs []*structs.Allocation, job *structs.Job, thresholdIndex uint64) []string { +// olderVersionTerminalAllocs returns a list of terminal allocations that belong +// to the evaluation and may be GCed. +func olderVersionTerminalAllocs(allocs []*structs.Allocation, job *structs.Job, cutoffTime time.Time) []string { var ret []string for _, alloc := range allocs { - if alloc.CreateIndex < job.JobModifyIndex && alloc.ModifyIndex < thresholdIndex && alloc.TerminalStatus() { + mi := time.Unix(0, alloc.ModifyTime) + if alloc.CreateIndex < job.JobModifyIndex && mi.Before(cutoffTime) && alloc.TerminalStatus() { ret = append(ret, alloc.ID) } } @@ -439,8 +470,14 @@ func (c *CoreScheduler) nodeGC(eval *structs.Evaluation) error { return err } - oldThreshold := c.getThreshold(eval, "node", - "node_gc_threshold", c.srv.config.NodeGCThreshold) + var threshold time.Duration + threshold = c.srv.config.NodeGCThreshold + + // custom threshold override + if c.customNodeGCThreshold != 0 { + threshold = c.customNodeGCThreshold + } + cutoffTime := c.getCutoffTime(threshold) // Collect the nodes to GC var gcNode []string @@ -453,7 +490,8 @@ OUTER: node := raw.(*structs.Node) // Ignore non-terminal and new nodes - if !node.TerminalStatus() || node.ModifyIndex > oldThreshold { + st := time.Unix(node.StatusUpdatedAt, 0) + if !node.TerminalStatus() || st.After(cutoffTime) { continue } @@ -528,7 +566,7 @@ func (c *CoreScheduler) nodeReap(eval *structs.Evaluation, nodeIDs []string) err } // deploymentGC is used to garbage collect old deployments -func (c *CoreScheduler) deploymentGC(eval *structs.Evaluation) error { +func (c *CoreScheduler) deploymentGC() error { // Iterate over the deployments ws := memdb.NewWatchSet() iter, err := c.snap.Deployments(ws, state.SortDefault) @@ -536,8 +574,14 @@ func (c *CoreScheduler) deploymentGC(eval *structs.Evaluation) error { return err } - oldThreshold := c.getThreshold(eval, "deployment", - "deployment_gc_threshold", c.srv.config.DeploymentGCThreshold) + var threshold time.Duration + threshold = c.srv.config.DeploymentGCThreshold + + // custom threshold override + if c.customDeploymentGCThreshold != 0 { + threshold = c.customDeploymentGCThreshold + } + cutoffTime := c.getCutoffTime(threshold) // Collect the deployments to GC var gcDeployment []string @@ -551,7 +595,8 @@ OUTER: deploy := raw.(*structs.Deployment) // Ignore non-terminal and new deployments - if deploy.Active() || deploy.ModifyIndex > oldThreshold { + mt := time.Unix(0, deploy.ModifyTime) + if deploy.Active() || mt.After(cutoffTime) { continue } @@ -628,9 +673,10 @@ func (c *CoreScheduler) partitionDeploymentReap(deployments []string, batchSize // allocGCEligible returns if the allocation is eligible to be garbage collected // according to its terminal status and its reschedule trackers -func allocGCEligible(a *structs.Allocation, job *structs.Job, gcTime time.Time, thresholdIndex uint64) bool { +func allocGCEligible(a *structs.Allocation, job *structs.Job, gcTime, cutoffTime time.Time) bool { // Not in a terminal status and old enough - if !a.TerminalStatus() || a.ModifyIndex > thresholdIndex { + mt := time.Unix(0, a.ModifyTime) + if !a.TerminalStatus() || mt.After(cutoffTime) { return false } @@ -728,14 +774,21 @@ func (c *CoreScheduler) csiVolumeClaimGC(eval *structs.Evaluation) error { return err } - oldThreshold := c.getThreshold(eval, "CSI volume claim", - "csi_volume_claim_gc_threshold", c.srv.config.CSIVolumeClaimGCThreshold) + var threshold time.Duration + threshold = c.srv.config.CSIVolumeClaimGCThreshold + + // custom threshold override + if c.customCSIVolumeClaimGCThreshold != 0 { + threshold = c.customCSIVolumeClaimGCThreshold + } + cutoffTime := c.getCutoffTime(threshold) for i := iter.Next(); i != nil; i = iter.Next() { vol := i.(*structs.CSIVolume) // Ignore new volumes - if vol.CreateIndex > oldThreshold { + mt := time.Unix(0, vol.ModifyTime) + if mt.After(cutoffTime) { continue } @@ -768,14 +821,21 @@ func (c *CoreScheduler) csiPluginGC(eval *structs.Evaluation) error { return err } - oldThreshold := c.getThreshold(eval, "CSI plugin", - "csi_plugin_gc_threshold", c.srv.config.CSIPluginGCThreshold) + var threshold time.Duration + threshold = c.srv.config.CSIPluginGCThreshold + + // custom threshold override + if c.customCSIPluginGCThreshold != 0 { + threshold = c.customCSIPluginGCThreshold + } + cutoffTime := c.getCutoffTime(threshold) for i := iter.Next(); i != nil; i = iter.Next() { plugin := i.(*structs.CSIPlugin) // Ignore new plugins - if plugin.CreateIndex > oldThreshold { + mt := time.Unix(0, plugin.ModifyTime) + if mt.After(cutoffTime) { continue } @@ -829,15 +889,14 @@ func (c *CoreScheduler) expiredACLTokenGC(eval *structs.Evaluation, global bool) return nil } - // The object name is logged within the getThreshold function, therefore we - // want to be clear what token type this trigger is for. - tokenScope := "local" - if global { - tokenScope = "global" - } + var threshold time.Duration + threshold = c.srv.config.ACLTokenExpirationGCThreshold - expiryThresholdIdx := c.getThreshold(eval, tokenScope+" expired ACL tokens", - "acl_token_expiration_gc_threshold", c.srv.config.ACLTokenExpirationGCThreshold) + // custom threshold override + if c.customACLTokenExpirationGCThreshold != 0 { + threshold = c.customACLTokenExpirationGCThreshold + } + cutoffTime := c.getCutoffTime(threshold) expiredIter, err := c.snap.ACLTokensByExpired(global) if err != nil { @@ -868,7 +927,7 @@ func (c *CoreScheduler) expiredACLTokenGC(eval *structs.Evaluation, global bool) // Check if the token is recent enough to skip, otherwise we'll delete // it. - if token.CreateIndex > expiryThresholdIdx { + if token.CreateTime.After(cutoffTime) { continue } @@ -944,11 +1003,19 @@ func (c *CoreScheduler) rootKeyGC(eval *structs.Evaluation, now time.Time) error return err } + var threshold time.Duration + threshold = c.srv.config.RootKeyGCThreshold + + // custom threshold override + if c.customRootKeyGCThreshold != 0 { + threshold = c.customRootKeyGCThreshold + } + // the threshold is longer than we can support with the time table, and we // never want to force-GC keys because that will orphan signed Workload // Identities rotationThreshold := now.Add(-1 * - (c.srv.config.RootKeyRotationThreshold + c.srv.config.RootKeyGCThreshold)) + (c.srv.config.RootKeyRotationThreshold + threshold)) for { raw := iter.Next() @@ -1289,26 +1356,7 @@ func (c *CoreScheduler) rotateVariables(iter memdb.ResultIterator, eval *structs return nil } -// getThreshold returns the index threshold for determining whether an -// object is old enough to GC -func (c *CoreScheduler) getThreshold(eval *structs.Evaluation, objectName, configName string, configThreshold time.Duration) uint64 { - var oldThreshold uint64 - if eval.JobID == structs.CoreJobForceGC { - // The GC was forced, so set the threshold to its maximum so - // everything will GC. - oldThreshold = math.MaxUint64 - c.logger.Debug(fmt.Sprintf("forced %s GC", objectName)) - } else { - // Compute the old threshold limit for GC using the FSM - // time table. This is a rough mapping of a time to the - // Raft index it belongs to. - tt := c.srv.fsm.TimeTable() - cutoff := time.Now().UTC().Add(-1 * configThreshold) - oldThreshold = tt.NearestIndex(cutoff) - c.logger.Debug( - fmt.Sprintf("%s GC scanning before cutoff index", objectName), - "index", oldThreshold, - configName, configThreshold) - } - return oldThreshold +// getCutoffTime returns a time.Time of the latest object that should be GCd +func (c *CoreScheduler) getCutoffTime(configThreshold time.Duration) time.Time { + return time.Now().UTC().Add(-1 * configThreshold) } diff --git a/nomad/core_sched_test.go b/nomad/core_sched_test.go index 1ad94d3b6c8..edf646305b4 100644 --- a/nomad/core_sched_test.go +++ b/nomad/core_sched_test.go @@ -31,14 +31,13 @@ func TestCoreScheduler_EvalGC(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" eval store := s1.fsm.State() eval := mock.Eval() + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() eval.Status = structs.EvalStatusFailed - store.UpsertJobSummary(999, mock.JobSummary(eval.JobID)) + must.NoError(t, store.UpsertJobSummary(999, mock.JobSummary(eval.JobID))) must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval})) // Insert mock job with rescheduling disabled @@ -64,7 +63,8 @@ func TestCoreScheduler_EvalGC(t *testing.T) { alloc2.ClientStatus = structs.AllocClientStatusLost alloc2.JobID = eval.JobID alloc2.TaskGroup = job.TaskGroups[0].Name - must.NoError(t, store.UpsertAllocs(structs.MsgTypeTestSetup, 1001, []*structs.Allocation{alloc, alloc2})) + must.NoError(t, store.UpsertAllocs( + structs.MsgTypeTestSetup, 1001, []*structs.Allocation{alloc, alloc2})) // Insert service for "dead" alloc service := &structs.ServiceRegistration{ @@ -81,10 +81,6 @@ func TestCoreScheduler_EvalGC(t *testing.T) { must.NoError(t, store.UpsertServiceRegistrations( structs.MsgTypeTestSetup, 1002, []*structs.ServiceRegistration{service})) - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold)) - // Create a core scheduler snap, err := store.Snapshot() must.NoError(t, err) @@ -121,30 +117,28 @@ func TestCoreScheduler_EvalGC_ReschedulingAllocs(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" eval store := s1.fsm.State() eval := mock.Eval() + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() eval.Status = structs.EvalStatusFailed - store.UpsertJobSummary(999, mock.JobSummary(eval.JobID)) - err := store.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval}) - require.Nil(t, err) + must.NoError(t, store.UpsertJobSummary(999, mock.JobSummary(eval.JobID))) + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval})) // Insert "pending" eval for same job eval2 := mock.Eval() eval2.JobID = eval.JobID - store.UpsertJobSummary(999, mock.JobSummary(eval2.JobID)) - err = store.UpsertEvals(structs.MsgTypeTestSetup, 1003, []*structs.Evaluation{eval2}) - require.Nil(t, err) + eval2.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval2.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + must.NoError(t, store.UpsertJobSummary(999, mock.JobSummary(eval2.JobID))) + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, 1003, []*structs.Evaluation{eval2})) // Insert mock job with default reschedule policy of 2 in 10 minutes job := mock.Job() job.ID = eval.JobID - err = store.UpsertJob(structs.MsgTypeTestSetup, 1001, nil, job) - require.Nil(t, err) + must.NoError(t, store.UpsertJob(structs.MsgTypeTestSetup, 1001, nil, job)) // Insert failed alloc with an old reschedule attempt, can be GCed alloc := mock.Alloc() @@ -158,7 +152,7 @@ func TestCoreScheduler_EvalGC_ReschedulingAllocs(t *testing.T) { alloc.RescheduleTracker = &structs.RescheduleTracker{ Events: []*structs.RescheduleEvent{ { - RescheduleTime: time.Now().Add(-1 * time.Hour).UTC().UnixNano(), + RescheduleTime: time.Now().Add(-time.Hour).UTC().UnixNano(), PrevNodeID: uuid.Generate(), PrevAllocID: uuid.Generate(), }, @@ -181,39 +175,31 @@ func TestCoreScheduler_EvalGC_ReschedulingAllocs(t *testing.T) { }, }, } - err = store.UpsertAllocs(structs.MsgTypeTestSetup, 1001, []*structs.Allocation{alloc, alloc2}) - require.Nil(t, err) - - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold)) + must.NoError(t, store.UpsertAllocs(structs.MsgTypeTestSetup, 1001, []*structs.Allocation{alloc, alloc2})) // Create a core scheduler snap, err := store.Snapshot() - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, err) core := NewCoreScheduler(s1, snap) // Attempt the GC, job has all terminal allocs and one pending eval gc := s1.coreJobEval(structs.CoreJobEvalGC, 2000) - err = core.Process(gc) - require.Nil(t, err) + must.NoError(t, core.Process(gc)) // Eval should still exist ws := memdb.NewWatchSet() out, err := store.EvalByID(ws, eval.ID) - require.Nil(t, err) - require.NotNil(t, out) - require.Equal(t, eval.ID, out.ID) + must.Nil(t, err) + must.NotNil(t, out) + must.Eq(t, eval.ID, out.ID) outA, err := store.AllocByID(ws, alloc.ID) - require.Nil(t, err) - require.Nil(t, outA) + must.Nil(t, err) + must.Nil(t, outA) outA2, err := store.AllocByID(ws, alloc2.ID) - require.Nil(t, err) - require.Equal(t, alloc2.ID, outA2.ID) + must.Nil(t, err) + must.Eq(t, alloc2.ID, outA2.ID) } @@ -225,24 +211,21 @@ func TestCoreScheduler_EvalGC_StoppedJob_Reschedulable(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" eval store := s1.fsm.State() eval := mock.Eval() eval.Status = structs.EvalStatusFailed - store.UpsertJobSummary(999, mock.JobSummary(eval.JobID)) - err := store.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval}) - require.Nil(t, err) + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + must.NoError(t, store.UpsertJobSummary(999, mock.JobSummary(eval.JobID))) + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval})) // Insert mock stopped job with default reschedule policy of 2 in 10 minutes job := mock.Job() job.ID = eval.JobID job.Stop = true - err = store.UpsertJob(structs.MsgTypeTestSetup, 1001, nil, job) - require.Nil(t, err) + must.NoError(t, store.UpsertJob(structs.MsgTypeTestSetup, 1001, nil, job)) // Insert failed alloc with a recent reschedule attempt alloc := mock.Alloc() @@ -260,12 +243,7 @@ func TestCoreScheduler_EvalGC_StoppedJob_Reschedulable(t *testing.T) { }, }, } - err = store.UpsertAllocs(structs.MsgTypeTestSetup, 1001, []*structs.Allocation{alloc}) - require.Nil(t, err) - - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold)) + must.NoError(t, store.UpsertAllocs(structs.MsgTypeTestSetup, 1001, []*structs.Allocation{alloc})) // Create a core scheduler snap, err := store.Snapshot() @@ -276,20 +254,18 @@ func TestCoreScheduler_EvalGC_StoppedJob_Reschedulable(t *testing.T) { // Attempt the GC gc := s1.coreJobEval(structs.CoreJobEvalGC, 2000) - err = core.Process(gc) - require.Nil(t, err) + must.NoError(t, core.Process(gc)) // Eval should not exist ws := memdb.NewWatchSet() out, err := store.EvalByID(ws, eval.ID) - require.Nil(t, err) - require.Nil(t, out) + must.Nil(t, err) + must.Nil(t, out) // Alloc should not exist outA, err := store.AllocByID(ws, alloc.ID) - require.Nil(t, err) - require.Nil(t, outA) - + must.Nil(t, err) + must.Nil(t, outA) } // An EvalGC should never reap a batch job that has not been stopped @@ -299,15 +275,13 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { s1, cleanupS1 := TestServer(t, func(c *Config) { // Set EvalGCThreshold past BatchEvalThreshold to make sure that only // BatchEvalThreshold affects the results. - c.BatchEvalGCThreshold = time.Hour - c.EvalGCThreshold = 2 * time.Hour + c.BatchEvalGCThreshold = 2 * time.Hour + c.EvalGCThreshold = 4 * time.Hour + c.JobGCThreshold = 2 * time.Hour }) defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 2, 10) - var jobModifyIdx uint64 = 1000 // A "stopped" job containing one "complete" eval with one terminal allocation. @@ -320,15 +294,14 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { Attempts: 0, Interval: 0 * time.Second, } - err := store.UpsertJob(structs.MsgTypeTestSetup, jobModifyIdx+1, nil, stoppedJob) - must.NoError(t, err) + must.NoError(t, store.UpsertJob(structs.MsgTypeTestSetup, jobModifyIdx+1, nil, stoppedJob)) stoppedJobEval := mock.Eval() stoppedJobEval.Status = structs.EvalStatusComplete stoppedJobEval.Type = structs.JobTypeBatch stoppedJobEval.JobID = stoppedJob.ID - err = store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx+2, []*structs.Evaluation{stoppedJobEval}) - must.NoError(t, err) + stoppedJobEval.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() // set to less than initial BatchEvalGCThreshold + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx+2, []*structs.Evaluation{stoppedJobEval})) stoppedJobStoppedAlloc := mock.Alloc() stoppedJobStoppedAlloc.Job = stoppedJob @@ -336,6 +309,7 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { stoppedJobStoppedAlloc.EvalID = stoppedJobEval.ID stoppedJobStoppedAlloc.DesiredStatus = structs.AllocDesiredStatusStop stoppedJobStoppedAlloc.ClientStatus = structs.AllocClientStatusFailed + stoppedJobStoppedAlloc.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() stoppedJobLostAlloc := mock.Alloc() stoppedJobLostAlloc.Job = stoppedJob @@ -343,9 +317,11 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { stoppedJobLostAlloc.EvalID = stoppedJobEval.ID stoppedJobLostAlloc.DesiredStatus = structs.AllocDesiredStatusRun stoppedJobLostAlloc.ClientStatus = structs.AllocClientStatusLost + stoppedJobLostAlloc.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() - err = store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx+3, []*structs.Allocation{stoppedJobStoppedAlloc, stoppedJobLostAlloc}) - must.NoError(t, err) + must.NoError(t, store.UpsertAllocs( + structs.MsgTypeTestSetup, jobModifyIdx+3, + []*structs.Allocation{stoppedJobStoppedAlloc, stoppedJobLostAlloc})) // A "dead" job containing one "complete" eval with: // 1. A "stopped" alloc @@ -354,15 +330,14 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { deadJob := mock.Job() deadJob.Type = structs.JobTypeBatch deadJob.Status = structs.JobStatusDead - err = store.UpsertJob(structs.MsgTypeTestSetup, jobModifyIdx, nil, deadJob) - must.NoError(t, err) + must.NoError(t, store.UpsertJob(structs.MsgTypeTestSetup, jobModifyIdx, nil, deadJob)) deadJobEval := mock.Eval() deadJobEval.Status = structs.EvalStatusComplete deadJobEval.Type = structs.JobTypeBatch deadJobEval.JobID = deadJob.ID - err = store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx+1, []*structs.Evaluation{deadJobEval}) - must.NoError(t, err) + deadJobEval.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() // set to less than initial BatchEvalGCThreshold + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx+1, []*structs.Evaluation{deadJobEval})) stoppedAlloc := mock.Alloc() stoppedAlloc.Job = deadJob @@ -370,6 +345,7 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { stoppedAlloc.EvalID = deadJobEval.ID stoppedAlloc.DesiredStatus = structs.AllocDesiredStatusStop stoppedAlloc.ClientStatus = structs.AllocClientStatusFailed + stoppedAlloc.ModifyTime = time.Now().UnixNano() lostAlloc := mock.Alloc() lostAlloc.Job = deadJob @@ -377,9 +353,9 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { lostAlloc.EvalID = deadJobEval.ID lostAlloc.DesiredStatus = structs.AllocDesiredStatusRun lostAlloc.ClientStatus = structs.AllocClientStatusLost + lostAlloc.ModifyTime = time.Now().UnixNano() - err = store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx+2, []*structs.Allocation{stoppedAlloc, lostAlloc}) - must.NoError(t, err) + must.NoError(t, store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx+2, []*structs.Allocation{stoppedAlloc, lostAlloc})) // An "alive" job #2 containing two complete evals. The first with: // 1. A "lost" alloc @@ -392,15 +368,14 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { activeJob := mock.Job() activeJob.Type = structs.JobTypeBatch activeJob.Status = structs.JobStatusDead - err = store.UpsertJob(structs.MsgTypeTestSetup, jobModifyIdx, nil, activeJob) - must.NoError(t, err) + must.NoError(t, store.UpsertJob(structs.MsgTypeTestSetup, jobModifyIdx, nil, activeJob)) activeJobEval := mock.Eval() activeJobEval.Status = structs.EvalStatusComplete activeJobEval.Type = structs.JobTypeBatch activeJobEval.JobID = activeJob.ID - err = store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx+1, []*structs.Evaluation{activeJobEval}) - must.NoError(t, err) + activeJobEval.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() // set to less than initial BatchEvalGCThreshold + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx+1, []*structs.Evaluation{activeJobEval})) activeJobRunningAlloc := mock.Alloc() activeJobRunningAlloc.Job = activeJob @@ -408,6 +383,7 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { activeJobRunningAlloc.EvalID = activeJobEval.ID activeJobRunningAlloc.DesiredStatus = structs.AllocDesiredStatusRun activeJobRunningAlloc.ClientStatus = structs.AllocClientStatusRunning + activeJobRunningAlloc.ModifyTime = time.Now().UnixNano() activeJobLostAlloc := mock.Alloc() activeJobLostAlloc.Job = activeJob @@ -415,16 +391,17 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { activeJobLostAlloc.EvalID = activeJobEval.ID activeJobLostAlloc.DesiredStatus = structs.AllocDesiredStatusRun activeJobLostAlloc.ClientStatus = structs.AllocClientStatusLost + activeJobLostAlloc.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() - err = store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Allocation{activeJobRunningAlloc, activeJobLostAlloc}) - must.NoError(t, err) + must.NoError(t, store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Allocation{activeJobRunningAlloc, activeJobLostAlloc})) activeJobCompleteEval := mock.Eval() activeJobCompleteEval.Status = structs.EvalStatusComplete activeJobCompleteEval.Type = structs.JobTypeBatch activeJobCompleteEval.JobID = activeJob.ID - err = store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Evaluation{activeJobCompleteEval}) - must.NoError(t, err) + activeJobCompleteEval.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() // set to less than initial BatchEvalGCThreshold + + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Evaluation{activeJobCompleteEval})) activeJobCompletedEvalCompletedAlloc := mock.Alloc() activeJobCompletedEvalCompletedAlloc.Job = activeJob @@ -432,23 +409,22 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { activeJobCompletedEvalCompletedAlloc.EvalID = activeJobCompleteEval.ID activeJobCompletedEvalCompletedAlloc.DesiredStatus = structs.AllocDesiredStatusStop activeJobCompletedEvalCompletedAlloc.ClientStatus = structs.AllocClientStatusComplete + activeJobCompletedEvalCompletedAlloc.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() - err = store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Allocation{activeJobCompletedEvalCompletedAlloc}) - must.NoError(t, err) + must.NoError(t, store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Allocation{activeJobCompletedEvalCompletedAlloc})) // A job that ran once and was then purged. purgedJob := mock.Job() purgedJob.Type = structs.JobTypeBatch purgedJob.Status = structs.JobStatusDead - err = store.UpsertJob(structs.MsgTypeTestSetup, jobModifyIdx, nil, purgedJob) - must.NoError(t, err) + must.NoError(t, store.UpsertJob(structs.MsgTypeTestSetup, jobModifyIdx, nil, purgedJob)) purgedJobEval := mock.Eval() purgedJobEval.Status = structs.EvalStatusComplete purgedJobEval.Type = structs.JobTypeBatch purgedJobEval.JobID = purgedJob.ID - err = store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx+1, []*structs.Evaluation{purgedJobEval}) - must.NoError(t, err) + purgedJobEval.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() // set to less than initial BatchEvalGCThreshold + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx+1, []*structs.Evaluation{purgedJobEval})) purgedJobCompleteAlloc := mock.Alloc() purgedJobCompleteAlloc.Job = purgedJob @@ -456,20 +432,20 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { purgedJobCompleteAlloc.EvalID = purgedJobEval.ID purgedJobCompleteAlloc.DesiredStatus = structs.AllocDesiredStatusRun purgedJobCompleteAlloc.ClientStatus = structs.AllocClientStatusLost + purgedJobCompleteAlloc.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() - err = store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Allocation{purgedJobCompleteAlloc}) - must.NoError(t, err) + must.NoError(t, store.UpsertAllocs(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Allocation{purgedJobCompleteAlloc})) purgedJobCompleteEval := mock.Eval() purgedJobCompleteEval.Status = structs.EvalStatusComplete purgedJobCompleteEval.Type = structs.JobTypeBatch purgedJobCompleteEval.JobID = purgedJob.ID - err = store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Evaluation{purgedJobCompleteEval}) - must.NoError(t, err) + purgedJobCompleteEval.ModifyTime = time.Now().UTC().Add(-1 * time.Hour).UnixNano() // set to less than initial BatchEvalGCThreshold + + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, jobModifyIdx-1, []*structs.Evaluation{purgedJobCompleteEval})) // Purge job. - err = store.DeleteJob(jobModifyIdx, purgedJob.Namespace, purgedJob.ID) - must.NoError(t, err) + must.NoError(t, store.DeleteJob(jobModifyIdx, purgedJob.Namespace, purgedJob.ID)) // A little helper for assertions assertCorrectJobEvalAlloc := func( @@ -519,12 +495,12 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { } } - // Create a core scheduler + // Create a core scheduler, no time modifications snap, err := store.Snapshot() must.NoError(t, err) core := NewCoreScheduler(s1, snap) - // Attempt the GC without moving the time at all + // Attempt the GC gc := s1.coreJobEval(structs.CoreJobEvalGC, jobModifyIdx) err = core.Process(gc) must.NoError(t, err) @@ -549,43 +525,12 @@ func TestCoreScheduler_EvalGC_Batch(t *testing.T) { []*structs.Allocation{}, ) - // Update the time tables by half of the BatchEvalGCThreshold which is too - // small to GC anything. - tt := s1.fsm.TimeTable() - tt.Witness(2*jobModifyIdx, time.Now().UTC().Add((-1)*s1.config.BatchEvalGCThreshold/2)) - - gc = s1.coreJobEval(structs.CoreJobEvalGC, jobModifyIdx*2) - err = core.Process(gc) - must.NoError(t, err) - - // Nothing is gone. - assertCorrectJobEvalAlloc( - memdb.NewWatchSet(), - []*structs.Job{deadJob, activeJob, stoppedJob}, - []*structs.Job{}, - []*structs.Evaluation{ - deadJobEval, - activeJobEval, activeJobCompleteEval, - stoppedJobEval, - purgedJobEval, - }, - []*structs.Evaluation{}, - []*structs.Allocation{ - stoppedAlloc, lostAlloc, - activeJobRunningAlloc, activeJobLostAlloc, activeJobCompletedEvalCompletedAlloc, - stoppedJobStoppedAlloc, stoppedJobLostAlloc, - }, - []*structs.Allocation{}, - ) - - // Update the time tables so that BatchEvalGCThreshold has elapsed. - s1.fsm.timetable.table = make([]TimeTableEntry, 2, 10) - tt = s1.fsm.TimeTable() - tt.Witness(2*jobModifyIdx, time.Now().UTC().Add(-1*s1.config.BatchEvalGCThreshold)) - + // set a shorter GC threshold this time gc = s1.coreJobEval(structs.CoreJobEvalGC, jobModifyIdx*2) - err = core.Process(gc) - must.NoError(t, err) + core.(*CoreScheduler).customBatchEvalGCThreshold = time.Minute + //core.(*CoreScheduler).customEvalGCThreshold = time.Minute + //core.(*CoreScheduler).customJobGCThreshold = time.Minute + must.NoError(t, core.Process(gc)) // We expect the following: // @@ -629,6 +574,9 @@ func TestCoreScheduler_EvalGC_JobVersionTag(t *testing.T) { eval := mock.Eval() eval.JobID = job.ID eval.Status = structs.EvalStatusComplete + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, 999, []*structs.Evaluation{eval})) // upsert a couple versions of the job, so the "jobs" table has one // and the "job_version" table has two. @@ -705,13 +653,13 @@ func TestCoreScheduler_EvalGC_Partial(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" eval store := s1.fsm.State() eval := mock.Eval() eval.Status = structs.EvalStatusComplete + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + store.UpsertJobSummary(999, mock.JobSummary(eval.JobID)) err := store.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval}) if err != nil { @@ -761,10 +709,6 @@ func TestCoreScheduler_EvalGC_Partial(t *testing.T) { err = store.UpsertJob(structs.MsgTypeTestSetup, 1001, nil, job) require.Nil(t, err) - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold)) - // Create a core scheduler snap, err := store.Snapshot() if err != nil { @@ -829,13 +773,13 @@ func TestCoreScheduler_EvalGC_Force(t *testing.T) { defer cleanup() testutil.WaitForLeader(t, server.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - server.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" eval store := server.fsm.State() eval := mock.Eval() eval.Status = structs.EvalStatusFailed + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + store.UpsertJobSummary(999, mock.JobSummary(eval.JobID)) err := store.UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval}) if err != nil { @@ -912,9 +856,6 @@ func TestCoreScheduler_NodeGC(t *testing.T) { defer cleanup() testutil.WaitForLeader(t, server.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - server.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" node store := server.fsm.State() node := mock.Node() @@ -924,10 +865,6 @@ func TestCoreScheduler_NodeGC(t *testing.T) { t.Fatalf("err: %v", err) } - // Update the time tables to make this work - tt := server.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*server.config.NodeGCThreshold)) - // Create a core scheduler snap, err := store.Snapshot() if err != nil { @@ -962,9 +899,6 @@ func TestCoreScheduler_NodeGC_TerminalAllocs(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" node store := s1.fsm.State() node := mock.Node() @@ -982,10 +916,6 @@ func TestCoreScheduler_NodeGC_TerminalAllocs(t *testing.T) { t.Fatalf("err: %v", err) } - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.NodeGCThreshold)) - // Create a core scheduler snap, err := store.Snapshot() if err != nil { @@ -1018,9 +948,6 @@ func TestCoreScheduler_NodeGC_RunningAllocs(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" node store := s1.fsm.State() node := mock.Node() @@ -1040,10 +967,6 @@ func TestCoreScheduler_NodeGC_RunningAllocs(t *testing.T) { t.Fatalf("err: %v", err) } - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.NodeGCThreshold)) - // Create a core scheduler snap, err := store.Snapshot() if err != nil { @@ -1076,9 +999,6 @@ func TestCoreScheduler_NodeGC_Force(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert "dead" node store := s1.fsm.State() node := mock.Node() @@ -1120,121 +1040,76 @@ func TestCoreScheduler_JobGC_OutstandingEvals(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert job. store := s1.fsm.State() job := mock.Job() job.Type = structs.JobTypeBatch job.Status = structs.JobStatusDead - err := store.UpsertJob(structs.MsgTypeTestSetup, 1000, nil, job) - if err != nil { - t.Fatalf("err: %v", err) - } + job.SubmitTime = time.Now().Add(-6 * time.Hour).UnixNano() + must.NoError(t, store.UpsertJob(structs.MsgTypeTestSetup, 1000, nil, job)) // Insert two evals, one terminal and one not eval := mock.Eval() eval.JobID = job.ID eval.Status = structs.EvalStatusComplete + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() eval2 := mock.Eval() eval2.JobID = job.ID eval2.Status = structs.EvalStatusPending - err = store.UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval, eval2}) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold)) + eval2.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval2.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval, eval2})) // Create a core scheduler snap, err := store.Snapshot() - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, err) core := NewCoreScheduler(s1, snap) // Attempt the GC gc := s1.coreJobEval(structs.CoreJobJobGC, 2000) - err = core.Process(gc) - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, core.Process(gc)) // Should still exist ws := memdb.NewWatchSet() out, err := store.JobByID(ws, job.Namespace, job.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if out == nil { - t.Fatalf("bad: %v", out) - } + must.NoError(t, err) + must.NotNil(t, out) outE, err := store.EvalByID(ws, eval.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if outE == nil { - t.Fatalf("bad: %v", outE) - } + must.NoError(t, err) + must.NotNil(t, outE) outE2, err := store.EvalByID(ws, eval2.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if outE2 == nil { - t.Fatalf("bad: %v", outE2) - } + must.NoError(t, err) + must.NotNil(t, outE2) // Update the second eval to be terminal eval2.Status = structs.EvalStatusComplete - err = store.UpsertEvals(structs.MsgTypeTestSetup, 1003, []*structs.Evaluation{eval2}) - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, store.UpsertEvals(structs.MsgTypeTestSetup, 1003, []*structs.Evaluation{eval2})) // Create a core scheduler snap, err = store.Snapshot() - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, err) core = NewCoreScheduler(s1, snap) // Attempt the GC gc = s1.coreJobEval(structs.CoreJobJobGC, 2000) - err = core.Process(gc) - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, core.Process(gc)) // Should not still exist out, err = store.JobByID(ws, job.Namespace, job.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if out != nil { - t.Fatalf("bad: %v", out) - } + must.NoError(t, err) + must.Nil(t, out) outE, err = store.EvalByID(ws, eval.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if outE != nil { - t.Fatalf("bad: %v", outE) - } + must.NoError(t, err) + must.Nil(t, outE) outE2, err = store.EvalByID(ws, eval2.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if outE2 != nil { - t.Fatalf("bad: %v", outE2) - } + must.NoError(t, err) + must.Nil(t, outE2) } func TestCoreScheduler_JobGC_OutstandingAllocs(t *testing.T) { @@ -1244,9 +1119,6 @@ func TestCoreScheduler_JobGC_OutstandingAllocs(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert job. store := s1.fsm.State() job := mock.Job() @@ -1265,6 +1137,8 @@ func TestCoreScheduler_JobGC_OutstandingAllocs(t *testing.T) { eval := mock.Eval() eval.JobID = job.ID eval.Status = structs.EvalStatusComplete + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() err = store.UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval}) if err != nil { t.Fatalf("err: %v", err) @@ -1290,10 +1164,6 @@ func TestCoreScheduler_JobGC_OutstandingAllocs(t *testing.T) { t.Fatalf("err: %v", err) } - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold)) - // Create a core scheduler snap, err := store.Snapshot() if err != nil { @@ -1390,9 +1260,6 @@ func TestCoreScheduler_JobGC_OneShot(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert job. store := s1.fsm.State() job := mock.Job() @@ -1435,10 +1302,6 @@ func TestCoreScheduler_JobGC_OneShot(t *testing.T) { // Force the jobs state to dead job.Status = structs.JobStatusDead - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold)) - // Create a core scheduler snap, err := store.Snapshot() if err != nil { @@ -1503,9 +1366,6 @@ func TestCoreScheduler_JobGC_Stopped(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert job. store := s1.fsm.State() job := mock.Job() @@ -1523,10 +1383,14 @@ func TestCoreScheduler_JobGC_Stopped(t *testing.T) { eval := mock.Eval() eval.JobID = job.ID eval.Status = structs.EvalStatusComplete + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() eval2 := mock.Eval() eval2.JobID = job.ID eval2.Status = structs.EvalStatusComplete + eval2.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval2.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() err = store.UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval, eval2}) if err != nil { @@ -1539,15 +1403,13 @@ func TestCoreScheduler_JobGC_Stopped(t *testing.T) { alloc.EvalID = eval.ID alloc.DesiredStatus = structs.AllocDesiredStatusStop alloc.TaskGroup = job.TaskGroups[0].Name + alloc.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() + alloc.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() err = store.UpsertAllocs(structs.MsgTypeTestSetup, 1002, []*structs.Allocation{alloc}) if err != nil { t.Fatalf("err: %v", err) } - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold)) - // Create a core scheduler snap, err := store.Snapshot() if err != nil { @@ -1611,9 +1473,6 @@ func TestCoreScheduler_JobGC_Force(t *testing.T) { defer cleanup() testutil.WaitForLeader(t, server.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - server.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert job. store := server.fsm.State() job := mock.Job() @@ -1628,6 +1487,9 @@ func TestCoreScheduler_JobGC_Force(t *testing.T) { eval := mock.Eval() eval.JobID = job.ID eval.Status = structs.EvalStatusComplete + eval.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + err = store.UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval}) if err != nil { t.Fatalf("err: %v", err) @@ -1676,9 +1538,6 @@ func TestCoreScheduler_JobGC_Parameterized(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert a parameterized job. store := s1.fsm.State() job := mock.Job() @@ -1756,9 +1615,6 @@ func TestCoreScheduler_JobGC_Periodic(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert a parameterized job. store := s1.fsm.State() job := mock.PeriodicJob() @@ -1841,16 +1697,22 @@ func TestCoreScheduler_jobGC(t *testing.T) { mockEval1.JobID = inputJob.ID mockEval1.Namespace = inputJob.Namespace mockEval1.Status = structs.EvalStatusComplete + mockEval1.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() // make sure objects we insert are older than GC thresholds + mockEval1.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() mockJob1Alloc1 := mock.Alloc() mockJob1Alloc1.EvalID = mockEval1.ID mockJob1Alloc1.JobID = inputJob.ID mockJob1Alloc1.ClientStatus = structs.AllocClientStatusRunning + mockJob1Alloc1.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() + mockJob1Alloc1.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() mockJob1Alloc2 := mock.Alloc() mockJob1Alloc2.EvalID = mockEval1.ID mockJob1Alloc2.JobID = inputJob.ID mockJob1Alloc2.ClientStatus = structs.AllocClientStatusRunning + mockJob1Alloc2.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() + mockJob1Alloc2.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() must.NoError(t, testServer.fsm.State().UpsertJob(structs.MsgTypeTestSetup, 10, nil, inputJob)) @@ -1955,49 +1817,47 @@ func TestCoreScheduler_DeploymentGC(t *testing.T) { s1, cleanupS1 := TestServer(t, nil) defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - assert := assert.New(t) - - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) // Insert an active, terminal, and terminal with allocations deployment store := s1.fsm.State() d1, d2, d3 := mock.Deployment(), mock.Deployment(), mock.Deployment() d1.Status = structs.DeploymentStatusFailed + d1.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() + d1.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() d3.Status = structs.DeploymentStatusSuccessful - assert.Nil(store.UpsertDeployment(1000, d1), "UpsertDeployment") - assert.Nil(store.UpsertDeployment(1001, d2), "UpsertDeployment") - assert.Nil(store.UpsertDeployment(1002, d3), "UpsertDeployment") + + must.Nil(t, store.UpsertDeployment(1000, d1), must.Sprint("UpsertDeployment")) + must.Nil(t, store.UpsertDeployment(1001, d2), must.Sprint("UpsertDeployment")) + must.Nil(t, store.UpsertDeployment(1002, d3), must.Sprint("UpsertDeployment")) a := mock.Alloc() a.JobID = d3.JobID a.DeploymentID = d3.ID - assert.Nil(store.UpsertAllocs(structs.MsgTypeTestSetup, 1003, []*structs.Allocation{a}), "UpsertAllocs") - - // Update the time tables to make this work - tt := s1.fsm.TimeTable() - tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.DeploymentGCThreshold)) + a.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() + a.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + must.Nil(t, store.UpsertAllocs(structs.MsgTypeTestSetup, 1003, []*structs.Allocation{a})) // Create a core scheduler snap, err := store.Snapshot() - assert.Nil(err, "Snapshot") + must.NoError(t, err) core := NewCoreScheduler(s1, snap) // Attempt the GC gc := s1.coreJobEval(structs.CoreJobDeploymentGC, 2000) - assert.Nil(core.Process(gc), "Process GC") + must.NoError(t, core.Process(gc)) // Should be gone ws := memdb.NewWatchSet() out, err := store.DeploymentByID(ws, d1.ID) - assert.Nil(err, "DeploymentByID") - assert.Nil(out, "Terminal Deployment") + must.NoError(t, err) + must.Nil(t, out) + out2, err := store.DeploymentByID(ws, d2.ID) - assert.Nil(err, "DeploymentByID") - assert.NotNil(out2, "Active Deployment") + must.NoError(t, err) + must.NotNil(t, out2) out3, err := store.DeploymentByID(ws, d3.ID) - assert.Nil(err, "DeploymentByID") - assert.NotNil(out3, "Terminal Deployment With Allocs") + must.NoError(t, err) + must.NotNil(t, out3) } func TestCoreScheduler_DeploymentGC_Force(t *testing.T) { @@ -2015,13 +1875,12 @@ func TestCoreScheduler_DeploymentGC_Force(t *testing.T) { testutil.WaitForLeader(t, server.RPC) assert := assert.New(t) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - server.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Insert terminal and active deployment store := server.fsm.State() d1, d2 := mock.Deployment(), mock.Deployment() d1.Status = structs.DeploymentStatusFailed + d1.CreateTime = time.Now().Add(-6 * time.Hour).UnixNano() + d1.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() assert.Nil(store.UpsertDeployment(1000, d1), "UpsertDeployment") assert.Nil(store.UpsertDeployment(1001, d2), "UpsertDeployment") @@ -2053,9 +1912,6 @@ func TestCoreScheduler_PartitionEvalReap(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Create a core scheduler snap, err := s1.fsm.State().Snapshot() if err != nil { @@ -2095,9 +1951,6 @@ func TestCoreScheduler_PartitionDeploymentReap(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0 - s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - // Create a core scheduler snap, err := s1.fsm.State().Snapshot() if err != nil { @@ -2160,63 +2013,63 @@ func TestAllocation_GCEligible(t *testing.T) { PreventRescheduleOnLost *bool AllocJobModifyIndex uint64 JobModifyIndex uint64 - ModifyIndex uint64 + ModifyTime int64 NextAllocID string ReschedulePolicy *structs.ReschedulePolicy RescheduleTrackers []*structs.RescheduleEvent - ThresholdIndex uint64 + CutoffTime time.Time ShouldGC bool } - fail := time.Now() + now := time.Now() harness := []testCase{ { - Desc: "Don't GC when non terminal", - ClientStatus: structs.AllocClientStatusPending, - DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, - ModifyIndex: 90, - ThresholdIndex: 90, - ShouldGC: false, + Desc: "Don't GC when non terminal", + ClientStatus: structs.AllocClientStatusPending, + DesiredStatus: structs.AllocDesiredStatusRun, + GCTime: now, + ModifyTime: now.UnixNano(), + CutoffTime: now, + ShouldGC: false, }, { - Desc: "Don't GC when non terminal and job stopped", - ClientStatus: structs.AllocClientStatusPending, - DesiredStatus: structs.AllocDesiredStatusRun, - JobStop: true, - GCTime: fail, - ModifyIndex: 90, - ThresholdIndex: 90, - ShouldGC: false, + Desc: "Don't GC when non terminal and job stopped", + ClientStatus: structs.AllocClientStatusPending, + DesiredStatus: structs.AllocDesiredStatusRun, + JobStop: true, + GCTime: now, + ModifyTime: now.UnixNano(), + CutoffTime: now, + ShouldGC: false, }, { - Desc: "Don't GC when non terminal and job dead", - ClientStatus: structs.AllocClientStatusPending, - DesiredStatus: structs.AllocDesiredStatusRun, - JobStatus: structs.JobStatusDead, - GCTime: fail, - ModifyIndex: 90, - ThresholdIndex: 90, - ShouldGC: false, + Desc: "Don't GC when non terminal and job dead", + ClientStatus: structs.AllocClientStatusPending, + DesiredStatus: structs.AllocDesiredStatusRun, + JobStatus: structs.JobStatusDead, + GCTime: now, + ModifyTime: now.UnixNano(), + CutoffTime: now, + ShouldGC: false, }, { - Desc: "Don't GC when non terminal on client and job dead", - ClientStatus: structs.AllocClientStatusRunning, - DesiredStatus: structs.AllocDesiredStatusStop, - JobStatus: structs.JobStatusDead, - GCTime: fail, - ModifyIndex: 90, - ThresholdIndex: 90, - ShouldGC: false, + Desc: "Don't GC when non terminal on client and job dead", + ClientStatus: structs.AllocClientStatusRunning, + DesiredStatus: structs.AllocDesiredStatusStop, + JobStatus: structs.JobStatusDead, + GCTime: now, + ModifyTime: now.UnixNano(), + CutoffTime: now, + ShouldGC: false, }, { Desc: "GC when terminal but not failed ", ClientStatus: structs.AllocClientStatusComplete, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, - ModifyIndex: 90, - ThresholdIndex: 90, + GCTime: now, + ModifyTime: now.UnixNano(), + CutoffTime: now, ReschedulePolicy: nil, ShouldGC: true, }, @@ -2224,9 +2077,9 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "Don't GC when threshold not met", ClientStatus: structs.AllocClientStatusComplete, DesiredStatus: structs.AllocDesiredStatusStop, - GCTime: fail, - ModifyIndex: 100, - ThresholdIndex: 90, + GCTime: now, + ModifyTime: now.UnixNano(), + CutoffTime: now.Add(-1 * time.Hour), ReschedulePolicy: nil, ShouldGC: false, }, @@ -2234,29 +2087,29 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC when no reschedule policy", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, + GCTime: now, ReschedulePolicy: nil, - ModifyIndex: 90, - ThresholdIndex: 90, + ModifyTime: now.UnixNano(), + CutoffTime: now, ShouldGC: true, }, { Desc: "GC when empty policy", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, + GCTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Attempts: 0, Interval: 0 * time.Minute}, - ModifyIndex: 90, - ThresholdIndex: 90, + ModifyTime: now.UnixNano(), + CutoffTime: now, ShouldGC: true, }, { Desc: "Don't GC when no previous reschedule attempts", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, - ModifyIndex: 90, - ThresholdIndex: 90, + GCTime: now, + ModifyTime: now.UnixNano(), + CutoffTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Attempts: 1, Interval: 1 * time.Minute}, ShouldGC: false, }, @@ -2265,12 +2118,12 @@ func TestAllocation_GCEligible(t *testing.T) { ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, ReschedulePolicy: &structs.ReschedulePolicy{Attempts: 2, Interval: 30 * time.Minute}, - GCTime: fail, - ModifyIndex: 90, - ThresholdIndex: 90, + GCTime: now, + ModifyTime: now.UnixNano(), + CutoffTime: now, RescheduleTrackers: []*structs.RescheduleEvent{ { - RescheduleTime: fail.Add(-5 * time.Minute).UTC().UnixNano(), + RescheduleTime: now.Add(-5 * time.Minute).UTC().UnixNano(), }, }, ShouldGC: false, @@ -2279,14 +2132,15 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC with prev reschedule attempt outside interval", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, + GCTime: now, + CutoffTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Attempts: 5, Interval: 30 * time.Minute}, RescheduleTrackers: []*structs.RescheduleEvent{ { - RescheduleTime: fail.Add(-45 * time.Minute).UTC().UnixNano(), + RescheduleTime: now.Add(-45 * time.Minute).UTC().UnixNano(), }, { - RescheduleTime: fail.Add(-60 * time.Minute).UTC().UnixNano(), + RescheduleTime: now.Add(-60 * time.Minute).UTC().UnixNano(), }, }, ShouldGC: true, @@ -2295,11 +2149,12 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC when next alloc id is set", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, + GCTime: now, + CutoffTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Attempts: 5, Interval: 30 * time.Minute}, RescheduleTrackers: []*structs.RescheduleEvent{ { - RescheduleTime: fail.Add(-3 * time.Minute).UTC().UnixNano(), + RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), }, }, NextAllocID: uuid.Generate(), @@ -2309,11 +2164,12 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "Don't GC when next alloc id is not set and unlimited restarts", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, + GCTime: now, + CutoffTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Unlimited: true, Delay: 5 * time.Second, DelayFunction: "constant"}, RescheduleTrackers: []*structs.RescheduleEvent{ { - RescheduleTime: fail.Add(-3 * time.Minute).UTC().UnixNano(), + RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), }, }, ShouldGC: false, @@ -2322,11 +2178,12 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC when job is stopped", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, + GCTime: now, + CutoffTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Attempts: 5, Interval: 30 * time.Minute}, RescheduleTrackers: []*structs.RescheduleEvent{ { - RescheduleTime: fail.Add(-3 * time.Minute).UTC().UnixNano(), + RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), }, }, JobStop: true, @@ -2336,7 +2193,8 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC when alloc is lost and eligible for reschedule", ClientStatus: structs.AllocClientStatusLost, DesiredStatus: structs.AllocDesiredStatusStop, - GCTime: fail, + GCTime: now, + CutoffTime: now, JobStatus: structs.JobStatusDead, ShouldGC: true, }, @@ -2344,11 +2202,12 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC when job status is dead", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, + GCTime: now, + CutoffTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Attempts: 5, Interval: 30 * time.Minute}, RescheduleTrackers: []*structs.RescheduleEvent{ { - RescheduleTime: fail.Add(-3 * time.Minute).UTC().UnixNano(), + RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), }, }, JobStatus: structs.JobStatusDead, @@ -2358,7 +2217,8 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC when desired status is stop, unlimited reschedule policy, no previous reschedule events", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusStop, - GCTime: fail, + GCTime: now, + CutoffTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Unlimited: true, Delay: 5 * time.Second, DelayFunction: "constant"}, ShouldGC: true, }, @@ -2366,11 +2226,12 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC when desired status is stop, limited reschedule policy, some previous reschedule events", ClientStatus: structs.AllocClientStatusFailed, DesiredStatus: structs.AllocDesiredStatusStop, - GCTime: fail, + GCTime: now, + CutoffTime: now, ReschedulePolicy: &structs.ReschedulePolicy{Attempts: 5, Interval: 30 * time.Minute}, RescheduleTrackers: []*structs.RescheduleEvent{ { - RescheduleTime: fail.Add(-3 * time.Minute).UTC().UnixNano(), + RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), }, }, ShouldGC: true, @@ -2379,7 +2240,8 @@ func TestAllocation_GCEligible(t *testing.T) { Desc: "GC when alloc is unknown and but desired state is running", ClientStatus: structs.AllocClientStatusUnknown, DesiredStatus: structs.AllocDesiredStatusRun, - GCTime: fail, + GCTime: now, + CutoffTime: now, JobStatus: structs.JobStatusRunning, ShouldGC: false, }, @@ -2387,7 +2249,7 @@ func TestAllocation_GCEligible(t *testing.T) { for _, tc := range harness { alloc := &structs.Allocation{} - alloc.ModifyIndex = tc.ModifyIndex + alloc.ModifyTime = tc.ModifyTime alloc.DesiredStatus = tc.DesiredStatus alloc.ClientStatus = tc.ClientStatus alloc.RescheduleTracker = &structs.RescheduleTracker{Events: tc.RescheduleTrackers} @@ -2404,7 +2266,7 @@ func TestAllocation_GCEligible(t *testing.T) { job.Stop = tc.JobStop t.Run(tc.Desc, func(t *testing.T) { - if got := allocGCEligible(alloc, job, tc.GCTime, tc.ThresholdIndex); got != tc.ShouldGC { + if got := allocGCEligible(alloc, job, tc.GCTime, tc.CutoffTime); got != tc.ShouldGC { t.Fatalf("expected %v but got %v", tc.ShouldGC, got) } }) @@ -2414,7 +2276,7 @@ func TestAllocation_GCEligible(t *testing.T) { // Verify nil job alloc := mock.Alloc() alloc.ClientStatus = structs.AllocClientStatusComplete - require.True(t, allocGCEligible(alloc, nil, time.Now(), 1000)) + require.True(t, allocGCEligible(alloc, nil, time.Now(), time.Now())) } func TestCoreScheduler_CSIPluginGC(t *testing.T) { @@ -2424,16 +2286,20 @@ func TestCoreScheduler_CSIPluginGC(t *testing.T) { defer cleanupSRV() testutil.WaitForLeader(t, srv.RPC) - srv.fsm.timetable.table = make([]TimeTableEntry, 1, 10) - deleteNodes := state.CreateTestCSIPlugin(srv.fsm.State(), "foo") defer deleteNodes() store := srv.fsm.State() - // Update the time tables to make this work - tt := srv.fsm.TimeTable() index := uint64(2000) - tt.Witness(index, time.Now().UTC().Add(-1*srv.config.CSIPluginGCThreshold)) + + ws := memdb.NewWatchSet() + plug, err := store.CSIPluginByID(ws, "foo") + must.NotNil(t, plug) + must.NoError(t, err) + // set the creation and modification times on the plugin in the past, otherwise + // they won't meet the GC threshold + plug.CreateTime = time.Now().Add(-10 * time.Hour).UnixNano() + plug.ModifyTime = time.Now().Add(-9 * time.Hour).UnixNano() // Create a core scheduler snap, err := store.Snapshot() @@ -2446,8 +2312,7 @@ func TestCoreScheduler_CSIPluginGC(t *testing.T) { must.NoError(t, core.Process(gc)) // Should not be gone (plugin in use) - ws := memdb.NewWatchSet() - plug, err := store.CSIPluginByID(ws, "foo") + plug, err = store.CSIPluginByID(ws, "foo") must.NotNil(t, plug) must.NoError(t, err) @@ -2455,6 +2320,7 @@ func TestCoreScheduler_CSIPluginGC(t *testing.T) { plug = plug.Copy() plug.Controllers = map[string]*structs.CSIInfo{} plug.Nodes = map[string]*structs.CSIInfo{} + plug.ModifyTime = time.Now().Add(-6 * time.Hour).UnixNano() job := mock.CSIPluginJob(structs.CSIPluginTypeController, plug.ID) index++ @@ -3064,44 +2930,42 @@ func TestCoreScheduler_ExpiredACLTokenGC(t *testing.T) { unexpiredLocal := mock.ACLToken() unexpiredLocal.ExpirationTime = pointer.Of(now.Add(2 * time.Hour)) + // Set creation time in the past for all the tokens, otherwise GC won't trigger + for _, token := range []*structs.ACLToken{expiredGlobal, unexpiredGlobal, expiredLocal, unexpiredLocal} { + token.CreateTime = time.Now().Add(-10 * time.Hour) + } + // Upsert these into state. err := testServer.State().UpsertACLTokens(structs.MsgTypeTestSetup, 10, []*structs.ACLToken{ expiredGlobal, unexpiredGlobal, expiredLocal, unexpiredLocal, }) - require.NoError(t, err) - - // Overwrite the timetable. The existing timetable has an entry due to the - // ACL bootstrapping which makes witnessing a new index at a timestamp in - // the past impossible. - tt := NewTimeTable(timeTableGranularity, timeTableLimit) - tt.Witness(20, time.Now().UTC().Add(-1*testServer.config.ACLTokenExpirationGCThreshold)) - testServer.fsm.timetable = tt + must.NoError(t, err) // Generate the core scheduler. snap, err := testServer.State().Snapshot() - require.NoError(t, err) + must.NoError(t, err) coreScheduler := NewCoreScheduler(testServer, snap) // Trigger global and local periodic garbage collection runs. index, err := testServer.State().LatestIndex() - require.NoError(t, err) + must.NoError(t, err) index++ globalGCEval := testServer.coreJobEval(structs.CoreJobGlobalTokenExpiredGC, index) - require.NoError(t, coreScheduler.Process(globalGCEval)) + must.NoError(t, coreScheduler.Process(globalGCEval)) localGCEval := testServer.coreJobEval(structs.CoreJobLocalTokenExpiredGC, index) - require.NoError(t, coreScheduler.Process(localGCEval)) + must.NoError(t, coreScheduler.Process(localGCEval)) // Ensure the ACL tokens stored within state are as expected. iter, err := testServer.State().ACLTokens(nil, state.SortDefault) - require.NoError(t, err) + must.NoError(t, err) var tokens []*structs.ACLToken for raw := iter.Next(); raw != nil; raw = iter.Next() { tokens = append(tokens, raw.(*structs.ACLToken)) } - require.ElementsMatch(t, []*structs.ACLToken{rootACLToken, unexpiredGlobal, unexpiredLocal}, tokens) + must.SliceContainsAll(t, []*structs.ACLToken{rootACLToken, unexpiredGlobal, unexpiredLocal}, tokens) } func TestCoreScheduler_ExpiredACLTokenGC_Force(t *testing.T) { @@ -3130,6 +2994,7 @@ func TestCoreScheduler_ExpiredACLTokenGC_Force(t *testing.T) { for i := 0; i < 20; i++ { mockedToken := mock.ACLToken() mockedToken.Global = true + mockedToken.CreateTime = time.Now().Add(-10 * time.Hour) if i%2 == 0 { expiredGlobalTokens = append(expiredGlobalTokens, mockedToken) mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(-24 * time.Hour)) @@ -3144,6 +3009,7 @@ func TestCoreScheduler_ExpiredACLTokenGC_Force(t *testing.T) { for i := 0; i < 20; i++ { mockedToken := mock.ACLToken() mockedToken.Global = false + mockedToken.CreateTime = time.Now().Add(-10 * time.Hour) if i%2 == 0 { expiredLocalTokens = append(expiredLocalTokens, mockedToken) mockedToken.ExpirationTime = pointer.Of(expiryTimeThreshold.Add(-24 * time.Hour)) @@ -3158,8 +3024,7 @@ func TestCoreScheduler_ExpiredACLTokenGC_Force(t *testing.T) { allTokens = append(allTokens, nonExpiredLocalTokens...) // Upsert them all. - err := testServer.State().UpsertACLTokens(structs.MsgTypeTestSetup, 10, allTokens) - require.NoError(t, err) + must.NoError(t, testServer.State().UpsertACLTokens(structs.MsgTypeTestSetup, 10, allTokens)) // This function provides an easy way to get all tokens out of the // iterator. @@ -3173,28 +3038,28 @@ func TestCoreScheduler_ExpiredACLTokenGC_Force(t *testing.T) { // Check all the tokens are correctly stored within state. iter, err := testServer.State().ACLTokens(nil, state.SortDefault) - require.NoError(t, err) + must.NoError(t, err) tokens := fromIteratorFunc(iter) - require.ElementsMatch(t, allTokens, tokens) + must.SliceContainsAll(t, allTokens, tokens) // Generate the core scheduler and trigger a forced garbage collection // which should delete all expired tokens. snap, err := testServer.State().Snapshot() - require.NoError(t, err) + must.NoError(t, err) coreScheduler := NewCoreScheduler(testServer, snap) index, err := testServer.State().LatestIndex() - require.NoError(t, err) + must.NoError(t, err) index++ forceGCEval := testServer.coreJobEval(structs.CoreJobForceGC, index) - require.NoError(t, coreScheduler.Process(forceGCEval)) + must.NoError(t, coreScheduler.Process(forceGCEval)) // List all the remaining ACL tokens to be sure they are as expected. iter, err = testServer.State().ACLTokens(nil, state.SortDefault) - require.NoError(t, err) + must.NoError(t, err) tokens = fromIteratorFunc(iter) - require.ElementsMatch(t, append(nonExpiredGlobalTokens, nonExpiredLocalTokens...), tokens) + must.SliceContainsAll(t, append(nonExpiredGlobalTokens, nonExpiredLocalTokens...), tokens) } diff --git a/nomad/csi_endpoint_test.go b/nomad/csi_endpoint_test.go index ecc46bc13b6..77d97c35354 100644 --- a/nomad/csi_endpoint_test.go +++ b/nomad/csi_endpoint_test.go @@ -673,9 +673,11 @@ func TestCSIVolumeEndpoint_Unpublish(t *testing.T) { Mode: structs.CSIVolumeClaimRead, } + now := time.Now().UnixNano() + index++ claim.State = structs.CSIVolumeClaimStateTaken - err = state.CSIVolumeClaim(index, ns, volID, claim) + err = state.CSIVolumeClaim(index, now, ns, volID, claim) must.NoError(t, err) // setup: claim the volume for our other alloc @@ -688,7 +690,7 @@ func TestCSIVolumeEndpoint_Unpublish(t *testing.T) { index++ otherClaim.State = structs.CSIVolumeClaimStateTaken - err = state.CSIVolumeClaim(index, ns, volID, otherClaim) + err = state.CSIVolumeClaim(index, now, ns, volID, otherClaim) must.NoError(t, err) // test: unpublish and check the results diff --git a/nomad/fsm.go b/nomad/fsm.go index 4e8494eac2d..292e8808b85 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -23,14 +23,6 @@ import ( "github.com/hashicorp/raft" ) -const ( - // timeTableGranularity is the granularity of index to time tracking - timeTableGranularity = 5 * time.Minute - - // timeTableLimit is the maximum limit of our tracking - timeTableLimit = 72 * time.Hour -) - // SnapshotType is prefixed to a record in the FSM snapshot // so that we can determine the type for restore type SnapshotType byte @@ -131,7 +123,6 @@ type nomadFSM struct { encrypter *Encrypter logger hclog.Logger state *state.StateStore - timetable *TimeTable // config is the FSM config config *FSMConfig @@ -153,8 +144,7 @@ type nomadFSM struct { // state in a way that can be accessed concurrently with operations // that may modify the live state. type nomadSnapshot struct { - snap *state.StateSnapshot - timetable *TimeTable + snap *state.StateSnapshot } // SnapshotHeader is the first entry in our snapshot @@ -217,7 +207,6 @@ func NewFSM(config *FSMConfig) (*nomadFSM, error) { logger: config.Logger.Named("fsm"), config: config, state: state, - timetable: NewTimeTable(timeTableGranularity, timeTableLimit), enterpriseAppliers: make(map[structs.MessageType]LogApplier, 8), enterpriseRestorers: make(map[SnapshotType]SnapshotRestorer, 8), } @@ -244,18 +233,10 @@ func (n *nomadFSM) State() *state.StateStore { return n.state } -// TimeTable returns the time table of transactions -func (n *nomadFSM) TimeTable() *TimeTable { - return n.timetable -} - func (n *nomadFSM) Apply(log *raft.Log) interface{} { buf := log.Data msgType := structs.MessageType(buf[0]) - // Witness this write - n.timetable.Witness(log.Index, time.Now().UTC()) - // Check if this message type should be ignored when unknown. This is // used so that new commands can be added with developer control if older // versions can safely ignore the command, or if they should crash. @@ -1416,7 +1397,7 @@ func (n *nomadFSM) applyCSIVolumeBatchClaim(buf []byte, index uint64) interface{ defer metrics.MeasureSince([]string{"nomad", "fsm", "apply_csi_volume_batch_claim"}, time.Now()) for _, req := range batch.Claims { - err := n.state.CSIVolumeClaim(index, req.RequestNamespace(), + err := n.state.CSIVolumeClaim(index, req.Timestamp, req.RequestNamespace(), req.VolumeID, req.ToClaim()) if err != nil { n.logger.Error("CSIVolumeClaim for batch failed", "error", err) @@ -1433,7 +1414,7 @@ func (n *nomadFSM) applyCSIVolumeClaim(buf []byte, index uint64) interface{} { } defer metrics.MeasureSince([]string{"nomad", "fsm", "apply_csi_volume_claim"}, time.Now()) - if err := n.state.CSIVolumeClaim(index, req.RequestNamespace(), req.VolumeID, req.ToClaim()); err != nil { + if err := n.state.CSIVolumeClaim(index, req.Timestamp, req.RequestNamespace(), req.VolumeID, req.ToClaim()); err != nil { n.logger.Error("CSIVolumeClaim failed", "error", err) return err } @@ -1518,8 +1499,7 @@ func (n *nomadFSM) Snapshot() (raft.FSMSnapshot, error) { } ns := &nomadSnapshot{ - snap: snap, - timetable: n.timetable, + snap: snap, } return ns, nil } @@ -1584,10 +1564,9 @@ func (n *nomadFSM) restoreImpl(old io.ReadCloser, filter *FSMFilter) error { snapType := SnapshotType(msgType[0]) switch snapType { case TimeTableSnapshot: - if err := n.timetable.Deserialize(dec); err != nil { - return fmt.Errorf("time table deserialize failed: %v", err) - } - + // COMPAT: Nomad 1.9.2 removed the timetable, this case kept to gracefully handle + // tt snapshot requests + return nil case NodeSnapshot: node := new(structs.Node) if err := dec.Decode(node); err != nil { @@ -2426,13 +2405,6 @@ func (s *nomadSnapshot) Persist(sink raft.SnapshotSink) error { return err } - // Write the time table - sink.Write([]byte{byte(TimeTableSnapshot)}) - if err := s.timetable.Serialize(encoder); err != nil { - sink.Cancel() - return err - } - // Write all the data out if err := s.persistIndexes(sink, encoder); err != nil { sink.Cancel() diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 335b7c81451..1d5375cbb1e 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -169,12 +169,6 @@ func TestFSM_UpsertNode(t *testing.T) { t.Fatalf("bad index: %d", node.CreateIndex) } - tt := fsm.TimeTable() - index := tt.NearestIndex(time.Now().UTC()) - if index != 1 { - t.Fatalf("bad: %d", index) - } - // Verify the eval was unblocked. testutil.WaitForResult(func() (bool, error) { bStats := fsm.blockedEvals.Stats() @@ -1600,12 +1594,6 @@ func TestFSM_UpsertVaultAccessor(t *testing.T) { if out1.CreateIndex != 1 { t.Fatalf("bad index: %d", out2.CreateIndex) } - - tt := fsm.TimeTable() - index := tt.NearestIndex(time.Now().UTC()) - if index != 1 { - t.Fatalf("bad: %d", index) - } } func TestFSM_DeregisterVaultAccessor(t *testing.T) { @@ -1643,12 +1631,6 @@ func TestFSM_DeregisterVaultAccessor(t *testing.T) { if out1 != nil { t.Fatalf("not deleted!") } - - tt := fsm.TimeTable() - index := tt.NearestIndex(time.Now().UTC()) - if index != 1 { - t.Fatalf("bad: %d", index) - } } func TestFSM_UpsertSITokenAccessor(t *testing.T) { @@ -1680,10 +1662,6 @@ func TestFSM_UpsertSITokenAccessor(t *testing.T) { r.NoError(err) r.NotNil(result2) r.Equal(uint64(1), result2.CreateIndex) - - tt := fsm.TimeTable() - latestIndex := tt.NearestIndex(time.Now()) - r.Equal(uint64(1), latestIndex) } func TestFSM_DeregisterSITokenAccessor(t *testing.T) { @@ -1718,10 +1696,6 @@ func TestFSM_DeregisterSITokenAccessor(t *testing.T) { result2, err := fsm.State().SITokenAccessor(ws, a2.AccessorID) r.NoError(err) r.Nil(result2) // should have been deleted - - tt := fsm.TimeTable() - latestIndex := tt.NearestIndex(time.Now()) - r.Equal(uint64(1), latestIndex) } func TestFSM_ApplyPlanResults(t *testing.T) { @@ -2567,28 +2541,6 @@ func TestFSM_SnapshotRestore_Indexes(t *testing.T) { } } -func TestFSM_SnapshotRestore_TimeTable(t *testing.T) { - ci.Parallel(t) - // Add some state - fsm := testFSM(t) - - tt := fsm.TimeTable() - start := time.Now().UTC() - tt.Witness(1000, start) - tt.Witness(2000, start.Add(10*time.Minute)) - - // Verify the contents - fsm2 := testSnapshotRestore(t, fsm) - - tt2 := fsm2.TimeTable() - if tt2.NearestTime(1500) != start { - t.Fatalf("bad") - } - if tt2.NearestIndex(start.Add(15*time.Minute)) != 2000 { - t.Fatalf("bad") - } -} - func TestFSM_SnapshotRestore_PeriodicLaunches(t *testing.T) { ci.Parallel(t) // Add some state diff --git a/nomad/leader.go b/nomad/leader.go index e17cc74ef1a..271635b01ad 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -2872,10 +2872,6 @@ func (s *Server) handleEvalBrokerStateChange(schedConfig *structs.SchedulerConfi s.logger.Info("blocked evals status modified", "paused", !enableBrokers) s.blockedEvals.SetEnabled(enableBrokers) restoreEvals = enableBrokers - - if enableBrokers { - s.blockedEvals.SetTimetable(s.fsm.TimeTable()) - } } return restoreEvals diff --git a/nomad/mock/csi.go b/nomad/mock/csi.go index aa4176c59d4..01807dd4e3f 100644 --- a/nomad/mock/csi.go +++ b/nomad/mock/csi.go @@ -5,6 +5,7 @@ package mock import ( "fmt" + "time" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" @@ -50,6 +51,8 @@ func CSIVolume(plugin *structs.CSIPlugin) *structs.CSIVolume { ControllersExpected: len(plugin.Controllers), NodesHealthy: plugin.NodesHealthy, NodesExpected: len(plugin.Nodes), + CreateTime: time.Now().Add(-6 * time.Hour).UnixNano(), + ModifyTime: time.Now().Add(-5 * time.Hour).UnixNano(), } } diff --git a/nomad/mock/job.go b/nomad/mock/job.go index d3a70904286..9e37f524209 100644 --- a/nomad/mock/job.go +++ b/nomad/mock/job.go @@ -144,6 +144,7 @@ func Job() *structs.Job { CreateIndex: 42, ModifyIndex: 99, JobModifyIndex: 99, + SubmitTime: time.Now().Add(-6 * time.Hour).UnixNano(), } job.Canonicalize() return job diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index 6636d210166..306928032a0 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -184,6 +184,8 @@ func Deployment() *structs.Deployment { StatusDescription: structs.DeploymentStatusDescriptionRunning, ModifyIndex: 23, CreateIndex: 21, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } } diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 687d5511832..ed6deefc298 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -18,7 +18,6 @@ import ( "github.com/hashicorp/nomad/testutil" "github.com/hashicorp/raft" "github.com/shoenig/test/must" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -52,23 +51,6 @@ func testRegisterNode(t *testing.T, s *Server, n *structs.Node) { } } -func testRegisterJob(t *testing.T, s *Server, j *structs.Job) { - // Create the register request - req := &structs.JobRegisterRequest{ - Job: j, - WriteRequest: structs.WriteRequest{Region: "global"}, - } - - // Fetch the response - var resp structs.JobRegisterResponse - if err := s.RPC("Job.Register", req, &resp); err != nil { - t.Fatalf("err: %v", err) - } - if resp.Index == 0 { - t.Fatalf("bad index: %d", resp.Index) - } -} - // COMPAT 0.11: Tests the older unoptimized code path for applyPlan func TestPlanApply_applyPlan(t *testing.T) { ci.Parallel(t) @@ -83,9 +65,7 @@ func TestPlanApply_applyPlan(t *testing.T) { // Register a fake deployment oldDeployment := mock.Deployment() - if err := s1.State().UpsertDeployment(900, oldDeployment); err != nil { - t.Fatalf("UpsertDeployment failed: %v", err) - } + must.NoError(t, s1.State().UpsertDeployment(900, oldDeployment)) // Create a deployment dnew := mock.Deployment() @@ -102,13 +82,11 @@ func TestPlanApply_applyPlan(t *testing.T) { // Register alloc, deployment and deployment update alloc := mock.Alloc() - s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) + must.NoError(t, s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID))) // Create an eval eval := mock.Eval() eval.JobID = alloc.JobID - if err := s1.State().UpsertEvals(structs.MsgTypeTestSetup, 1, []*structs.Evaluation{eval}); err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, s1.State().UpsertEvals(structs.MsgTypeTestSetup, 1, []*structs.Evaluation{eval})) planRes := &structs.PlanResult{ NodeAllocation: map[string][]*structs.Allocation{ @@ -120,9 +98,7 @@ func TestPlanApply_applyPlan(t *testing.T) { // Snapshot the state snap, err := s1.State().Snapshot() - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, err) // Create the plan with a deployment plan := &structs.Plan{ @@ -134,50 +110,49 @@ func TestPlanApply_applyPlan(t *testing.T) { // Apply the plan future, err := s1.applyPlan(plan, planRes, snap) - assert := assert.New(t) - assert.Nil(err) + must.NoError(t, err) // Verify our optimistic snapshot is updated ws := memdb.NewWatchSet() allocOut, err := snap.AllocByID(ws, alloc.ID) - assert.Nil(err) - assert.NotNil(allocOut) + must.NoError(t, err) + must.NotNil(t, allocOut) deploymentOut, err := snap.DeploymentByID(ws, plan.Deployment.ID) - assert.Nil(err) - assert.NotNil(deploymentOut) + must.NoError(t, err) + must.NotNil(t, deploymentOut) // Check plan does apply cleanly index, err := planWaitFuture(future) - assert.Nil(err) - assert.NotEqual(0, index) + must.NoError(t, err) + must.NotNil(t, index) // Lookup the allocation fsmState := s1.fsm.State() allocOut, err = fsmState.AllocByID(ws, alloc.ID) - assert.Nil(err) - assert.NotNil(allocOut) - assert.True(allocOut.CreateTime > 0) - assert.True(allocOut.ModifyTime > 0) - assert.Equal(allocOut.CreateTime, allocOut.ModifyTime) + must.NoError(t, err) + must.NotNil(t, allocOut) + must.True(t, allocOut.CreateTime > 0) + must.True(t, allocOut.ModifyTime > 0) + must.Eq(t, allocOut.CreateTime, allocOut.ModifyTime) // Lookup the new deployment dout, err := fsmState.DeploymentByID(ws, plan.Deployment.ID) - assert.Nil(err) - assert.NotNil(dout) + must.NoError(t, err) + must.NotNil(t, dout) // Lookup the updated deployment dout2, err := fsmState.DeploymentByID(ws, oldDeployment.ID) - assert.Nil(err) - assert.NotNil(dout2) - assert.Equal(desiredStatus, dout2.Status) - assert.Equal(desiredStatusDescription, dout2.StatusDescription) + must.NoError(t, err) + must.NotNil(t, dout2) + must.Eq(t, desiredStatus, dout2.Status) + must.Eq(t, desiredStatusDescription, dout2.StatusDescription) // Lookup updated eval evalOut, err := fsmState.EvalByID(ws, eval.ID) - assert.Nil(err) - assert.NotNil(evalOut) - assert.Equal(index, evalOut.ModifyIndex) + must.NoError(t, err) + must.NotNil(t, evalOut) + must.Eq(t, index, evalOut.ModifyIndex) // Evict alloc, Register alloc2 allocEvict := new(structs.Allocation) @@ -186,7 +161,7 @@ func TestPlanApply_applyPlan(t *testing.T) { job := allocEvict.Job allocEvict.Job = nil alloc2 := mock.Alloc() - s1.State().UpsertJobSummary(1500, mock.JobSummary(alloc2.JobID)) + must.NoError(t, s1.State().UpsertJobSummary(1500, mock.JobSummary(alloc2.JobID))) planRes = &structs.PlanResult{ NodeUpdate: map[string][]*structs.Allocation{ node.ID: {allocEvict}, @@ -198,7 +173,7 @@ func TestPlanApply_applyPlan(t *testing.T) { // Snapshot the state snap, err = s1.State().Snapshot() - assert.Nil(err) + must.NoError(t, err) // Apply the plan plan = &structs.Plan{ @@ -206,40 +181,40 @@ func TestPlanApply_applyPlan(t *testing.T) { EvalID: eval.ID, } future, err = s1.applyPlan(plan, planRes, snap) - assert.Nil(err) + must.NoError(t, err) // Check that our optimistic view is updated out, _ := snap.AllocByID(ws, allocEvict.ID) if out.DesiredStatus != structs.AllocDesiredStatusEvict && out.DesiredStatus != structs.AllocDesiredStatusStop { - assert.Equal(structs.AllocDesiredStatusEvict, out.DesiredStatus) + must.Eq(t, structs.AllocDesiredStatusEvict, out.DesiredStatus) } // Verify plan applies cleanly index, err = planWaitFuture(future) - assert.Nil(err) - assert.NotEqual(0, index) + must.NoError(t, err) + must.NotEq(t, 0, index) // Lookup the allocation allocOut, err = s1.fsm.State().AllocByID(ws, alloc.ID) - assert.Nil(err) + must.NoError(t, err) if allocOut.DesiredStatus != structs.AllocDesiredStatusEvict && allocOut.DesiredStatus != structs.AllocDesiredStatusStop { - assert.Equal(structs.AllocDesiredStatusEvict, allocOut.DesiredStatus) + must.Eq(t, structs.AllocDesiredStatusEvict, allocOut.DesiredStatus) } - assert.NotNil(allocOut.Job) - assert.True(allocOut.ModifyTime > 0) + must.NotNil(t, allocOut.Job) + must.True(t, allocOut.ModifyTime > 0) // Lookup the allocation allocOut, err = s1.fsm.State().AllocByID(ws, alloc2.ID) - assert.Nil(err) - assert.NotNil(allocOut) - assert.NotNil(allocOut.Job) + must.NoError(t, err) + must.NotNil(t, allocOut) + must.NotNil(t, allocOut.Job) // Lookup updated eval evalOut, err = fsmState.EvalByID(ws, eval.ID) - assert.Nil(err) - assert.NotNil(evalOut) - assert.Equal(index, evalOut.ModifyIndex) + must.NoError(t, err) + must.NotNil(t, evalOut) + must.Eq(t, index, evalOut.ModifyIndex) } // Verifies that applyPlan properly updates the constituent objects in MemDB, @@ -289,8 +264,9 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { ID: preemptedAlloc.ID, PreemptedByAllocation: alloc.ID, } - s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) - s1.State().UpsertAllocs(structs.MsgTypeTestSetup, 1100, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) + must.NoError(t, s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID))) + must.NoError(t, s1.State().UpsertAllocs(structs.MsgTypeTestSetup, 1100, []*structs.Allocation{stoppedAlloc, preemptedAlloc})) + // Create an eval eval := mock.Eval() eval.JobID = alloc.JobID @@ -298,7 +274,7 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { t.Fatalf("err: %v", err) } - timestampBeforeCommit := time.Now().UTC().UnixNano() + timestampBeforeCommit := time.Now().UnixNano() planRes := &structs.PlanResult{ NodeAllocation: map[string][]*structs.Allocation{ node.ID: {alloc}, @@ -315,9 +291,7 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { // Snapshot the state snap, err := s1.State().Snapshot() - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, err) // Create the plan with a deployment plan := &structs.Plan{ @@ -327,72 +301,69 @@ func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { EvalID: eval.ID, } - require := require.New(t) - assert := assert.New(t) - // Apply the plan future, err := s1.applyPlan(plan, planRes, snap) - require.NoError(err) + must.NoError(t, err) // Verify our optimistic snapshot is updated ws := memdb.NewWatchSet() allocOut, err := snap.AllocByID(ws, alloc.ID) - require.NoError(err) - require.NotNil(allocOut) + must.NoError(t, err) + must.NotNil(t, allocOut) deploymentOut, err := snap.DeploymentByID(ws, plan.Deployment.ID) - require.NoError(err) - require.NotNil(deploymentOut) + must.NoError(t, err) + must.NotNil(t, deploymentOut) // Check plan does apply cleanly index, err := planWaitFuture(future) - require.NoError(err) - assert.NotEqual(0, index) + must.NoError(t, err) + must.NotEq(t, 0, index) // Lookup the allocation fsmState := s1.fsm.State() allocOut, err = fsmState.AllocByID(ws, alloc.ID) - require.NoError(err) - require.NotNil(allocOut) - assert.True(allocOut.CreateTime > 0) - assert.True(allocOut.ModifyTime > 0) - assert.Equal(allocOut.CreateTime, allocOut.ModifyTime) + must.NoError(t, err) + must.NotNil(t, allocOut) + must.True(t, allocOut.CreateTime > 0) + must.True(t, allocOut.ModifyTime > 0) + must.Eq(t, allocOut.CreateTime, allocOut.ModifyTime) // Verify stopped alloc diff applied cleanly updatedStoppedAlloc, err := fsmState.AllocByID(ws, stoppedAlloc.ID) - require.NoError(err) - require.NotNil(updatedStoppedAlloc) - assert.True(updatedStoppedAlloc.ModifyTime > timestampBeforeCommit) - assert.Equal(updatedStoppedAlloc.DesiredDescription, stoppedAllocDiff.DesiredDescription) - assert.Equal(updatedStoppedAlloc.ClientStatus, stoppedAllocDiff.ClientStatus) - assert.Equal(updatedStoppedAlloc.DesiredStatus, structs.AllocDesiredStatusStop) + must.NoError(t, err) + must.NotNil(t, updatedStoppedAlloc) + must.True(t, updatedStoppedAlloc.ModifyTime > timestampBeforeCommit) + must.Eq(t, updatedStoppedAlloc.DesiredDescription, stoppedAllocDiff.DesiredDescription) + must.Eq(t, updatedStoppedAlloc.ClientStatus, stoppedAllocDiff.ClientStatus) + must.Eq(t, updatedStoppedAlloc.DesiredStatus, structs.AllocDesiredStatusStop) // Verify preempted alloc diff applied cleanly updatedPreemptedAlloc, err := fsmState.AllocByID(ws, preemptedAlloc.ID) - require.NoError(err) - require.NotNil(updatedPreemptedAlloc) - assert.True(updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) - assert.Equal(updatedPreemptedAlloc.DesiredDescription, + must.NoError(t, err) + must.NotNil(t, updatedPreemptedAlloc) + must.True(t, updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) + must.Eq(t, updatedPreemptedAlloc.DesiredDescription, "Preempted by alloc ID "+preemptedAllocDiff.PreemptedByAllocation) - assert.Equal(updatedPreemptedAlloc.DesiredStatus, structs.AllocDesiredStatusEvict) + must.Eq(t, updatedPreemptedAlloc.DesiredStatus, structs.AllocDesiredStatusEvict) // Lookup the new deployment dout, err := fsmState.DeploymentByID(ws, plan.Deployment.ID) - require.NoError(err) - require.NotNil(dout) + must.NoError(t, err) + must.NotNil(t, dout) // Lookup the updated deployment dout2, err := fsmState.DeploymentByID(ws, oldDeployment.ID) - require.NoError(err) - require.NotNil(dout2) - assert.Equal(desiredStatus, dout2.Status) - assert.Equal(desiredStatusDescription, dout2.StatusDescription) + must.NoError(t, err) + must.NotNil(t, dout2) + must.Eq(t, desiredStatus, dout2.Status) + must.Eq(t, desiredStatusDescription, dout2.StatusDescription) // Lookup updated eval evalOut, err := fsmState.EvalByID(ws, eval.ID) - require.NoError(err) - require.NotNil(evalOut) - assert.Equal(index, evalOut.ModifyIndex) + must.NoError(t, err) + must.NotNil(t, evalOut) + must.Eq(t, index, evalOut.ModifyIndex) } func TestPlanApply_signAllocIdentities(t *testing.T) { diff --git a/nomad/server.go b/nomad/server.go index 4e5a33d9c9c..d69cb2b8fc7 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1388,6 +1388,7 @@ func (s *Server) setupRaft() error { EventBufferSize: s.config.EventBufferSize, JobTrackedVersions: s.config.JobTrackedVersions, } + var err error s.fsm, err = NewFSM(fsmConfig) if err != nil { diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index f24fb7a84a4..545c3f3201b 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -415,7 +415,7 @@ func (s *StateStore) UpsertPlanResults(msgType structs.MessageType, index uint64 // Update the status of deployments effected by the plan. if len(results.DeploymentUpdates) != 0 { - s.upsertDeploymentUpdates(index, results.DeploymentUpdates, txn) + s.upsertDeploymentUpdates(index, results.UpdatedAt, results.DeploymentUpdates, txn) } if results.EvalID != "" { @@ -515,7 +515,7 @@ func addComputedAllocAttrs(allocs []*structs.Allocation, job *structs.Job) { // upsertDeploymentUpdates updates the deployments given the passed status // updates. -func (s *StateStore) upsertDeploymentUpdates(index uint64, updates []*structs.DeploymentStatusUpdate, txn *txn) error { +func (s *StateStore) upsertDeploymentUpdates(index uint64, now int64, updates []*structs.DeploymentStatusUpdate, txn *txn) error { for _, u := range updates { if err := s.updateDeploymentStatusImpl(index, u, txn); err != nil { return err @@ -591,7 +591,7 @@ func (s *StateStore) upsertDeploymentImpl(index uint64, deployment *structs.Depl return fmt.Errorf("deployment lookup failed: %v", err) } - // Setup the indexes correctly + // Setup the indexes and timestamps correctly if existing != nil { deployment.CreateIndex = existing.(*structs.Deployment).CreateIndex deployment.ModifyIndex = index @@ -2779,7 +2779,7 @@ func (s *StateStore) csiVolumesByNamespaceImpl(txn *txn, ws memdb.WatchSet, name } // CSIVolumeClaim updates the volume's claim count and allocation list -func (s *StateStore) CSIVolumeClaim(index uint64, namespace, id string, claim *structs.CSIVolumeClaim) error { +func (s *StateStore) CSIVolumeClaim(index uint64, now int64, namespace, id string, claim *structs.CSIVolumeClaim) error { txn := s.db.WriteTxn(index) defer txn.Abort() @@ -2805,9 +2805,6 @@ func (s *StateStore) CSIVolumeClaim(index uint64, namespace, id string, claim *s } if alloc == nil { s.logger.Error("AllocByID failed to find alloc", "alloc_id", claim.AllocationID) - if err != nil { - return fmt.Errorf(structs.ErrUnknownAllocationPrefix) - } } } @@ -2831,6 +2828,7 @@ func (s *StateStore) CSIVolumeClaim(index uint64, namespace, id string, claim *s } volume.ModifyIndex = index + volume.ModifyTime = now // Allocations are copy on write, so we want to keep the Allocation ID // but we need to clear the pointer so that we don't store it when we @@ -3174,6 +3172,7 @@ func (s *StateStore) UpsertCSIPlugin(index uint64, plug *structs.CSIPlugin) erro plug.ModifyIndex = index if existing != nil { plug.CreateIndex = existing.(*structs.CSIPlugin).CreateIndex + plug.CreateTime = existing.(*structs.CSIPlugin).CreateTime } err = txn.Insert("csi_plugins", plug) @@ -4866,6 +4865,7 @@ func (s *StateStore) updateDeploymentStatusImpl(index uint64, u *structs.Deploym copy.Status = u.Status copy.StatusDescription = u.StatusDescription copy.ModifyIndex = index + copy.ModifyTime = u.UpdatedAt // Insert the deployment if err := txn.Insert("deployment", copy); err != nil { @@ -5125,6 +5125,9 @@ func (s *StateStore) UpdateDeploymentPromotion(msgType structs.MessageType, inde copy.StatusDescription = structs.DeploymentStatusDescriptionRunning } + // Update modify time to the time of deployment promotion + copy.ModifyTime = req.PromotedAt + // Insert the deployment if err := s.upsertDeploymentImpl(index, copy, txn); err != nil { return err @@ -5199,6 +5202,7 @@ func (s *StateStore) UpdateDeploymentAllocHealth(msgType structs.MessageType, in copy.DeploymentStatus.Healthy = pointer.Of(healthy) copy.DeploymentStatus.Timestamp = ts copy.DeploymentStatus.ModifyIndex = index + copy.ModifyTime = req.Timestamp.UnixNano() copy.ModifyIndex = index if err := s.updateDeploymentWithAlloc(index, copy, old, txn); err != nil { @@ -5971,6 +5975,7 @@ func (s *StateStore) updateDeploymentWithAlloc(index uint64, alloc, existing *st // Create a copy of the deployment object deploymentCopy := deployment.Copy() deploymentCopy.ModifyIndex = index + deploymentCopy.ModifyTime = alloc.ModifyTime dstate := deploymentCopy.TaskGroups[alloc.TaskGroup] dstate.PlacedAllocs += placed diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index abb59857669..c0efa31112f 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -192,7 +192,8 @@ func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { } require := require.New(t) - require.NoError(state.UpsertAllocs(structs.MsgTypeTestSetup, 900, []*structs.Allocation{stoppedAlloc, preemptedAlloc})) + require.NoError(state.UpsertAllocs( + structs.MsgTypeTestSetup, 900, []*structs.Allocation{stoppedAlloc, preemptedAlloc})) require.NoError(state.UpsertJob(structs.MsgTypeTestSetup, 999, nil, job)) // modify job and ensure that stopped and preempted alloc point to original Job @@ -3986,6 +3987,8 @@ func TestStateStore_CSIVolume(t *testing.T) { require.NoError(t, err) defer state.DeleteNode(structs.MsgTypeTestSetup, 9999, []string{pluginID}) + now := time.Now().UnixNano() + index++ err = state.UpsertAllocs(structs.MsgTypeTestSetup, index, []*structs.Allocation{alloc}) require.NoError(t, err) @@ -4086,10 +4089,10 @@ func TestStateStore_CSIVolume(t *testing.T) { } index++ - err = state.CSIVolumeClaim(index, ns, vol0, claim0) + err = state.CSIVolumeClaim(index, now, ns, vol0, claim0) require.NoError(t, err) index++ - err = state.CSIVolumeClaim(index, ns, vol0, claim1) + err = state.CSIVolumeClaim(index, now, ns, vol0, claim1) require.NoError(t, err) ws = memdb.NewWatchSet() @@ -4101,7 +4104,7 @@ func TestStateStore_CSIVolume(t *testing.T) { claim2 := new(structs.CSIVolumeClaim) *claim2 = *claim0 claim2.Mode = u - err = state.CSIVolumeClaim(2, ns, vol0, claim2) + err = state.CSIVolumeClaim(2, now, ns, vol0, claim2) require.NoError(t, err) ws = memdb.NewWatchSet() iter, err = state.CSIVolumesByPluginID(ws, ns, "", "minnie") @@ -4129,12 +4132,12 @@ func TestStateStore_CSIVolume(t *testing.T) { claim3 := new(structs.CSIVolumeClaim) *claim3 = *claim2 claim3.State = structs.CSIVolumeClaimStateReadyToFree - err = state.CSIVolumeClaim(index, ns, vol0, claim3) + err = state.CSIVolumeClaim(index, now, ns, vol0, claim3) require.NoError(t, err) index++ claim1.Mode = u claim1.State = structs.CSIVolumeClaimStateReadyToFree - err = state.CSIVolumeClaim(index, ns, vol0, claim1) + err = state.CSIVolumeClaim(index, now, ns, vol0, claim1) require.NoError(t, err) index++ @@ -7438,7 +7441,8 @@ func TestStateStore_AllocsByIDPrefix_Namespaces(t *testing.T) { alloc2.Namespace = ns2.Name require.NoError(t, state.UpsertNamespaces(998, []*structs.Namespace{ns1, ns2})) - require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc1, alloc2})) + require.NoError(t, state.UpsertAllocs( + structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc1, alloc2})) gatherAllocs := func(iter memdb.ResultIterator) []*structs.Allocation { var allocs []*structs.Allocation @@ -8223,6 +8227,7 @@ func TestStateStore_UpsertDeploymentStatusUpdate_Successful(t *testing.T) { ci.Parallel(t) state := testStateStore(t) + now := time.Now().UnixNano() // Insert a job job := mock.Job() @@ -8231,7 +8236,7 @@ func TestStateStore_UpsertDeploymentStatusUpdate_Successful(t *testing.T) { } // Insert a deployment - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, now) if err := state.UpsertDeployment(2, d); err != nil { t.Fatalf("bad: %v", err) } diff --git a/nomad/state/testing.go b/nomad/state/testing.go index cb955ffa46c..e238dbedfed 100644 --- a/nomad/state/testing.go +++ b/nomad/state/testing.go @@ -92,6 +92,7 @@ func createTestCSIPlugin(s *StateStore, id string, requiresController bool) func SupportsCreateDeleteSnapshot: true, SupportsListSnapshots: true, }, + UpdateTime: time.Now().Add(-6 * time.Hour), }, } diff --git a/nomad/structs/csi.go b/nomad/structs/csi.go index a0eac38a6c5..8dd85db766e 100644 --- a/nomad/structs/csi.go +++ b/nomad/structs/csi.go @@ -313,6 +313,10 @@ type CSIVolume struct { CreateIndex uint64 ModifyIndex uint64 + + // Creation and modification times stored as UnixNano + CreateTime int64 + ModifyTime int64 } // GetID implements the IDGetter interface, required for pagination. @@ -364,14 +368,21 @@ type CSIVolListStub struct { CreateIndex uint64 ModifyIndex uint64 + + // Create and modify times stored as UnixNano + CreateTime int64 + ModifyTime int64 } // NewCSIVolume creates the volume struct. No side-effects func NewCSIVolume(volumeID string, index uint64) *CSIVolume { + now := time.Now().UnixNano() out := &CSIVolume{ ID: volumeID, CreateIndex: index, ModifyIndex: index, + CreateTime: now, + ModifyTime: now, } out.newStructs() @@ -421,6 +432,8 @@ func (v *CSIVolume) Stub() *CSIVolListStub { ResourceExhausted: v.ResourceExhausted, CreateIndex: v.CreateIndex, ModifyIndex: v.ModifyIndex, + CreateTime: v.CreateTime, + ModifyTime: v.ModifyTime, } } @@ -841,7 +854,8 @@ func (v *CSIVolume) Merge(other *CSIVolume) error { // Request and response wrappers type CSIVolumeRegisterRequest struct { - Volumes []*CSIVolume + Volumes []*CSIVolume + Timestamp int64 // UnixNano WriteRequest } @@ -860,7 +874,8 @@ type CSIVolumeDeregisterResponse struct { } type CSIVolumeCreateRequest struct { - Volumes []*CSIVolume + Volumes []*CSIVolume + Timestamp int64 // UnixNano WriteRequest } @@ -917,6 +932,7 @@ type CSIVolumeClaimRequest struct { AccessMode CSIVolumeAccessMode AttachmentMode CSIVolumeAttachmentMode State CSIVolumeClaimState + Timestamp int64 // UnixNano WriteRequest } @@ -1097,14 +1113,21 @@ type CSIPlugin struct { CreateIndex uint64 ModifyIndex uint64 + + // Create and modify times stored as UnixNano + CreateTime int64 + ModifyTime int64 } // NewCSIPlugin creates the plugin struct. No side-effects func NewCSIPlugin(id string, index uint64) *CSIPlugin { + now := time.Now().UnixNano() out := &CSIPlugin{ ID: id, CreateIndex: index, ModifyIndex: index, + CreateTime: now, + ModifyTime: now, } out.newStructs() diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 3a5305f7a7e..004da160c37 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1381,6 +1381,9 @@ type DeploymentPromoteRequest struct { // Groups is used to set the promotion status per task group Groups []string + // PromotedAt is the timestamp stored as Unix nano + PromotedAt int64 + WriteRequest } @@ -2169,7 +2172,7 @@ type Node struct { StatusDescription string // StatusUpdatedAt is the time stamp at which the state of the node was - // updated + // updated, stored as Unix (no nano seconds!) StatusUpdatedAt int64 // Events is the most recent set of events generated for the node, @@ -10629,10 +10632,14 @@ type Deployment struct { CreateIndex uint64 ModifyIndex uint64 + + // Creation and modification times, stored as UnixNano + CreateTime int64 + ModifyTime int64 } // NewDeployment creates a new deployment given the job. -func NewDeployment(job *Job, evalPriority int) *Deployment { +func NewDeployment(job *Job, evalPriority int, now int64) *Deployment { return &Deployment{ ID: uuid.Generate(), Namespace: job.Namespace, @@ -10646,6 +10653,7 @@ func NewDeployment(job *Job, evalPriority int) *Deployment { StatusDescription: DeploymentStatusDescriptionRunning, TaskGroups: make(map[string]*DeploymentState, len(job.TaskGroups)), EvalPriority: evalPriority, + CreateTime: now, } } @@ -10825,6 +10833,9 @@ type DeploymentStatusUpdate struct { // StatusDescription is the new status description of the deployment. StatusDescription string + + // UpdatedAt is the time of the update, stored as UnixNano + UpdatedAt int64 } // RescheduleTracker encapsulates previous reschedule events @@ -11154,10 +11165,10 @@ type Allocation struct { AllocModifyIndex uint64 // CreateTime is the time the allocation has finished scheduling and been - // verified by the plan applier. + // verified by the plan applier, stored as UnixNano. CreateTime int64 - // ModifyTime is the time the allocation was last updated. + // ModifyTime is the time the allocation was last updated stored as UnixNano. ModifyTime int64 } @@ -12543,6 +12554,7 @@ type Evaluation struct { CreateIndex uint64 ModifyIndex uint64 + // Creation and modification times stored as UnixNano CreateTime int64 ModifyTime int64 } diff --git a/nomad/system_endpoint_test.go b/nomad/system_endpoint_test.go index c1467adefbc..5d879b83481 100644 --- a/nomad/system_endpoint_test.go +++ b/nomad/system_endpoint_test.go @@ -7,6 +7,7 @@ import ( "fmt" "reflect" "testing" + "time" memdb "github.com/hashicorp/go-memdb" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc/v2" @@ -15,6 +16,8 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" + "github.com/shoenig/test/wait" "github.com/stretchr/testify/assert" ) @@ -31,16 +34,16 @@ func TestSystemEndpoint_GarbageCollect(t *testing.T) { job := mock.Job() job.Type = structs.JobTypeBatch job.Stop = true - if err := state.UpsertJob(structs.MsgTypeTestSetup, 1000, nil, job); err != nil { - t.Fatalf("UpsertJob() failed: %v", err) - } + // submit time must be older than default job GC + job.SubmitTime = time.Now().Add(-6 * time.Hour).UnixNano() + must.NoError(t, state.UpsertJob(structs.MsgTypeTestSetup, 1000, nil, job)) eval := mock.Eval() eval.Status = structs.EvalStatusComplete eval.JobID = job.ID - if err := state.UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval}); err != nil { - t.Fatalf("UpsertEvals() failed: %v", err) - } + // modify time must be older than default eval GC + eval.ModifyTime = time.Now().Add(-5 * time.Hour).UnixNano() + must.NoError(t, state.UpsertEvals(structs.MsgTypeTestSetup, 1001, []*structs.Evaluation{eval})) // Make the GC request req := &structs.GenericRequest{ @@ -49,11 +52,9 @@ func TestSystemEndpoint_GarbageCollect(t *testing.T) { }, } var resp structs.GenericResponse - if err := msgpackrpc.CallWithCodec(codec, "System.GarbageCollect", req, &resp); err != nil { - t.Fatalf("expect err") - } + must.NoError(t, msgpackrpc.CallWithCodec(codec, "System.GarbageCollect", req, &resp)) - testutil.WaitForResult(func() (bool, error) { + must.Wait(t, wait.InitialSuccess(wait.TestFunc(func() (bool, error) { // Check if the job has been GC'd ws := memdb.NewWatchSet() exist, err := state.JobByID(ws, job.Namespace, job.ID) @@ -64,9 +65,7 @@ func TestSystemEndpoint_GarbageCollect(t *testing.T) { return false, fmt.Errorf("job %+v wasn't garbage collected", job) } return true, nil - }, func(err error) { - t.Fatalf("err: %s", err) - }) + }), wait.Timeout(3*time.Second))) } func TestSystemEndpoint_GarbageCollect_ACL(t *testing.T) { diff --git a/nomad/timetable.go b/nomad/timetable.go deleted file mode 100644 index cc99d6f2c14..00000000000 --- a/nomad/timetable.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package nomad - -import ( - "sort" - "sync" - "time" - - "github.com/hashicorp/go-msgpack/v2/codec" -) - -// TimeTable is used to associate a Raft index with a timestamp. -// This is used so that we can quickly go from a timestamp to an -// index or visa versa. -type TimeTable struct { - granularity time.Duration - limit time.Duration - table []TimeTableEntry - l sync.RWMutex -} - -// TimeTableEntry is used to track a time and index -type TimeTableEntry struct { - Index uint64 - Time time.Time -} - -// NewTimeTable creates a new time table which stores entries -// at a given granularity for a maximum limit. The storage space -// required is (limit/granularity) -func NewTimeTable(granularity time.Duration, limit time.Duration) *TimeTable { - size := limit / granularity - if size < 1 { - size = 1 - } - t := &TimeTable{ - granularity: granularity, - limit: limit, - table: make([]TimeTableEntry, 1, size), - } - return t -} - -// Serialize is used to serialize the time table -func (t *TimeTable) Serialize(enc *codec.Encoder) error { - t.l.RLock() - defer t.l.RUnlock() - return enc.Encode(t.table) -} - -// Deserialize is used to deserialize the time table -// and restore the state -func (t *TimeTable) Deserialize(dec *codec.Decoder) error { - // Decode the table - var table []TimeTableEntry - if err := dec.Decode(&table); err != nil { - return err - } - - // Witness from oldest to newest - n := len(table) - for i := n - 1; i >= 0; i-- { - t.Witness(table[i].Index, table[i].Time) - } - return nil -} - -// Witness is used to witness a new index and time. -func (t *TimeTable) Witness(index uint64, when time.Time) { - t.l.Lock() - defer t.l.Unlock() - - // Ensure monotonic indexes - if t.table[0].Index > index { - return - } - - // Skip if we already have a recent enough entry - if when.Sub(t.table[0].Time) < t.granularity { - return - } - - // Grow the table if we haven't reached the size - if len(t.table) < cap(t.table) { - t.table = append(t.table, TimeTableEntry{}) - } - - // Add this entry - copy(t.table[1:], t.table[:len(t.table)-1]) - t.table[0].Index = index - t.table[0].Time = when -} - -// NearestIndex returns the nearest index older than the given time -func (t *TimeTable) NearestIndex(when time.Time) uint64 { - t.l.RLock() - defer t.l.RUnlock() - - n := len(t.table) - idx := sort.Search(n, func(i int) bool { - return !t.table[i].Time.After(when) - }) - if idx < n && idx >= 0 { - return t.table[idx].Index - } - return 0 -} - -// NearestTime returns the nearest time older than the given index -func (t *TimeTable) NearestTime(index uint64) time.Time { - t.l.RLock() - defer t.l.RUnlock() - - n := len(t.table) - idx := sort.Search(n, func(i int) bool { - return t.table[i].Index <= index - }) - if idx < n && idx >= 0 { - return t.table[idx].Time - } - return time.Time{} -} diff --git a/nomad/timetable_test.go b/nomad/timetable_test.go deleted file mode 100644 index 5396218fab9..00000000000 --- a/nomad/timetable_test.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package nomad - -import ( - "bytes" - "sync" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/hashicorp/go-msgpack/v2/codec" - "github.com/hashicorp/nomad/ci" - "github.com/hashicorp/nomad/nomad/structs" -) - -func TestTimeTable(t *testing.T) { - ci.Parallel(t) - tt := NewTimeTable(time.Second, time.Minute) - - index := tt.NearestIndex(time.Now()) - if index != 0 { - t.Fatalf("bad: %v", index) - } - - when := tt.NearestTime(1000) - if !when.IsZero() { - t.Fatalf("bad: %v", when) - } - - // Witness some data - start := time.Now() - plusOne := start.Add(time.Minute) - plusTwo := start.Add(2 * time.Minute) - plusFive := start.Add(5 * time.Minute) - plusThirty := start.Add(30 * time.Minute) - plusHour := start.Add(60 * time.Minute) - plusHourHalf := start.Add(90 * time.Minute) - - tt.Witness(2, start) - tt.Witness(2, start) - - tt.Witness(10, plusOne) - tt.Witness(10, plusOne) - - tt.Witness(20, plusTwo) - tt.Witness(20, plusTwo) - - tt.Witness(30, plusFive) - tt.Witness(30, plusFive) - - tt.Witness(40, plusThirty) - tt.Witness(40, plusThirty) - - tt.Witness(50, plusHour) - tt.Witness(50, plusHour) - - type tcase struct { - when time.Time - expectIndex uint64 - - index uint64 - expectWhen time.Time - } - cases := []tcase{ - // Exact match - {start, 2, 2, start}, - {plusOne, 10, 10, plusOne}, - {plusHour, 50, 50, plusHour}, - - // Before the newest entry - {plusHourHalf, 50, 51, plusHour}, - - // After the oldest entry - {time.Time{}, 0, 1, time.Time{}}, - - // Mid range - {start.Add(3 * time.Minute), 20, 25, plusTwo}, - } - - for _, tc := range cases { - index := tt.NearestIndex(tc.when) - if index != tc.expectIndex { - t.Fatalf("bad: %v %v", index, tc.expectIndex) - } - - when := tt.NearestTime(tc.index) - if when != tc.expectWhen { - t.Fatalf("bad: for %d %v %v", tc.index, when, tc.expectWhen) - } - } -} - -func TestTimeTable_SerializeDeserialize(t *testing.T) { - ci.Parallel(t) - tt := NewTimeTable(time.Second, time.Minute) - - // Witness some data - start := time.Now() - plusOne := start.Add(time.Minute) - plusTwo := start.Add(2 * time.Minute) - plusFive := start.Add(5 * time.Minute) - plusThirty := start.Add(30 * time.Minute) - plusHour := start.Add(60 * time.Minute) - - tt.Witness(2, start) - tt.Witness(10, plusOne) - tt.Witness(20, plusTwo) - tt.Witness(30, plusFive) - tt.Witness(40, plusThirty) - tt.Witness(50, plusHour) - - var buf bytes.Buffer - enc := codec.NewEncoder(&buf, structs.MsgpackHandle) - - err := tt.Serialize(enc) - if err != nil { - t.Fatalf("err: %v", err) - } - - dec := codec.NewDecoder(&buf, structs.MsgpackHandle) - - tt2 := NewTimeTable(time.Second, time.Minute) - err = tt2.Deserialize(dec) - if err != nil { - t.Fatalf("err: %v", err) - } - - o := cmp.AllowUnexported(TimeTable{}) - o2 := cmpopts.IgnoreTypes(sync.RWMutex{}) - if !cmp.Equal(tt.table, tt2.table, o, o2) { - t.Fatalf("bad: %s", cmp.Diff(tt, tt2, o, o2)) - } -} - -func TestTimeTable_Overflow(t *testing.T) { - ci.Parallel(t) - tt := NewTimeTable(time.Second, 3*time.Second) - - // Witness some data - start := time.Now() - plusOne := start.Add(time.Second) - plusTwo := start.Add(2 * time.Second) - plusThree := start.Add(3 * time.Second) - - tt.Witness(10, start) - tt.Witness(20, plusOne) - tt.Witness(30, plusTwo) - tt.Witness(40, plusThree) - - if len(tt.table) != 3 { - t.Fatalf("bad") - } - - index := tt.NearestIndex(start) - if index != 0 { - t.Fatalf("bad: %v %v", index, 0) - } - - when := tt.NearestTime(15) - if !when.IsZero() { - t.Fatalf("bad: %v", when) - } -} diff --git a/nomad/volumewatcher/volumes_watcher_test.go b/nomad/volumewatcher/volumes_watcher_test.go index 83dc2e1f449..0ecfa67d386 100644 --- a/nomad/volumewatcher/volumes_watcher_test.go +++ b/nomad/volumewatcher/volumes_watcher_test.go @@ -48,7 +48,7 @@ func TestVolumeWatch_EnableDisable(t *testing.T) { State: structs.CSIVolumeClaimStateNodeDetached, } index++ - err = srv.State().CSIVolumeClaim(index, vol.Namespace, vol.ID, claim) + err = srv.State().CSIVolumeClaim(index, time.Now().UnixNano(), vol.Namespace, vol.ID, claim) require.NoError(t, err) require.Eventually(t, func() bool { watcher.wlock.RLock() @@ -127,7 +127,7 @@ func TestVolumeWatch_LeadershipTransition(t *testing.T) { State: structs.CSIVolumeClaimStateUnpublishing, } index++ - err = srv.State().CSIVolumeClaim(index, vol.Namespace, vol.ID, claim) + err = srv.State().CSIVolumeClaim(index, time.Now().UnixNano(), vol.Namespace, vol.ID, claim) require.NoError(t, err) // create a new watcher and enable it to simulate the leadership @@ -197,11 +197,11 @@ func TestVolumeWatch_StartStop(t *testing.T) { } index++ - err = srv.State().CSIVolumeClaim(index, vol.Namespace, vol.ID, claim) + err = srv.State().CSIVolumeClaim(index, time.Now().UnixNano(), vol.Namespace, vol.ID, claim) require.NoError(t, err) claim.AllocationID = alloc2.ID index++ - err = srv.State().CSIVolumeClaim(index, vol.Namespace, vol.ID, claim) + err = srv.State().CSIVolumeClaim(index, time.Now().UnixNano(), vol.Namespace, vol.ID, claim) require.NoError(t, err) // reap the volume and assert nothing has happened @@ -210,7 +210,7 @@ func TestVolumeWatch_StartStop(t *testing.T) { NodeID: node.ID, } index++ - err = srv.State().CSIVolumeClaim(index, vol.Namespace, vol.ID, claim) + err = srv.State().CSIVolumeClaim(index, time.Now().UnixNano(), vol.Namespace, vol.ID, claim) require.NoError(t, err) ws := memdb.NewWatchSet() @@ -225,7 +225,7 @@ func TestVolumeWatch_StartStop(t *testing.T) { require.NoError(t, err) index++ claim.State = structs.CSIVolumeClaimStateReadyToFree - err = srv.State().CSIVolumeClaim(index, vol.Namespace, vol.ID, claim) + err = srv.State().CSIVolumeClaim(index, time.Now().UnixNano(), vol.Namespace, vol.ID, claim) require.NoError(t, err) // watcher stops and 1 claim has been released @@ -270,7 +270,7 @@ func TestVolumeWatch_Delete(t *testing.T) { // write a GC claim to the volume and then immediately delete, to // potentially hit the race condition between updates and deletes index++ - must.NoError(t, srv.State().CSIVolumeClaim(index, vol.Namespace, vol.ID, + must.NoError(t, srv.State().CSIVolumeClaim(index, time.Now().UnixNano(), vol.Namespace, vol.ID, &structs.CSIVolumeClaim{ Mode: structs.CSIVolumeClaimGC, State: structs.CSIVolumeClaimStateReadyToFree, diff --git a/scheduler/reconcile.go b/scheduler/reconcile.go index 2d1451d8bc0..f07e83dd5a5 100644 --- a/scheduler/reconcile.go +++ b/scheduler/reconcile.go @@ -995,7 +995,7 @@ func (a *allocReconciler) createDeployment(groupName string, strategy *structs.U // A previous group may have made the deployment already. If not create one. if a.deployment == nil { - a.deployment = structs.NewDeployment(a.job, a.evalPriority) + a.deployment = structs.NewDeployment(a.job, a.evalPriority, a.now.UnixNano()) a.result.deployment = a.deployment } diff --git a/scheduler/reconcile_test.go b/scheduler/reconcile_test.go index 958f55ce4d5..653ea1d9a0d 100644 --- a/scheduler/reconcile_test.go +++ b/scheduler/reconcile_test.go @@ -1625,7 +1625,7 @@ func TestReconciler_MultiTG_SingleUpdateBlock(t *testing.T) { } } - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredTotal: 10, } @@ -2384,7 +2384,7 @@ func TestReconciler_RescheduleNow_Service_WithCanaries(t *testing.T) { job2 := job.Copy() job2.Version++ - d := structs.NewDeployment(job2, 50) + d := structs.NewDeployment(job2, 50, time.Now().UnixNano()) d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion s := &structs.DeploymentState{ DesiredCanaries: 2, @@ -2491,7 +2491,7 @@ func TestReconciler_RescheduleNow_Service_Canaries(t *testing.T) { job2 := job.Copy() job2.Version++ - d := structs.NewDeployment(job2, 50) + d := structs.NewDeployment(job2, 50, time.Now().UnixNano()) d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion s := &structs.DeploymentState{ DesiredCanaries: 2, @@ -2619,7 +2619,7 @@ func TestReconciler_RescheduleNow_Service_Canaries_Limit(t *testing.T) { job2 := job.Copy() job2.Version++ - d := structs.NewDeployment(job2, 50) + d := structs.NewDeployment(job2, 50, time.Now().UnixNano()) d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion s := &structs.DeploymentState{ DesiredCanaries: 2, @@ -2790,8 +2790,8 @@ func TestReconciler_CancelDeployment_JobStop(t *testing.T) { job := mock.Job() job.Stop = true - running := structs.NewDeployment(job, 50) - failed := structs.NewDeployment(job, 50) + running := structs.NewDeployment(job, 50, time.Now().UnixNano()) + failed := structs.NewDeployment(job, 50, time.Now().UnixNano()) failed.Status = structs.DeploymentStatusFailed cases := []struct { @@ -2891,8 +2891,8 @@ func TestReconciler_CancelDeployment_JobUpdate(t *testing.T) { job := mock.Job() // Create two deployments - running := structs.NewDeployment(job, 50) - failed := structs.NewDeployment(job, 50) + running := structs.NewDeployment(job, 50, time.Now().UnixNano()) + failed := structs.NewDeployment(job, 50, time.Now().UnixNano()) failed.Status = structs.DeploymentStatusFailed // Make the job newer than the deployment @@ -2985,7 +2985,9 @@ func TestReconciler_CreateDeployment_RollingUpgrade_Destructive(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - d := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + d := structs.NewDeployment(job, 50, r.deployment.CreateTime) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredTotal: 10, } @@ -3031,7 +3033,9 @@ func TestReconciler_CreateDeployment_RollingUpgrade_Inplace(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - d := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + d := structs.NewDeployment(job, 50, r.deployment.CreateTime) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredTotal: 10, } @@ -3076,7 +3080,9 @@ func TestReconciler_CreateDeployment_NewerCreateIndex(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - d := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + d := structs.NewDeployment(job, 50, r.deployment.CreateTime) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredTotal: 5, } @@ -3167,7 +3173,7 @@ func TestReconciler_PausedOrFailedDeployment_NoMoreCanaries(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { // Create a deployment that is paused/failed and has placed some canaries - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = c.deploymentStatus d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: false, @@ -3248,7 +3254,7 @@ func TestReconciler_PausedOrFailedDeployment_NoMorePlacements(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { // Create a deployment that is paused and has placed some canaries - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = c.deploymentStatus d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: false, @@ -3314,7 +3320,7 @@ func TestReconciler_PausedOrFailedDeployment_NoMoreDestructiveUpdates(t *testing for _, c := range cases { t.Run(c.name, func(t *testing.T) { // Create a deployment that is paused and has placed some canaries - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = c.deploymentStatus d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: false, @@ -3374,7 +3380,7 @@ func TestReconciler_DrainNode_Canary(t *testing.T) { job.TaskGroups[0].Update = canaryUpdate // Create a deployment that is paused and has placed some canaries - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) s := &structs.DeploymentState{ Promoted: false, DesiredTotal: 10, @@ -3449,7 +3455,7 @@ func TestReconciler_LostNode_Canary(t *testing.T) { job.TaskGroups[0].Update = canaryUpdate // Create a deployment that is paused and has placed some canaries - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) s := &structs.DeploymentState{ Promoted: false, DesiredTotal: 10, @@ -3525,7 +3531,7 @@ func TestReconciler_StopOldCanaries(t *testing.T) { job.TaskGroups[0].Update = canaryUpdate // Create an old deployment that has placed some canaries - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) s := &structs.DeploymentState{ Promoted: false, DesiredTotal: 10, @@ -3567,7 +3573,9 @@ func TestReconciler_StopOldCanaries(t *testing.T) { allocs, nil, "", 50, true) r := reconciler.Compute() - newD := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + newD := structs.NewDeployment(job, 50, r.deployment.CreateTime) newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion newD.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredCanaries: 2, @@ -3623,7 +3631,9 @@ func TestReconciler_NewCanaries(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - newD := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + newD := structs.NewDeployment(job, 50, r.deployment.CreateTime) newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion newD.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredCanaries: 2, @@ -3674,7 +3684,9 @@ func TestReconciler_NewCanaries_CountGreater(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - newD := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + newD := structs.NewDeployment(job, 50, r.deployment.CreateTime) newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion state := &structs.DeploymentState{ DesiredCanaries: 7, @@ -3728,7 +3740,9 @@ func TestReconciler_NewCanaries_MultiTG(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - newD := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + newD := structs.NewDeployment(job, 50, r.deployment.CreateTime) newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion state := &structs.DeploymentState{ DesiredCanaries: 2, @@ -3784,7 +3798,9 @@ func TestReconciler_NewCanaries_ScaleUp(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - newD := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + newD := structs.NewDeployment(job, 50, r.deployment.CreateTime) newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion newD.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredCanaries: 2, @@ -3835,7 +3851,9 @@ func TestReconciler_NewCanaries_ScaleDown(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - newD := structs.NewDeployment(job, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + newD := structs.NewDeployment(job, 50, r.deployment.CreateTime) newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion newD.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredCanaries: 2, @@ -3876,7 +3894,7 @@ func TestReconciler_NewCanaries_FillNames(t *testing.T) { } // Create an existing deployment that has placed some canaries - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) s := &structs.DeploymentState{ Promoted: false, DesiredTotal: 10, @@ -3942,7 +3960,7 @@ func TestReconciler_PromoteCanaries_Unblock(t *testing.T) { // Create an existing deployment that has placed some canaries and mark them // promoted - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) s := &structs.DeploymentState{ Promoted: true, DesiredTotal: 10, @@ -4018,7 +4036,7 @@ func TestReconciler_PromoteCanaries_CanariesEqualCount(t *testing.T) { // Create an existing deployment that has placed some canaries and mark them // promoted - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) s := &structs.DeploymentState{ Promoted: true, DesiredTotal: 2, @@ -4123,7 +4141,7 @@ func TestReconciler_DeploymentLimit_HealthAccounting(t *testing.T) { t.Run(fmt.Sprintf("%d healthy", c.healthy), func(t *testing.T) { // Create an existing deployment that has placed some canaries and mark them // promoted - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: true, DesiredTotal: 10, @@ -4195,7 +4213,7 @@ func TestReconciler_TaintedNode_RollingUpgrade(t *testing.T) { job.TaskGroups[0].Update = noCanaryUpdate // Create an existing deployment that has some placed allocs - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: true, DesiredTotal: 10, @@ -4282,7 +4300,7 @@ func TestReconciler_FailedDeployment_TaintedNodes(t *testing.T) { job.TaskGroups[0].Update = noCanaryUpdate // Create an existing failed deployment that has some placed allocs - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = structs.DeploymentStatusFailed d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: true, @@ -4367,7 +4385,7 @@ func TestReconciler_CompleteDeployment(t *testing.T) { job := mock.Job() job.TaskGroups[0].Update = canaryUpdate - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = structs.DeploymentStatusSuccessful d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: true, @@ -4421,7 +4439,7 @@ func TestReconciler_MarkDeploymentComplete_FailedAllocations(t *testing.T) { job := mock.Job() job.TaskGroups[0].Update = noCanaryUpdate - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredTotal: 10, PlacedAllocs: 20, @@ -4489,7 +4507,7 @@ func TestReconciler_FailedDeployment_CancelCanaries(t *testing.T) { job.TaskGroups[1].Name = "two" // Create an existing failed deployment that has promoted one task group - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = structs.DeploymentStatusFailed s0 := &structs.DeploymentState{ Promoted: true, @@ -4582,7 +4600,7 @@ func TestReconciler_FailedDeployment_NewJob(t *testing.T) { job.TaskGroups[0].Update = noCanaryUpdate // Create an existing failed deployment that has some placed allocs - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = structs.DeploymentStatusFailed d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: true, @@ -4625,7 +4643,9 @@ func TestReconciler_FailedDeployment_NewJob(t *testing.T) { d, allocs, nil, "", 50, true) r := reconciler.Compute() - dnew := structs.NewDeployment(jobNew, 50) + // reconciler sets the creation time automatically so we have to copy here, + // otherwise there will be a discrepancy + dnew := structs.NewDeployment(jobNew, 50, r.deployment.CreateTime) dnew.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredTotal: 10, } @@ -4653,7 +4673,7 @@ func TestReconciler_MarkDeploymentComplete(t *testing.T) { job := mock.Job() job.TaskGroups[0].Update = noCanaryUpdate - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: true, DesiredTotal: 10, @@ -4715,7 +4735,7 @@ func TestReconciler_JobChange_ScaleUp_SecondEval(t *testing.T) { job.TaskGroups[0].Count = 30 // Create a deployment that is paused and has placed some canaries - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: false, DesiredTotal: 30, @@ -4791,7 +4811,7 @@ func TestReconciler_RollingUpgrade_MissingAllocs(t *testing.T) { nil, allocs, nil, "", 50, true) r := reconciler.Compute() - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, r.deployment.CreateTime) d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ DesiredTotal: 10, } @@ -4874,7 +4894,7 @@ func TestReconciler_FailedDeployment_DontReschedule(t *testing.T) { tgName := job.TaskGroups[0].Name now := time.Now() // Create an existing failed deployment that has some placed allocs - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = structs.DeploymentStatusFailed d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: true, @@ -4934,7 +4954,7 @@ func TestReconciler_DeploymentWithFailedAllocs_DontReschedule(t *testing.T) { now := time.Now() // Mock deployment with failed allocs, but deployment watcher hasn't marked it as failed yet - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = structs.DeploymentStatusRunning d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: false, @@ -5010,7 +5030,7 @@ func TestReconciler_FailedDeployment_AutoRevert_CancelCanaries(t *testing.T) { jobv2.Version = 2 jobv2.TaskGroups[0].Meta = map[string]string{"version": "2"} - d := structs.NewDeployment(jobv2, 50) + d := structs.NewDeployment(jobv2, 50, time.Now().UnixNano()) state := &structs.DeploymentState{ Promoted: true, DesiredTotal: 3, @@ -5092,7 +5112,7 @@ func TestReconciler_SuccessfulDeploymentWithFailedAllocs_Reschedule(t *testing.T now := time.Now() // Mock deployment with failed allocs, but deployment watcher hasn't marked it as failed yet - d := structs.NewDeployment(job, 50) + d := structs.NewDeployment(job, 50, time.Now().UnixNano()) d.Status = structs.DeploymentStatusSuccessful d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: false, @@ -6181,7 +6201,7 @@ func TestReconciler_Client_Disconnect_Canaries(t *testing.T) { // Validate tc.canaryAllocs against tc.deploymentState must.Eq(t, tc.deploymentState.PlacedAllocs, canariesConfigured, must.Sprintf("invalid canary configuration: expect %d got %d", tc.deploymentState.PlacedAllocs, canariesConfigured)) - deployment := structs.NewDeployment(updatedJob, 50) + deployment := structs.NewDeployment(updatedJob, 50, time.Now().UnixNano()) deployment.TaskGroups[updatedJob.TaskGroups[0].Name] = tc.deploymentState // Build a map of tainted nodes that contains the last canary @@ -6331,7 +6351,7 @@ func TestReconciler_ComputeDeploymentPaused(t *testing.T) { // fetched by the scheduler before handing it to the // reconciler. if job.UsesDeployments() { - deployment = structs.NewDeployment(job, 100) + deployment = structs.NewDeployment(job, 100, time.Now().UnixNano()) deployment.Status = structs.DeploymentStatusInitializing deployment.StatusDescription = structs.DeploymentStatusDescriptionPendingForPeer } diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index c1fac53a6cd..8e008ee6687 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -13,6 +13,14 @@ upgrade. However, specific versions of Nomad may have more details provided for their upgrades as a result of new features or changed behavior. This page is used to document those details separately from the standard upgrade flow. +## Nomad 1.9.2 + +In Nomad 1.9.2, the mechanism used for calculating when objects are eligible +for garbage collection changes to a clock-based one. This has two consequences. +First, it allows to set arbitrarily long GC intervals. Second, it requires that +Nomad servers are kept roughly in sync time-wise, because GC can originate in a +follower. + ## Nomad 1.9.0 #### Dropped support for older clients