Skip to content

Commit

Permalink
schedule: fix datarace in operator.check (#8264) (#8578)
Browse files Browse the repository at this point in the history
close #8263

Signed-off-by: husharp <[email protected]>

Co-authored-by: husharp <[email protected]>
Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
  • Loading branch information
3 people authored Sep 10, 2024
1 parent 002b454 commit 9fb2837
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 2 deletions.
5 changes: 3 additions & 2 deletions pkg/schedule/operator/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,10 +311,11 @@ func (o *Operator) Check(region *core.RegionInfo) OpStep {
defer func() { _ = o.CheckTimeout() }()
for step := atomic.LoadInt32(&o.currentStep); int(step) < len(o.steps); step++ {
if o.steps[int(step)].IsFinish(region) {
if atomic.CompareAndSwapInt64(&(o.stepsTime[step]), 0, time.Now().UnixNano()) {
current := time.Now()
if atomic.CompareAndSwapInt64(&(o.stepsTime[step]), 0, current.UnixNano()) {
startTime, _ := o.getCurrentTimeAndStep()
operatorStepDuration.WithLabelValues(reflect.TypeOf(o.steps[int(step)]).Name()).
Observe(time.Unix(0, o.stepsTime[step]).Sub(startTime).Seconds())
Observe(current.Sub(startTime).Seconds())
}
atomic.StoreInt32(&o.currentStep, step+1)
} else {
Expand Down
24 changes: 24 additions & 0 deletions pkg/schedule/operator/operator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
Expand Down Expand Up @@ -529,3 +530,26 @@ func (suite *operatorTestSuite) TestRecord() {
suite.Equal(now, ob.FinishTime)
suite.Greater(ob.duration.Seconds(), time.Second.Seconds())
}

func (suite *operatorTestSuite) TestOperatorCheckConcurrently() {
region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2})
// addPeer1, transferLeader1, removePeer3
steps := []OpStep{
AddPeer{ToStore: 1, PeerID: 1},
TransferLeader{FromStore: 3, ToStore: 1},
RemovePeer{FromStore: 3},
}
op := NewTestOperator(1, &metapb.RegionEpoch{}, OpAdmin|OpLeader|OpRegion, steps...)
suite.Equal(constant.Urgent, op.GetPriorityLevel())
suite.checkSteps(op, steps)
op.Start()
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
suite.Nil(op.Check(region))
}()
}
wg.Wait()
}

0 comments on commit 9fb2837

Please sign in to comment.