Skip to content

Commit

Permalink
resource_group: add retry configurations (tikv#8041)
Browse files Browse the repository at this point in the history
close tikv#8040

resource_group: add retry configurations

Signed-off-by: disksing <[email protected]>
  • Loading branch information
disksing authored Apr 10, 2024
1 parent fecffc3 commit b9240a0
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 11 deletions.
16 changes: 16 additions & 0 deletions client/resource_group/controller/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ const (
defaultTargetPeriod = 5 * time.Second
// defaultMaxWaitDuration is the max duration to wait for the token before throwing error.
defaultMaxWaitDuration = 30 * time.Second
// defaultWaitRetryTimes is the times to retry when waiting for the token.
defaultWaitRetryTimes = 10
// defaultWaitRetryInterval is the interval to retry when waiting for the token.
defaultWaitRetryInterval = 50 * time.Millisecond
)

const (
Expand Down Expand Up @@ -85,6 +89,12 @@ type Config struct {
// LTBMaxWaitDuration is the max wait time duration for local token bucket.
LTBMaxWaitDuration Duration `toml:"ltb-max-wait-duration" json:"ltb-max-wait-duration"`

// WaitRetryInterval is the interval to retry when waiting for the token.
WaitRetryInterval Duration `toml:"wait-retry-interval" json:"wait-retry-interval"`

// WaitRetryTimes is the times to retry when waiting for the token.
WaitRetryTimes int `toml:"wait-retry-times" json:"wait-retry-times"`

// RequestUnit is the configuration determines the coefficients of the RRU and WRU cost.
// This configuration should be modified carefully.
RequestUnit RequestUnitConfig `toml:"request-unit" json:"request-unit"`
Expand All @@ -98,6 +108,8 @@ func DefaultConfig() *Config {
return &Config{
DegradedModeWaitDuration: NewDuration(defaultDegradedModeWaitDuration),
LTBMaxWaitDuration: NewDuration(defaultMaxWaitDuration),
WaitRetryInterval: NewDuration(defaultWaitRetryInterval),
WaitRetryTimes: defaultWaitRetryTimes,
RequestUnit: DefaultRequestUnitConfig(),
EnableControllerTraceLog: false,
}
Expand Down Expand Up @@ -155,6 +167,8 @@ type RUConfig struct {

// some config for client
LTBMaxWaitDuration time.Duration
WaitRetryInterval time.Duration
WaitRetryTimes int
DegradedModeWaitDuration time.Duration
}

Expand All @@ -176,6 +190,8 @@ func GenerateRUConfig(config *Config) *RUConfig {
WriteBytesCost: RequestUnit(config.RequestUnit.WriteCostPerByte),
CPUMsCost: RequestUnit(config.RequestUnit.CPUMsCost),
LTBMaxWaitDuration: config.LTBMaxWaitDuration.Duration,
WaitRetryInterval: config.WaitRetryInterval.Duration,
WaitRetryTimes: config.WaitRetryTimes,
DegradedModeWaitDuration: config.DegradedModeWaitDuration.Duration,
}
}
26 changes: 19 additions & 7 deletions client/resource_group/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ import (

const (
controllerConfigPath = "resource_group/controller"
maxRetry = 10
retryInterval = 50 * time.Millisecond
maxNotificationChanLen = 200
needTokensAmplification = 1.1
trickleReserveDuration = 1250 * time.Millisecond
Expand Down Expand Up @@ -105,6 +103,20 @@ func WithMaxWaitDuration(d time.Duration) ResourceControlCreateOption {
}
}

// WithWaitRetryInterval is the option to set the retry interval when waiting for the token.
func WithWaitRetryInterval(d time.Duration) ResourceControlCreateOption {
return func(controller *ResourceGroupsController) {
controller.ruConfig.WaitRetryInterval = d
}
}

// WithWaitRetryTimes is the option to set the times to retry when waiting for the token.
func WithWaitRetryTimes(times int) ResourceControlCreateOption {
return func(controller *ResourceGroupsController) {
controller.ruConfig.WaitRetryTimes = times
}
}

var _ ResourceGroupKVInterceptor = (*ResourceGroupsController)(nil)

// ResourceGroupsController implements ResourceGroupKVInterceptor.
Expand Down Expand Up @@ -186,7 +198,7 @@ func loadServerConfig(ctx context.Context, provider ResourceGroupProvider) (*Con
log.Warn("[resource group controller] server does not save config, load config failed")
return DefaultConfig(), nil
}
config := &Config{}
config := DefaultConfig()
err = json.Unmarshal(kvs[0].GetValue(), config)
if err != nil {
return nil, err
Expand Down Expand Up @@ -367,7 +379,7 @@ func (c *ResourceGroupsController) Start(ctx context.Context) {
}
for _, item := range resp {
cfgRevision = item.Kv.ModRevision
config := &Config{}
config := DefaultConfig()
if err := json.Unmarshal(item.Kv.Value, config); err != nil {
continue
}
Expand Down Expand Up @@ -1206,7 +1218,7 @@ func (gc *groupCostController) onRequestWait(
var i int
var d time.Duration
retryLoop:
for i = 0; i < maxRetry; i++ {
for i = 0; i < gc.mainCfg.WaitRetryTimes; i++ {
switch gc.mode {
case rmpb.GroupMode_RawMode:
res := make([]*Reservation, 0, len(requestResourceLimitTypeList))
Expand All @@ -1230,8 +1242,8 @@ func (gc *groupCostController) onRequestWait(
}
}
gc.requestRetryCounter.Inc()
time.Sleep(retryInterval)
waitDuration += retryInterval
time.Sleep(gc.mainCfg.WaitRetryInterval)
waitDuration += gc.mainCfg.WaitRetryInterval
}
if err != nil {
gc.failedRequestCounter.Inc()
Expand Down
15 changes: 11 additions & 4 deletions tests/integrations/mcs/resourcemanager/resource_manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"github.com/tikv/pd/client/resource_group/controller"
"github.com/tikv/pd/pkg/mcs/resourcemanager/server"
"github.com/tikv/pd/pkg/utils/testutil"
"github.com/tikv/pd/pkg/utils/typeutil"
"github.com/tikv/pd/tests"
"go.uber.org/goleak"

Expand Down Expand Up @@ -1436,14 +1437,20 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupControllerConfigCh
waitDuration := 10 * time.Second
readBaseCost := 1.5
defaultCfg := controller.DefaultConfig()
// failpoint enableDegradedMode will setup and set it be 1s.
defaultCfg.DegradedModeWaitDuration.Duration = time.Second
expectCfg := server.ControllerConfig{
// failpoint enableDegradedMode will setup and set it be 1s.
DegradedModeWaitDuration: typeutil.NewDuration(time.Second),
LTBMaxWaitDuration: typeutil.Duration(defaultCfg.LTBMaxWaitDuration),
RequestUnit: server.RequestUnitConfig(defaultCfg.RequestUnit),
EnableControllerTraceLog: defaultCfg.EnableControllerTraceLog,
}
expectRUCfg := controller.GenerateRUConfig(defaultCfg)
expectRUCfg.DegradedModeWaitDuration = time.Second
// initial config verification
respString := sendRequest("GET", getAddr()+configURL, nil)
defaultString, err := json.Marshal(defaultCfg)
expectStr, err := json.Marshal(expectCfg)
re.NoError(err)
re.JSONEq(string(respString), string(defaultString))
re.JSONEq(string(respString), string(expectStr))
re.EqualValues(expectRUCfg, c1.GetConfig())

testCases := []struct {
Expand Down

0 comments on commit b9240a0

Please sign in to comment.