From e20e17c7a968baf1dd1347a0f5465c21de08f14b Mon Sep 17 00:00:00 2001 From: Ted Date: Sat, 22 Jul 2023 09:00:18 -0400 Subject: [PATCH] Add housekeeping project fetch size option to the CLI flag * set project fetch size to 100 in config.sample.yml fix lint --- cmd/yorkie/server.go | 6 ++ server/backend/housekeeping/config.go | 61 +++++++++++++++++++++ server/backend/housekeeping/config_test.go | 48 ++++++++++++++++ server/backend/housekeeping/housekeeping.go | 25 --------- server/config.sample.yml | 4 +- test/helper/helper.go | 6 +- 6 files changed, 121 insertions(+), 29 deletions(-) create mode 100644 server/backend/housekeeping/config.go create mode 100644 server/backend/housekeeping/config_test.go diff --git a/cmd/yorkie/server.go b/cmd/yorkie/server.go index 583188698..af94648a4 100644 --- a/cmd/yorkie/server.go +++ b/cmd/yorkie/server.go @@ -227,6 +227,12 @@ func init() { server.DefaultHousekeepingCandidatesLimitPerProject, "candidates limit per project for a single housekeeping run", ) + cmd.Flags().IntVar( + &conf.Housekeeping.HousekeepingProjectFetchSize, + "housekeeping-project-fetch-size", + server.DefaultHousekeepingProjectFetchSize, + "housekeeping project fetch size for a single housekeeping run", + ) cmd.Flags().StringVar( &mongoConnectionURI, "mongo-connection-uri", diff --git a/server/backend/housekeeping/config.go b/server/backend/housekeeping/config.go new file mode 100644 index 000000000..091ae5e5c --- /dev/null +++ b/server/backend/housekeeping/config.go @@ -0,0 +1,61 @@ +/* + * Copyright 2023 The Yorkie Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package housekeeping + +import ( + "fmt" + "time" +) + +// Config is the configuration for the housekeeping service. +type Config struct { + // Interval is the time between housekeeping runs. + Interval string `yaml:"Interval"` + + // CandidatesLimitPerProject is the maximum number of candidates to be returned per project. + CandidatesLimitPerProject int `yaml:"CandidatesLimitPerProject"` + + // HousekeepingProjectFetchSize is the maximum number of projects to be returned to deactivate candidates. + HousekeepingProjectFetchSize int `yaml:"HousekeepingProjectFetchSize"` +} + +// Validate validates the configuration. +func (c *Config) Validate() error { + if _, err := time.ParseDuration(c.Interval); err != nil { + return fmt.Errorf( + `invalid argument %s for "--housekeeping-interval" flag: %w`, + c.Interval, + err, + ) + } + + if c.CandidatesLimitPerProject <= 0 { + return fmt.Errorf( + `invalid argument %d for "--housekeeping-candidates-limit-per-project" flag`, + c.HousekeepingProjectFetchSize, + ) + } + + if c.HousekeepingProjectFetchSize <= 0 { + return fmt.Errorf( + `invalid argument %d for "--housekeeping-project-fetc-size" flag`, + c.HousekeepingProjectFetchSize, + ) + } + + return nil +} diff --git a/server/backend/housekeeping/config_test.go b/server/backend/housekeeping/config_test.go new file mode 100644 index 000000000..a67cd0b59 --- /dev/null +++ b/server/backend/housekeeping/config_test.go @@ -0,0 +1,48 @@ +/* + * Copyright 2023 The Yorkie Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package housekeeping_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/yorkie-team/yorkie/server/backend/housekeeping" +) + +func TestConfig(t *testing.T) { + t.Run("validate test", func(t *testing.T) { + validConf := housekeeping.Config{ + Interval: "1m", + CandidatesLimitPerProject: 100, + HousekeepingProjectFetchSize: 100, + } + assert.NoError(t, validConf.Validate()) + + conf1 := validConf + conf1.Interval = "hour" + assert.Error(t, conf1.Validate()) + + conf2 := validConf + conf2.CandidatesLimitPerProject = 0 + assert.Error(t, conf2.Validate()) + + conf3 := validConf + conf3.HousekeepingProjectFetchSize = -1 + assert.Error(t, conf3.Validate()) + }) +} diff --git a/server/backend/housekeeping/housekeeping.go b/server/backend/housekeeping/housekeeping.go index 792020fde..fd394a01c 100644 --- a/server/backend/housekeeping/housekeeping.go +++ b/server/backend/housekeeping/housekeeping.go @@ -34,31 +34,6 @@ const ( deactivateCandidatesKey = "housekeeping/deactivateCandidates" ) -// Config is the configuration for the housekeeping service. -type Config struct { - // Interval is the time between housekeeping runs. - Interval string `yaml:"Interval"` - - // CandidatesLimitPerProject is the maximum number of candidates to be returned per project. - CandidatesLimitPerProject int `yaml:"CandidatesLimitPerProject"` - - // HousekeepingProjectFetchSize is the maximum number of projects to be returned to deactivate candidates. - HousekeepingProjectFetchSize int `yaml:"HousekeepingProjectFetchSize"` -} - -// Validate validates the configuration. -func (c *Config) Validate() error { - if _, err := time.ParseDuration(c.Interval); err != nil { - return fmt.Errorf( - `invalid argument %s for "--housekeeping-interval" flag: %w`, - c.Interval, - err, - ) - } - - return nil -} - // Housekeeping is the housekeeping service. It periodically runs housekeeping // tasks. It is responsible for deactivating clients that have not been active // for a long time. diff --git a/server/config.sample.yml b/server/config.sample.yml index 3f9c0e5d5..cee9cfca5 100644 --- a/server/config.sample.yml +++ b/server/config.sample.yml @@ -36,8 +36,8 @@ Housekeeping: # CandidatesLimitPerProject is the maximum number of candidates to be returned per project (default: 100). CandidatesLimitPerProject: 100 - # HousekeepingProjectFetchSize is the maximum number of projects to be returned to deactivate candidates. (default: 100) - HousekeepingProjectFetchSize: 10 + # HousekeepingProjectFetchSize is the maximum number of projects to be returned to deactivate candidates. (default: 100). + HousekeepingProjectFetchSize: 100 # Backend is the configuration for the backend of Yorkie. Backend: diff --git a/test/helper/helper.go b/test/helper/helper.go index ba6c2da1f..7a331e40c 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -60,6 +60,7 @@ var ( AdminPassword = server.DefaultAdminPassword HousekeepingInterval = 10 * gotime.Second HousekeepingCandidatesLimitPerProject = 10 + HousekeepingProjectFetchSize = 10 AdminTokenDuration = "10s" ClientDeactivateThreshold = "10s" @@ -221,8 +222,9 @@ func TestConfig() *server.Config { Port: ProfilingPort + portOffset, }, Housekeeping: &housekeeping.Config{ - Interval: HousekeepingInterval.String(), - CandidatesLimitPerProject: HousekeepingCandidatesLimitPerProject, + Interval: HousekeepingInterval.String(), + CandidatesLimitPerProject: HousekeepingCandidatesLimitPerProject, + HousekeepingProjectFetchSize: HousekeepingProjectFetchSize, }, Backend: &backend.Config{ AdminUser: server.DefaultAdminUser,