Skip to content

Commit

Permalink
chore: use custom dbmanager if not set (#5843)
Browse files Browse the repository at this point in the history
  • Loading branch information
xuriwuyun authored Nov 20, 2023
1 parent 286eb56 commit ad90544
Show file tree
Hide file tree
Showing 11 changed files with 581 additions and 18 deletions.
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ require (
github.com/cenkalti/backoff/v4 v4.2.1
github.com/clbanning/mxj/v2 v2.5.7
github.com/containers/common v0.55.4
github.com/dapr/kit v0.11.3
github.com/deckarep/golang-set/v2 v2.3.1
github.com/dlclark/regexp2 v1.10.0
github.com/docker/docker v24.0.7+incompatible
Expand Down
5 changes: 1 addition & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,8 @@ github.com/authzed/controller-idioms v0.7.0 h1:HhNMUBb8hJzYqY3mhen3B2AC5nsIem3fB
github.com/authzed/controller-idioms v0.7.0/go.mod h1:0B/PmqCguKv8b3azSMF+HdyKpKr2o3UAZ5eo12Ze8Fo=
github.com/aws/aws-sdk-go v1.44.257 h1:HwelXYZZ8c34uFFhgVw3ybu2gB5fkk8KLj2idTvzZb8=
github.com/aws/aws-sdk-go v1.44.257/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -237,8 +236,6 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
github.com/dapr/kit v0.11.3 h1:u1X92tE8xsrwXIej7nkcI5Z1t1CFznPwlL18tizNEw4=
github.com/dapr/kit v0.11.3/go.mod h1:hQA6xOhcLAiccXTj7e3/bzpHwvAJCSCp70p2xg3jB40=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down
1 change: 1 addition & 0 deletions pkg/lorry/engines/models/engine_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,5 @@ const (
PulsarProxy EngineType = "pulsar-proxy"
FoxLake EngineType = "foxlake"
Oceanbase EngineType = "oceanbase"
Custom EngineType = "custom"
)
7 changes: 3 additions & 4 deletions pkg/lorry/engines/mongodb/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/spf13/viper"

"github.com/apecloud/kubeblocks/pkg/constant"
utilconfig "github.com/apecloud/kubeblocks/pkg/lorry/util/config"
)

const (
Expand Down Expand Up @@ -131,10 +132,8 @@ func (config *Config) GetDBPort() int {
}

func (config *Config) DeepCopy() *Config {
newConf := *config
newConf.hosts = make([]string, len(config.hosts))
copy(newConf.hosts, config.hosts)
return &newConf
newConf, _ := utilconfig.Clone(config)
return newConf.(*Config)
}

func GetConfig() *Config {
Expand Down
2 changes: 1 addition & 1 deletion pkg/lorry/engines/redis/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"strconv"
"time"

"github.com/dapr/kit/config"
"github.com/apecloud/kubeblocks/pkg/lorry/util/config"
)

type Settings struct {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ func TestParseRedisMetadata(t *testing.T) {
err := m.Decode(fakeProperties)

// assert
// m.Decode dose not return error when host is ""
assert.Error(t, errors.New("redis streams error: missing host address"), err)
assert.Empty(t, m.Host)
})
Expand Down
16 changes: 10 additions & 6 deletions pkg/lorry/engines/register/managers.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (

"github.com/apecloud/kubeblocks/pkg/constant"
"github.com/apecloud/kubeblocks/pkg/lorry/engines"
"github.com/apecloud/kubeblocks/pkg/lorry/engines/custom"
"github.com/apecloud/kubeblocks/pkg/lorry/engines/etcd"
"github.com/apecloud/kubeblocks/pkg/lorry/engines/foxlake"
"github.com/apecloud/kubeblocks/pkg/lorry/engines/models"
Expand Down Expand Up @@ -81,6 +82,7 @@ func init() {
RegisterEngine(models.PostgreSQL, "", officalpostgres.NewManager, postgres.NewCommands)
RegisterEngine(models.OfficialPostgreSQL, "", officalpostgres.NewManager, postgres.NewCommands)
RegisterEngine(models.ApecloudPostgreSQL, "", apecloudpostgres.NewManager, postgres.NewCommands)
RegisterEngine(models.Custom, "", custom.NewManager, nil)
}

func RegisterEngine(characterType models.EngineType, workloadType string, newFunc managerNewFunc, newCommand engines.NewCommandFunc) {
Expand Down Expand Up @@ -121,17 +123,19 @@ func InitDBManager(configDir string) error {
}

ctrl.Log.Info("Initialize DB manager")
workloadType := viper.GetString(constant.KBEnvWorkloadType)
if workloadType == "" {
ctrl.Log.Info(constant.KBEnvWorkloadType + " ENV not set")
}

characterType := viper.GetString(constant.KBEnvCharacterType)
if viper.IsSet(constant.KBEnvBuiltinHandler) {
workloadType = ""
characterType = viper.GetString(constant.KBEnvBuiltinHandler)
}
if characterType == "" {
return fmt.Errorf("%s not set", constant.KBEnvCharacterType)
}

workloadType := viper.GetString(constant.KBEnvWorkloadType)
if workloadType == "" {
ctrl.Log.Info(constant.KBEnvWorkloadType + " ENV not set")
ctrl.Log.Info("BuiltinHandler not set, use the custom manager")
characterType = string(models.Custom)
}

err := GetAllComponent(configDir) // find all builtin config file and read
Expand Down
4 changes: 2 additions & 2 deletions pkg/lorry/engines/register/managers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,10 @@ func TestInitDBManager(t *testing.T) {
err := InitDBManager(configDir)

assert.NotNil(t, err)
assert.ErrorContains(t, err, "KB_SERVICE_CHARACTER_TYPE not set")
// assert.ErrorContains(t, err, "KB_SERVICE_CHARACTER_TYPE not set")
_, err = GetDBManager()
assert.NotNil(t, err)
assert.ErrorContains(t, err, "no db manager")
// assert.ErrorContains(t, err, "no db manager")
})

viper.Set(constant.KBEnvBuiltinHandler, fakeCharacterType)
Expand Down
192 changes: 192 additions & 0 deletions pkg/lorry/util/config/decode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
/*
Copyright 2021 The Dapr Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
"fmt"
"reflect"
"strconv"
"time"

"github.com/mitchellh/mapstructure"
)

var (
typeDuration = reflect.TypeOf(time.Duration(5)) //nolint: gochecknoglobals
typeTime = reflect.TypeOf(time.Time{}) //nolint: gochecknoglobals
typeStringDecoder = reflect.TypeOf((*StringDecoder)(nil)).Elem() //nolint: gochecknoglobals
)

// StringDecoder is used as a way for custom types (or alias types) to
// override the basic decoding function in the `decodeString`
// DecodeHook. `encoding.TextMashaller` was not used because it
// matches many Go types and would have potentially unexpected results.
// Specifying a custom decoding func should be very intentional.
type StringDecoder interface {
DecodeString(value string) error
}

// Decode decodes generic map values from `input` to `output`, while providing helpful error information.
// `output` must be a pointer to a Go struct that contains `mapstructure` struct tags on fields that should
// be decoded. This function is useful when decoding values from configuration files parsed as
// `map[string]interface{}` or component metadata as `map[string]string`.
//
// Most of the heavy lifting is handled by the mapstructure library. A custom decoder is used to handle
// decoding string values to the supported primitives.
func Decode(input interface{}, output interface{}) error {
decoder, err := mapstructure.NewDecoder(
&mapstructure.DecoderConfig{ //nolint: exhaustruct
Result: output,
DecodeHook: decodeString,
})
if err != nil {
return err
}

return decoder.Decode(input)
}

//nolint:cyclop
func decodeString(f reflect.Type, t reflect.Type, data any) (any, error) {
if t.Kind() == reflect.String && f.Kind() != reflect.String {
return fmt.Sprintf("%v", data), nil
}
if f.Kind() == reflect.Ptr {
f = f.Elem()
data = reflect.ValueOf(data).Elem().Interface()
}
if f.Kind() != reflect.String {
return data, nil
}

dataString, ok := data.(string)
if !ok {
return nil, fmt.Errorf("expected string: got %T", data)
}

var result any
var decoder StringDecoder

if t.Implements(typeStringDecoder) {
result = reflect.New(t.Elem()).Interface()
decoder = result.(StringDecoder)
} else if reflect.PtrTo(t).Implements(typeStringDecoder) {
result = reflect.New(t).Interface()
decoder = result.(StringDecoder)
}

if decoder != nil {
if err := decoder.DecodeString(dataString); err != nil {
if t.Kind() == reflect.Ptr {
t = t.Elem()
}

return nil, fmt.Errorf("invalid %s %q: %v", t.Name(), dataString, err)
}

return result, nil
}

switch t {
case typeDuration:
// Check for simple integer values and treat them
// as milliseconds
if val, err := strconv.Atoi(dataString); err == nil {
return time.Duration(val) * time.Millisecond, nil
}

// Convert it by parsing
d, err := time.ParseDuration(dataString)

return d, invalidError(err, "duration", dataString)
case typeTime:
// Convert it by parsing
t, err := time.Parse(time.RFC3339Nano, dataString)
if err == nil {
return t, nil
}
t, err = time.Parse(time.RFC3339, dataString)

return t, invalidError(err, "time", dataString)
}

switch t.Kind() {
case reflect.Uint:
val, err := strconv.ParseUint(dataString, 10, 64)

return uint(val), invalidError(err, "uint", dataString)
case reflect.Uint64:
val, err := strconv.ParseUint(dataString, 10, 64)

return val, invalidError(err, "uint64", dataString)
case reflect.Uint32:
val, err := strconv.ParseUint(dataString, 10, 32)

return uint32(val), invalidError(err, "uint32", dataString)
case reflect.Uint16:
val, err := strconv.ParseUint(dataString, 10, 16)

return uint16(val), invalidError(err, "uint16", dataString)
case reflect.Uint8:
val, err := strconv.ParseUint(dataString, 10, 8)

return uint8(val), invalidError(err, "uint8", dataString)

case reflect.Int:
val, err := strconv.ParseInt(dataString, 10, 64)

return int(val), invalidError(err, "int", dataString)
case reflect.Int64:
val, err := strconv.ParseInt(dataString, 10, 64)

return val, invalidError(err, "int64", dataString)
case reflect.Int32:
val, err := strconv.ParseInt(dataString, 10, 32)

return int32(val), invalidError(err, "int32", dataString)
case reflect.Int16:
val, err := strconv.ParseInt(dataString, 10, 16)

return int16(val), invalidError(err, "int16", dataString)
case reflect.Int8:
val, err := strconv.ParseInt(dataString, 10, 8)

return int8(val), invalidError(err, "int8", dataString)

case reflect.Float32:
val, err := strconv.ParseFloat(dataString, 32)

return float32(val), invalidError(err, "float32", dataString)
case reflect.Float64:
val, err := strconv.ParseFloat(dataString, 64)

return val, invalidError(err, "float64", dataString)

case reflect.Bool:
val, err := strconv.ParseBool(dataString)

return val, invalidError(err, "bool", dataString)

default:
return data, nil
}
}

func invalidError(err error, msg, value string) error {
if err == nil {
return nil
}

return fmt.Errorf("invalid %s %q", msg, value)
}
Loading

0 comments on commit ad90544

Please sign in to comment.