Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Configurable QoS resources for cluster-manager and klusterlet operators and their managed containers #400

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pkg/cmd/init/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
"If set, the command will initialize the OCM control plan in foreground.")
cmd.Flags().StringVarP(&o.output, "output", "o", "text", "output foramt, should be json or text")
cmd.Flags().BoolVar(&o.singleton, "singleton", false, "If true, deploy singleton controlplane instead of cluster-manager. This is an alpha stage flag.")
cmd.Flags().StringVar(&o.resourceQosClass, "resource-qos-class", "Default", "the resource QoS class of all the containers managed by the cluster manager and the cluster manager operator. Can be one of Default, BestEffort or ResourceRequirement.")
cmd.Flags().StringToStringVar(&o.resourceLimits, "resource-limits", nil, "the resource limits of all the containers managed by the cluster manager and the cluster manager operator, for example: cpu=800m,memory=800Mi")
cmd.Flags().StringToStringVar(&o.resourceRequests, "resource-requests", nil, "the resource requests of all the containers managed by the cluster manager and the cluster manager operator, for example: cpu=500m,memory=500Mi")
cmd.Flags().BoolVar(&o.createNamespace, "create-namespace", true, "If true, create open-cluster-management namespace, otherwise use existing one")

//clusterManagetSet contains the flags for deploy cluster-manager
clusterManagerSet := pflag.NewFlagSet("clusterManagerSet", pflag.ExitOnError)
Expand Down
14 changes: 12 additions & 2 deletions pkg/cmd/init/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
clusteradmjson "open-cluster-management.io/clusteradm/pkg/helpers/json"
preflightinterface "open-cluster-management.io/clusteradm/pkg/helpers/preflight"
"open-cluster-management.io/clusteradm/pkg/helpers/reader"
"open-cluster-management.io/clusteradm/pkg/helpers/resourcerequirement"
"open-cluster-management.io/clusteradm/pkg/helpers/version"
helperwait "open-cluster-management.io/clusteradm/pkg/helpers/wait"
)
Expand Down Expand Up @@ -67,6 +68,11 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
WorkFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubWorkFeatureGates),
AddonFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubAddonManagerFeatureGates),
}
resourceRequirement, err := resourcerequirement.NewResourceRequirement(o.resourceQosClass, o.resourceLimits, o.resourceRequests)
if err != nil {
return err
}
o.values.ResourceRequirement = *resourceRequirement
} else {
o.Helm.WithNamespace(o.SingletonName)
}
Expand Down Expand Up @@ -150,9 +156,13 @@ func (o *Options) run() error {
} else {
token := fmt.Sprintf("%s.%s", o.values.Hub.TokenID, o.values.Hub.TokenSecret)

files := []string{
"init/namespace.yaml",
files := []string{}
if o.createNamespace {
files = append(files, "init/namespace.yaml")
} else {
fmt.Fprintf(o.Streams.Out, "skip creating namespace\n")
}

if o.useBootstrapToken {
files = append(files,
"init/bootstrap-token-secret.yaml",
Expand Down
13 changes: 13 additions & 0 deletions pkg/cmd/init/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
operatorv1 "open-cluster-management.io/api/operator/v1"
genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
"open-cluster-management.io/clusteradm/pkg/helpers/helm"
"open-cluster-management.io/clusteradm/pkg/helpers/resourcerequirement"
)

// Options is holding all the command-line options
Expand All @@ -29,6 +30,14 @@ type Options struct {
SingletonName string
Helm *helm.Helm

// Resource requirement for the containers managed by the cluster manager and the cluster manager operator
resourceQosClass string
resourceLimits map[string]string
resourceRequests map[string]string

// If create ns or use existing ns
createNamespace bool

//If set, will be persisting the generated join command to a local file
outputJoinCommandFile string
//If set, the command will hold until the OCM control plane initialized
Expand Down Expand Up @@ -70,6 +79,10 @@ type Values struct {

// Features is the slice of feature for addon manager
AddonFeatures []operatorv1.FeatureGate

// ResourceRequirement is the resource requirement setting for the containers managed by the cluster manager
// and the cluster manager operator
ResourceRequirement resourcerequirement.ResourceRequirement
}

// Hub: The hub values for the template
Expand Down
6 changes: 6 additions & 0 deletions pkg/cmd/init/scenario/init/clustermanager.cr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ spec:
deployOption:
mode: Default
{{if .RegistrationFeatures}}
resourceRequirement:
type: {{ .ResourceRequirement.Type }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resourceRequirements:
{{ .ResourceRequirement.ResourceRequirements | indent 6 }}
{{- end }}
registrationConfiguration:
{{if .AutoApprove}}
autoApproveUsers:
Expand Down
15 changes: 15 additions & 0 deletions pkg/cmd/init/scenario/init/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ spec:
labels:
app: cluster-manager
spec:
volumes:
- emptyDir: {}
name: tmpdir
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
Expand Down Expand Up @@ -56,16 +59,28 @@ spec:
initialDelaySeconds: 2
periodSeconds: 10
name: registration-operator
volumeMounts:
- mountPath: /tmp
name: tmpdir
readinessProbe:
httpGet:
path: /healthz
port: 8443
scheme: HTTPS
initialDelaySeconds: 2
{{- if or (eq .ResourceRequirement.Type "Default") (eq .ResourceRequirement.Type "") }}
resources:
requests:
cpu: 100m
memory: 128Mi
{{- end }}
{{- if eq .ResourceRequirement.Type "BestEffort" }}
resources: {}
{{- end }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resources:
{{ .ResourceRequirement.ResourceRequirements | indent 10 }}
{{- end }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
Expand Down
4 changes: 3 additions & 1 deletion pkg/cmd/join/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,9 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
cmd.Flags().BoolVar(&o.singleton, "singleton", false, "If true, deploy singleton mode of klusterlet to have registration and work agents run in a single pod. This is an alpha stage flag.")
cmd.Flags().StringVar(&o.proxyURL, "proxy-url", "", "the URL of a forward proxy server that will be used by agents to connect to the hub cluster.")
cmd.Flags().StringVar(&o.proxyCAFile, "proxy-ca-file", "", "the file path to proxy ca, optional")
cmd.Flags().StringVar(&o.resourceQosClass, "resource-qos-class", "Default", "the resource QoS class of the klusterlet pod. Can be one of Default or BestEffort")
cmd.Flags().StringVar(&o.resourceQosClass, "resource-qos-class", "Default", "the resource QoS class of all the containers managed by the klusterlet and the klusterlet operator. Can be one of Default, BestEffort or ResourceRequirement.")
cmd.Flags().StringToStringVar(&o.resourceLimits, "resource-limits", nil, "the resource limits of all the containers managed by the klusterlet and the klusterlet operator, for example: cpu=800m,memory=800Mi")
cmd.Flags().StringToStringVar(&o.resourceRequests, "resource-requests", nil, "the resource requests of all the containers managed by the klusterlet and the klusterlet operator, for example: cpu=500m,memory=500Mi")
cmd.Flags().BoolVar(&o.createNameSpace, "create-namespace", true, "If true, create open-cluster-management namespace, otherwise use existing one")

return cmd
Expand Down
9 changes: 7 additions & 2 deletions pkg/cmd/join/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
preflightinterface "open-cluster-management.io/clusteradm/pkg/helpers/preflight"
"open-cluster-management.io/clusteradm/pkg/helpers/printer"
"open-cluster-management.io/clusteradm/pkg/helpers/reader"
"open-cluster-management.io/clusteradm/pkg/helpers/resourcerequirement"
"open-cluster-management.io/clusteradm/pkg/helpers/version"
"open-cluster-management.io/clusteradm/pkg/helpers/wait"
)
Expand Down Expand Up @@ -119,9 +120,13 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
Name: klusterletName,
KlusterletNamespace: klusterletNamespace,
}
o.values.ResourceRequirement = ResourceRequirement{
Type: o.resourceQosClass,

resourceRequirement, err := resourcerequirement.NewResourceRequirement(o.resourceQosClass, o.resourceLimits, o.resourceRequests)
if err != nil {
return err
}
o.values.ResourceRequirement = *resourceRequirement

o.values.ManagedKubeconfig = o.managedKubeconfigFile
o.values.RegistrationFeatures = genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeRegistrationFeatureGates)
o.values.WorkFeatures = genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeWorkFeatureGates)
Expand Down
16 changes: 8 additions & 8 deletions pkg/cmd/join/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1"
operatorv1 "open-cluster-management.io/api/operator/v1"
genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
"open-cluster-management.io/clusteradm/pkg/helpers/resourcerequirement"
)

// Options: The structure holding all the command-line options
Expand Down Expand Up @@ -62,8 +63,10 @@ type Options struct {
//The proxy server ca-file(optional)
proxyCAFile string

// Resource requirement
// Resource requirement for the containers managed by klusterlet and the klusterlet operator
resourceQosClass string
resourceLimits map[string]string
resourceRequests map[string]string

// If create ns or use existing ns
createNameSpace bool
Expand All @@ -84,8 +87,6 @@ type Values struct {
Hub Hub
//Klusterlet is the klusterlet related configuration
Klusterlet Klusterlet
//ResourceRequirement is the resource requirement
ResourceRequirement ResourceRequirement
//Registry is the image registry related configuration
Registry string
//bundle version
Expand All @@ -98,6 +99,10 @@ type Values struct {

// Features is the slice of feature for work
WorkFeatures []operatorv1.FeatureGate

// ResourceRequirement is the resource requirement setting for the containers managed by the klusterlet
// and the klusterlet operator
ResourceRequirement resourcerequirement.ResourceRequirement
}

// Hub: The hub values for the template
Expand All @@ -117,11 +122,6 @@ type Klusterlet struct {
KlusterletNamespace string
}

// ResourceRequirement is for templating resource requirement
type ResourceRequirement struct {
Type string
}

type BundleVersion struct {
// registration image version
RegistrationImageVersion string
Expand Down
4 changes: 4 additions & 0 deletions pkg/cmd/join/scenario/join/klusterlets.cr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ spec:
{{ end }}
resourceRequirement:
type: {{ .ResourceRequirement.Type }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resourceRequirements:
{{ .ResourceRequirement.ResourceRequirements | indent 6 }}
{{- end }}
{{if .RegistrationFeatures}}
registrationConfiguration:
featureGates:
Expand Down
11 changes: 9 additions & 2 deletions pkg/cmd/join/scenario/join/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,19 @@ spec:
scheme: HTTPS
port: 8443
initialDelaySeconds: 2
{{ if ne .ResourceRequirement.Type "BestEffort" }}
{{- if or (eq .ResourceRequirement.Type "Default") (eq .ResourceRequirement.Type "") }}
resources:
requests:
cpu: 100m
memory: 128Mi
{{ end }}
{{- end }}
{{- if eq .ResourceRequirement.Type "BestEffort" }}
resources: {}
{{- end }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resources:
{{ .ResourceRequirement.ResourceRequirements | indent 10 }}
{{- end }}
volumeMounts:
- name: tmpdir
mountPath: /tmp
Expand Down
34 changes: 34 additions & 0 deletions pkg/helpers/resourcerequirement/fake_deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright Contributors to the Open Cluster Management project
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
a: b
name: fake-deployment
namespace: fake-namespace
spec:
replicas: 1
selector:
matchLabels:
a: b
template:
metadata:
labels:
a: b
spec:
containers:
- name: my-container
image: nginx
{{- if or (eq .ResourceRequirement.Type "Default") (eq .ResourceRequirement.Type "") }}
resources:
requests:
cpu: 100m
memory: 128Mi
{{- end }}
{{- if eq .ResourceRequirement.Type "BestEffort" }}
resources: {}
{{- end }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resources:
{{ .ResourceRequirement.ResourceRequirements | indent 10 }}
{{- end }}
78 changes: 78 additions & 0 deletions pkg/helpers/resourcerequirement/resourcerequirement.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright Contributors to the Open Cluster Management project

package resourcerequirement

import (
"fmt"

"github.com/ghodss/yaml"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
operatorv1 "open-cluster-management.io/api/operator/v1"
)

// ResourceRequirement is for templating resource requirement
type ResourceRequirement struct {
Type string
ResourceRequirements []byte
}

func NewResourceRequirement(resourceType string, limits, requests map[string]string) (*ResourceRequirement, error) {
if len(limits)+len(requests) == 0 {
if resourceType == string(operatorv1.ResourceQosClassResourceRequirement) {
return nil, fmt.Errorf("resource type is %s but both limits and requests are not set", resourceType)
}
return &ResourceRequirement{
Type: resourceType,
}, nil
}
if resourceType == "" {
resourceType = string(operatorv1.ResourceQosClassResourceRequirement)
} else if resourceType != string(operatorv1.ResourceQosClassResourceRequirement) {
return nil, fmt.Errorf("resource type must be %s when resource limits or requests are set", string(operatorv1.ResourceQosClassResourceRequirement))
}
rr := &corev1.ResourceRequirements{
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
}
for rsc, quantityStr := range limits {
quantity, err := resource.ParseQuantity(quantityStr)
if err != nil {
return nil, err
}
rr.Limits[corev1.ResourceName(rsc)] = quantity
}
for rsc, quantityStr := range requests {
quantity, err := resource.ParseQuantity(quantityStr)
if err != nil {
return nil, err
}
rr.Requests[corev1.ResourceName(rsc)] = quantity
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not sure if it is necessary to add some checks to ensure that the requirements <= limits?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure, updated

if err := ensureQuantity(rr); err != nil {
return nil, err
}
marshal, err := yaml.Marshal(rr)
if err != nil {
return nil, err
}
return &ResourceRequirement{
Type: resourceType,
ResourceRequirements: marshal,
}, nil
}

func ensureQuantity(r *corev1.ResourceRequirements) error {
for rsc, limitsQuantity := range r.Limits {
requestsQuantity, ok := r.Requests[rsc]
if !ok {
continue
}
if requestsQuantity.Cmp(limitsQuantity) <= 0 {
continue
}
return fmt.Errorf("requests %s must be less than or equal to limits %s",
requestsQuantity.String(), limitsQuantity.String())
}
return nil
}
Loading
Loading