From c265e414bb36cb79fcdf90db00dc0614937ed87f Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Thu, 14 Nov 2024 10:44:40 +0100 Subject: [PATCH 1/2] add a hostNetwork setting to the Driver and ControllerPlugin Specs This bool setting can be added to the controllerPlugin section of the Driver Spec. It will be propagated to all controller plugin pods. This implements the following design: https://github.com/ceph/ceph-csi-operator/blob/main/docs/design/hostNetwork.md Signed-off-by: Michael Adam --- api/v1alpha1/driver_types.go | 2 ++ api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ config/crd/bases/csi.ceph.io_drivers.yaml | 4 ++++ config/crd/bases/csi.ceph.io_operatorconfigs.yaml | 4 ++++ deploy/all-in-one/install.yaml | 8 ++++++++ deploy/multifile/crd.yaml | 8 ++++++++ internal/controller/driver_controller.go | 1 + .../ceph/ceph-csi-operator/api/v1alpha1/driver_types.go | 3 +++ .../api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 9 files changed, 40 insertions(+) diff --git a/api/v1alpha1/driver_types.go b/api/v1alpha1/driver_types.go index b562eeaa..24052f06 100644 --- a/api/v1alpha1/driver_types.go +++ b/api/v1alpha1/driver_types.go @@ -210,6 +210,8 @@ type ControllerPluginResourcesSpec struct { } type ControllerPluginSpec struct { + // hostNetwork setting to be propagated to CSI controller plugin pods + HostNetwork *bool `json:"hostNetwork,omitempty"` // Embedded common pods spec PodCommonSpec `json:",inline"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a8542df9..bb617807 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -437,6 +437,11 @@ func (in *ControllerPluginResourcesSpec) DeepCopy() *ControllerPluginResourcesSp // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControllerPluginSpec) DeepCopyInto(out *ControllerPluginSpec) { *out = *in + if in.HostNetwork != nil { + in, out := &in.HostNetwork, &out.HostNetwork + *out = new(bool) + **out = **in + } in.PodCommonSpec.DeepCopyInto(&out.PodCommonSpec) if in.DeploymentStrategy != nil { in, out := &in.DeploymentStrategy, &out.DeploymentStrategy diff --git a/config/crd/bases/csi.ceph.io_drivers.yaml b/config/crd/bases/csi.ceph.io_drivers.yaml index 7fe11209..6816ab66 100644 --- a/config/crd/bases/csi.ceph.io_drivers.yaml +++ b/config/crd/bases/csi.ceph.io_drivers.yaml @@ -1051,6 +1051,10 @@ spec: Default is RollingUpdate. type: string type: object + hostNetwork: + description: hostNetwork setting to be propagated to CSI controller + plugin pods + type: boolean imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. diff --git a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml index 9e4884a3..fde8ebcc 100644 --- a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml +++ b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml @@ -1060,6 +1060,10 @@ spec: "RollingUpdate". Default is RollingUpdate. type: string type: object + hostNetwork: + description: hostNetwork setting to be propagated to CSI controller + plugin pods + type: boolean imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. diff --git a/deploy/all-in-one/install.yaml b/deploy/all-in-one/install.yaml index 8b6c7744..df8beea8 100644 --- a/deploy/all-in-one/install.yaml +++ b/deploy/all-in-one/install.yaml @@ -1301,6 +1301,10 @@ spec: Default is RollingUpdate. type: string type: object + hostNetwork: + description: hostNetwork setting to be propagated to CSI controller + plugin pods + type: boolean imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. @@ -8161,6 +8165,10 @@ spec: "RollingUpdate". Default is RollingUpdate. type: string type: object + hostNetwork: + description: hostNetwork setting to be propagated to CSI controller + plugin pods + type: boolean imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. diff --git a/deploy/multifile/crd.yaml b/deploy/multifile/crd.yaml index 09b88da6..41f87fd8 100644 --- a/deploy/multifile/crd.yaml +++ b/deploy/multifile/crd.yaml @@ -1292,6 +1292,10 @@ spec: Default is RollingUpdate. type: string type: object + hostNetwork: + description: hostNetwork setting to be propagated to CSI controller + plugin pods + type: boolean imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. @@ -8152,6 +8156,10 @@ spec: "RollingUpdate". Default is RollingUpdate. type: string type: object + hostNetwork: + description: hostNetwork setting to be propagated to CSI controller + plugin pods + type: boolean imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index e5a1ac1e..509ee733 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -575,6 +575,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Spec: corev1.PodSpec{ ServiceAccountName: serviceAccountName, PriorityClassName: ptr.Deref(pluginSpec.PrioritylClassName, ""), + HostNetwork: ptr.Deref(pluginSpec.HostNetwork, false), Affinity: getControllerPluginPodAffinity(pluginSpec, &appSelector), Tolerations: pluginSpec.Tolerations, Containers: utils.Call(func() []corev1.Container { diff --git a/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/driver_types.go b/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/driver_types.go index b562eeaa..af2c0607 100644 --- a/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/driver_types.go +++ b/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/driver_types.go @@ -210,6 +210,8 @@ type ControllerPluginResourcesSpec struct { } type ControllerPluginSpec struct { + // hostNetwork setting to be propagated to CSI controller plugin pods + HostNetwork *bool `json:"hostNetwork,omitempty"` // Embedded common pods spec PodCommonSpec `json:",inline"` @@ -274,6 +276,7 @@ const ( // DriverSpec defines the desired state of Driver type DriverSpec struct { + // Logging configuration for driver's pods //+kubebuilder:validation:Optional Log *LogSpec `json:"log,omitempty"` diff --git a/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/zz_generated.deepcopy.go b/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/zz_generated.deepcopy.go index a8542df9..bb617807 100644 --- a/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -437,6 +437,11 @@ func (in *ControllerPluginResourcesSpec) DeepCopy() *ControllerPluginResourcesSp // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControllerPluginSpec) DeepCopyInto(out *ControllerPluginSpec) { *out = *in + if in.HostNetwork != nil { + in, out := &in.HostNetwork, &out.HostNetwork + *out = new(bool) + **out = **in + } in.PodCommonSpec.DeepCopyInto(&out.PodCommonSpec) if in.DeploymentStrategy != nil { in, out := &in.DeploymentStrategy, &out.DeploymentStrategy From bca85a92bb0c16f62fdd74b01b6d7d0fefb10f44 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Wed, 5 Feb 2025 18:21:53 +0100 Subject: [PATCH 2/2] csi: use different ports for controller and node plugins Usubg host network produces port collisions. So we use different ports for controller plugin deployments and node plugin deamonsets to avoid collisions. We also make sure that rbd and cephfs drivers don't collide Signed-off-by: Michael Adam --- internal/controller/driver_controller.go | 24 ++++++++++++++++++++---- internal/utils/csi.go | 19 ++++++++++++++++--- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index 509ee733..5edbf713 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -759,6 +759,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { } // Addons Sidecar Container if !r.isNfsDriver() && ptr.Deref(r.driver.Spec.DeployCsiAddons, false) { + port := r.controllerPluginCsiAddonsContainerPort() containers = append(containers, corev1.Container{ Name: "csi-addons", Image: r.images["addons"], @@ -772,7 +773,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.PodContainerArg, utils.PodUidContainerArg, utils.CsiAddonsAddressContainerArg, - utils.ControllerPortContainerArg, + utils.ContainerPortArg(port), utils.NamespaceContainerArg, utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), @@ -780,7 +781,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { ), ), Ports: []corev1.ContainerPort{ - utils.CsiAddonsContainerPort, + port, }, Env: []corev1.EnvVar{ utils.NodeIdEnvVar, @@ -922,6 +923,20 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { return err } +func (r *driverReconcile) controllerPluginCsiAddonsContainerPort() corev1.ContainerPort { + + // the cephFS and rbd drivers need to use different ports + // to avoid port collisions with host network. + port := utils.ControllerPluginCsiAddonsContainerRbdPort + if r.isCephFsDriver() { + port = utils.ControllerPluginCsiAddonsContainerCephFsPort + + } + + return port + +} + func (r *driverReconcile) reconcileNodePluginDeamonSet() error { daemonSet := &appsv1.DaemonSet{} daemonSet.Name = r.generateName("nodeplugin") @@ -1107,6 +1122,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { } // CSI Addons Sidecar Container if r.isRdbDriver() && ptr.Deref(r.driver.Spec.DeployCsiAddons, false) { + port := utils.NodePluginCsiAddonsContainerPort containers = append(containers, corev1.Container{ Name: "csi-addons", Image: r.images["addons"], @@ -1122,7 +1138,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.CsiAddonsNodeIdContainerArg, utils.LogVerbosityContainerArg(logVerbosity), utils.CsiAddonsAddressContainerArg, - utils.ControllerPortContainerArg, + utils.ContainerPortArg(port), utils.PodContainerArg, utils.NamespaceContainerArg, utils.PodUidContainerArg, @@ -1133,7 +1149,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { }, ), Ports: []corev1.ContainerPort{ - utils.CsiAddonsContainerPort, + port, }, Env: []corev1.EnvVar{ utils.NodeIdEnvVar, diff --git a/internal/utils/csi.go b/internal/utils/csi.go index 59c12cfa..5b6237d9 100644 --- a/internal/utils/csi.go +++ b/internal/utils/csi.go @@ -337,11 +337,25 @@ var DriverNamespaceEnvVar = corev1.EnvVar{ }, } -// CSI Addons container port definition -var CsiAddonsContainerPort = corev1.ContainerPort{ +// CSI Addons container port definitions +var ControllerPluginCsiAddonsContainerRbdPort = corev1.ContainerPort{ ContainerPort: int32(9070), } +var ControllerPluginCsiAddonsContainerCephFsPort = corev1.ContainerPort{ + ContainerPort: int32(9080), +} + +var NodePluginCsiAddonsContainerPort = corev1.ContainerPort{ + ContainerPort: int32(9071), +} + +func ContainerPortArg(port corev1.ContainerPort) string { + + return fmt.Sprintf("--controller-port=%d", port.ContainerPort) + +} + // Ceph CSI common container arguments var CsiAddressContainerArg = fmt.Sprintf("--csi-address=%s", csiEndpoint) var EndpointContainerArg = fmt.Sprintf("--endpoint=%s", csiEndpoint) @@ -359,7 +373,6 @@ var HandleVolumeInuseErrorContainerArg = "--handle-volume-inuse-error=false" var PodUidContainerArg = fmt.Sprintf("--pod-uid=$(%s)", PodUidEnvVar.Name) var PodContainerArg = fmt.Sprintf("--pod=$(%s)", PodNameEnvVar.Name) var NamespaceContainerArg = fmt.Sprintf("--namespace=$(%s)", PodNamespaceEnvVar.Name) -var ControllerPortContainerArg = fmt.Sprintf("--controller-port=%d", CsiAddonsContainerPort.ContainerPort) var DriverNamespaceContainerArg = fmt.Sprintf("--drivernamespace=$(%s)", DriverNamespaceEnvVar.Name) var MetricsPathContainerArg = "--metricspath=/metrics" var PoolTimeContainerArg = "--polltime=60s"