diff --git a/api/v1alpha1/condition_consts.go b/api/v1alpha1/condition_consts.go index 1540802..78c8511 100644 --- a/api/v1alpha1/condition_consts.go +++ b/api/v1alpha1/condition_consts.go @@ -48,3 +48,15 @@ const ( // to be available before proceeding. WaitingForBootstrapDataReason = "WaitingForBoostrapData" ) + +const ( + // ExternalLoadBalancerEndpointAvailableCondition is a condition that indicates that the API server Load Balancer is available. + ExternalLoadBalancerEndpointAvailableCondition clusterv1.ConditionType = "ExternalLoadBalancerEndpointAvailable" + + // ExternalLoadBalancerEndpointNotAvailableReason is used to indicate any error with the + // availability of the load balancer. + ExternalLoadBalancerEndpointFailedReason = "ExternalLoadBalancerEndpointFailed" + + // ExternalLoadBalancerEndpointNotAvailableReason is used to indicate that the load balancer isn't available. + ExternalLoadBalancerEndpointNotAvailableReason = "ExternalLoadBalancerEndpointNotAvailable" +) diff --git a/api/v1alpha1/externalloadbalancer_types.go b/api/v1alpha1/externalloadbalancer_types.go index bffe45d..16c0855 100644 --- a/api/v1alpha1/externalloadbalancer_types.go +++ b/api/v1alpha1/externalloadbalancer_types.go @@ -4,15 +4,35 @@ package v1alpha1 import ( + "strconv" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) +type ExternalLoadBalancerEndpoint struct { + // The hostname on which the API server is serving. + // +required + Host string `json:"host"` + + // The port on which the API server is serving. + // +optional + // +kubebuilder:default=6443 + Port int32 `json:"port"` +} + +func (ep *ExternalLoadBalancerEndpoint) String() string { + port := strconv.Itoa(int(ep.Port)) + + return ep.Host + ":" + port +} + // ExternalLoadBalancerSpec defines the desired state for a ExternalLoadBalancer. type ExternalLoadBalancerSpec struct { // Endpoint represents the endpoint for the load balancer. This endpoint will - // best tested to see if its available. - Endpoint clusterv1.APIEndpoint `json:"endpoint"` + // be tested to see if its available. + Endpoint ExternalLoadBalancerEndpoint `json:"endpoint"` + ClusterName string `json:"clusterName"` } type ExternalLoadBalancerStatus struct { diff --git a/api/v1alpha1/microvmcluster_types.go b/api/v1alpha1/microvmcluster_types.go index 4586bce..787197b 100644 --- a/api/v1alpha1/microvmcluster_types.go +++ b/api/v1alpha1/microvmcluster_types.go @@ -4,19 +4,13 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) // MicrovmClusterSpec defines the desired state of MicrovmCluster. type MicrovmClusterSpec struct { - // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. - // - // See https://cluster-api.sigs.k8s.io/developer/architecture/controllers/cluster.html - // for more details. - // - // +optional - ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` // SSHPublicKey is an SSH public key that will be used with the default user. If specified // this will apply to all machine created unless you specify a different key at the // machine level. @@ -25,6 +19,8 @@ type MicrovmClusterSpec struct { // Placement specifies how machines for the cluster should be placed onto hosts (i.e. where the microvms are created). // +kubebuilder:validation:Required Placement Placement `json:"placement"` + // LoadBalancerRef + LoadBalancerRef *corev1.ObjectReference `json:"loadBalancerRef,omitempty"` } // MicrovmClusterStatus defines the observed state of MicrovmCluster. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 5818eb1..5b18023 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -9,6 +9,7 @@ package v1alpha1 import ( + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/errors" @@ -56,6 +57,21 @@ func (in *ExternalLoadBalancer) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalLoadBalancerEndpoint) DeepCopyInto(out *ExternalLoadBalancerEndpoint) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalLoadBalancerEndpoint. +func (in *ExternalLoadBalancerEndpoint) DeepCopy() *ExternalLoadBalancerEndpoint { + if in == nil { + return nil + } + out := new(ExternalLoadBalancerEndpoint) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExternalLoadBalancerList) DeepCopyInto(out *ExternalLoadBalancerList) { *out = *in @@ -188,8 +204,12 @@ func (in *MicrovmClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MicrovmClusterSpec) DeepCopyInto(out *MicrovmClusterSpec) { *out = *in - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint in.Placement.DeepCopyInto(&out.Placement) + if in.LoadBalancerRef != nil { + in, out := &in.LoadBalancerRef, &out.LoadBalancerRef + *out = new(v1.ObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MicrovmClusterSpec. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_externalloadbalancers.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_externalloadbalancers.yaml index b61c13f..a0b3bb3 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_externalloadbalancers.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_externalloadbalancers.yaml @@ -50,22 +50,25 @@ spec: description: ExternalLoadBalancerSpec defines the desired state for a ExternalLoadBalancer. properties: + clusterName: + type: string endpoint: description: Endpoint represents the endpoint for the load balancer. - This endpoint will best tested to see if its available. + This endpoint will be tested to see if its available. properties: host: description: The hostname on which the API server is serving. type: string port: + default: 6443 description: The port on which the API server is serving. format: int32 type: integer required: - host - - port type: object required: + - clusterName - endpoint type: object status: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmclusters.yaml index c75fbed..a8c0391 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmclusters.yaml @@ -53,21 +53,41 @@ spec: spec: description: MicrovmClusterSpec defines the desired state of MicrovmCluster. properties: - controlPlaneEndpoint: - description: "ControlPlaneEndpoint represents the endpoint used to - communicate with the control plane. \n See https://cluster-api.sigs.k8s.io/developer/architecture/controllers/cluster.html - for more details." + loadBalancerRef: + description: LoadBalancerRef properties: - host: - description: The hostname on which the API server is serving. + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port type: object placement: description: Placement specifies how machines for the cluster should diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index c9471d7..3a64cdb 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -8,6 +8,7 @@ resources: - bases/infrastructure.cluster.x-k8s.io_microvmclusters.yaml - bases/infrastructure.cluster.x-k8s.io_microvmmachines.yaml - bases/infrastructure.cluster.x-k8s.io_microvmmachinetemplates.yaml +- bases/infrastructure.cluster.x-k8s.io_externalloadbalancers.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 72efa86..6b1282b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -50,6 +50,26 @@ rules: - get - list - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - externalloadbalancers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - externalloadbalancers/status + verbs: + - get + - patch + - update - apiGroups: - infrastructure.cluster.x-k8s.io resources: diff --git a/controllers/errors.go b/controllers/errors.go index be408b4..b908ce4 100644 --- a/controllers/errors.go +++ b/controllers/errors.go @@ -6,10 +6,11 @@ package controllers import "errors" var ( - errControlplaneEndpointRequired = errors.New("controlplane endpoint is required on cluster or mvmcluster") - errClientFactoryFuncRequired = errors.New("factory function required to create grpc client") - errMicrovmFailed = errors.New("microvm is in a failed state") - errMicrovmUnknownState = errors.New("microvm is in an unknown/unsupported state") - errExpectedMicrovmCluster = errors.New("expected microvm cluster") - errNoPlacement = errors.New("no placement specified") + errExternalLoadBalancerEndpointRefRequired = errors.New("endpointRef is required on mvmcluster") + errClientFactoryFuncRequired = errors.New("factory function required to create grpc client") + errMicrovmFailed = errors.New("microvm is in a failed state") + errMicrovmUnknownState = errors.New("microvm is in an unknown/unsupported state") + errExpectedMicrovmCluster = errors.New("expected microvm cluster") + errNoPlacement = errors.New("no placement specified") + errInvalidLoadBalancerResponseStatusCode = errors.New("endpoint returned a 5XX status code") ) diff --git a/controllers/externalloadbalancer_controller.go b/controllers/externalloadbalancer_controller.go new file mode 100644 index 0000000..e7fcb6b --- /dev/null +++ b/controllers/externalloadbalancer_controller.go @@ -0,0 +1,235 @@ +// Copyright 2022 Weaveworks or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MPL-2.0. + +package controllers + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/cluster-api/util/predicates" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/log" + + infrav1 "github.com/weaveworks/cluster-api-provider-microvm/api/v1alpha1" + "github.com/weaveworks/cluster-api-provider-microvm/internal/defaults" +) + +type ExternalLoadBalancerReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + WatchFilterValue string + HTTPClient *http.Client +} + +const ( + httpErrorStatusCode = 50 + defaultHTTPTimeout = 5 * time.Second +) + +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=externalloadbalancers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=externalloadbalancers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile +func (r *ExternalLoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx) + + loadbalancer := &infrav1.ExternalLoadBalancer{} + if err := r.Get(ctx, req.NamespacedName, loadbalancer); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + log.Error(err, "error getting externalloadbalancer", "id", req.NamespacedName) + + return ctrl.Result{}, err + } + + defer func() { + if err := r.Patch(ctx, loadbalancer); err != nil { + log.Error(err, "attempting to patch loadbalancer object") + } + }() + + if err := r.ensureClusterOwnerRef(ctx, req, loadbalancer); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + log.Error(err, "retrieving cluster from clusterName", "clusterName", loadbalancer.ClusterName) + + return ctrl.Result{}, err + } + + if !loadbalancer.ObjectMeta.DeletionTimestamp.IsZero() { + log.Info("loadbalancer being deleted, nothing to do") + + return ctrl.Result{}, nil + } + + if err := r.sendTestRequest(ctx, loadbalancer); err != nil { + if os.IsTimeout(err) { + log.Error(err, "request timed out attempting to contact endpoint", "endpoint", loadbalancer.Spec.Endpoint.String()) + conditions.MarkFalse( + loadbalancer, + infrav1.ExternalLoadBalancerEndpointAvailableCondition, + infrav1.ExternalLoadBalancerEndpointNotAvailableReason, + clusterv1.ConditionSeverityInfo, "request to loadbalancer endpoint timed out", + ) + + return ctrl.Result{}, fmt.Errorf("request timed out attempting to contact endpoint: %s: %w", loadbalancer.Spec.Endpoint.String(), err) + } + + if errors.Is(err, errInvalidLoadBalancerResponseStatusCode) { + log.Error(err, "request to endpoint", "endpoint", loadbalancer.Spec.Endpoint.String()) + conditions.MarkFalse( + loadbalancer, + infrav1.ExternalLoadBalancerEndpointAvailableCondition, + infrav1.ExternalLoadBalancerEndpointNotAvailableReason, + clusterv1.ConditionSeverityInfo, "loadbalancer endpoint responded with error", + ) + + return ctrl.Result{}, nil + } + + log.Error(err, "attempting to contact specified endpoint", "endpoint", loadbalancer.Spec.Endpoint.String()) + conditions.MarkFalse( + loadbalancer, + infrav1.ExternalLoadBalancerEndpointAvailableCondition, + infrav1.ExternalLoadBalancerEndpointFailedReason, + clusterv1.ConditionSeverityInfo, "request to loadbalancer endpoint failed: %s", + err.Error(), + ) + + return ctrl.Result{}, fmt.Errorf("attempting to contact specified endpoint: %s: %w", loadbalancer.Spec.Endpoint.String(), err) + } + + loadbalancer.Status.Ready = true + conditions.MarkTrue(loadbalancer, infrav1.ExternalLoadBalancerEndpointAvailableCondition) + + return ctrl.Result{}, nil +} + +// Patch persists the resource and status. +func (r *ExternalLoadBalancerReconciler) Patch(ctx context.Context, lb *infrav1.ExternalLoadBalancer) error { + applicableConditions := []clusterv1.ConditionType{ + infrav1.ExternalLoadBalancerEndpointAvailableCondition, + } + + conditions.SetSummary(lb, + conditions.WithConditions(applicableConditions...), + conditions.WithStepCounterIf(lb.DeletionTimestamp.IsZero()), + conditions.WithStepCounter(), + ) + + patchHelper, err := patch.NewHelper(lb, r.Client) + if err != nil { + return err + } + if patchErr := patchHelper.Patch( + ctx, + lb, + patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ + clusterv1.ReadyCondition, + infrav1.LoadBalancerAvailableCondition, + }}); patchErr != nil { + return err + } + + return nil +} + +// sendTestRequest makes an HTTP call to ${KUBE_VIP_HOST}:${KUBE_VIP_PORT}/livez, which, if the loadbalancer is live, +// should reach the /livez endpoint on the Kubernetes API server. +func (r *ExternalLoadBalancerReconciler) sendTestRequest(ctx context.Context, lb *infrav1.ExternalLoadBalancer) error { + endpoint := fmt.Sprintf("https://%s/livez", lb.Spec.Endpoint.String()) + epReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) // use livez endpoint + if err != nil { + return fmt.Errorf("creating endpoint request: %w", err) + } + + log.Log.V(defaults.LogLevelDebug).Info("attempting request to API server livez endpoint via loadbalancer", "endpoint_address", endpoint) + resp, err := r.HTTPClient.Do(epReq) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode >= httpErrorStatusCode { + return errInvalidLoadBalancerResponseStatusCode + } + + return nil +} + +func (r *ExternalLoadBalancerReconciler) ensureClusterOwnerRef(ctx context.Context, req ctrl.Request, lb *infrav1.ExternalLoadBalancer) error { + clusterNamespaceName := types.NamespacedName{ + Namespace: req.NamespacedName.Namespace, + Name: lb.Spec.ClusterName, + } + + cluster := &clusterv1.Cluster{} + if err := r.Get(ctx, clusterNamespaceName, cluster); err != nil { + return err + } + + lb.OwnerReferences = util.EnsureOwnerRef(lb.OwnerReferences, metav1.OwnerReference{ + APIVersion: cluster.APIVersion, + Kind: cluster.Kind, + Name: cluster.Name, + UID: cluster.UID, + }) + + return nil +} + +func (r *ExternalLoadBalancerReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + log := ctrl.LoggerFrom(ctx) + + if r.HTTPClient == nil { + r.HTTPClient = &http.Client{Timeout: defaultHTTPTimeout} + } + + builder := ctrl.NewControllerManagedBy(mgr). + WithOptions(options). + For(&infrav1.ExternalLoadBalancer{}). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log, r.WatchFilterValue)). + WithEventFilter(predicates.ResourceIsNotExternallyManaged(log)) //. + // Watches( + // &source.Kind{Type: &clusterv1.Cluster{}}, + // handler.EnqueueRequestsFromMapFunc( + // util.ClusterToInfrastructureMapFunc(infrav1.GroupVersion.WithKind("ExternalLoadBalancer")), + // ), + // builder.WithPredicates( + // predicates.ClusterUnpaused(log), + // ), + // ) + + if err := builder.Complete(r); err != nil { + return fmt.Errorf("creating external loadbalancer controller: %w", err) + } + + return nil +} diff --git a/controllers/microvmcluster_controller.go b/controllers/microvmcluster_controller.go index ef702c6..d233fc6 100644 --- a/controllers/microvmcluster_controller.go +++ b/controllers/microvmcluster_controller.go @@ -8,9 +8,9 @@ import ( "fmt" "time" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -51,6 +51,7 @@ type MicrovmClusterReconciler struct { // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=microvmclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=microvmclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=microvmclusters/finalizers,verbs=update +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=externalloadbalancers,verbs=get;list;watch // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -127,8 +128,8 @@ func (r *MicrovmClusterReconciler) reconcileDelete(_ context.Context, clusterSco func (r *MicrovmClusterReconciler) reconcileNormal(ctx context.Context, clusterScope *scope.ClusterScope) (reconcile.Result, error) { clusterScope.Info("Reconciling MicrovmCluster") - if clusterScope.Cluster.Spec.ControlPlaneEndpoint.IsZero() && clusterScope.MvmCluster.Spec.ControlPlaneEndpoint.IsZero() { - return reconcile.Result{}, errControlplaneEndpointRequired + if clusterScope.MvmCluster.Spec.LoadBalancerRef == nil { + return reconcile.Result{}, errExternalLoadBalancerEndpointRefRequired } clusterScope.MvmCluster.Status.Ready = true @@ -149,25 +150,18 @@ func (r *MicrovmClusterReconciler) reconcileNormal(ctx context.Context, clusterS } func (r *MicrovmClusterReconciler) isAPIServerAvailable(ctx context.Context, clusterScope *scope.ClusterScope) bool { - clusterScope.V(defaults.LogLevelDebug).Info("checking if api server is available", "cluster", clusterScope.ClusterName()) - - clusterKey := client.ObjectKey{ - Name: clusterScope.Cluster.Name, - Namespace: clusterScope.Cluster.Namespace, + endpoint := &infrav1.ExternalLoadBalancer{} + eprnn := types.NamespacedName{ + Namespace: clusterScope.MvmCluster.ObjectMeta.Namespace, + Name: clusterScope.MvmCluster.Spec.LoadBalancerRef.Name, } - - remoteClient, err := r.RemoteClientGetter(ctx, clusterScope.ClusterName(), r.Client, clusterKey) - if err != nil { - clusterScope.Error(err, "creating remote cluster client") + if err := r.Get(ctx, eprnn, endpoint); err != nil { + clusterScope.Error(err, "get referenced ExternalLoadBalancerEndpoint") return false } - nodes := &corev1.NodeList{} - if err = remoteClient.List(ctx, nodes); err != nil { - return false - } - if len(nodes.Items) == 0 { + if !endpoint.Status.Ready { return false } diff --git a/controllers/microvmcluster_controller_test.go b/controllers/microvmcluster_controller_test.go index 76abb9b..32eedc7 100644 --- a/controllers/microvmcluster_controller_test.go +++ b/controllers/microvmcluster_controller_test.go @@ -44,75 +44,35 @@ func TestClusterReconciliationNoEndpoint(t *testing.T) { g.Expect(c).To(BeNil()) } -func TestClusterReconciliationWithClusterEndpoint(t *testing.T) { - g := NewWithT(t) - - cluster := createCluster(testClusterName, testClusterNamespace) - cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{ - Host: "192.168.8.15", - Port: 6443, - } - - tenantClusterNodes := &corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - }, - }, - } - - objects := []runtime.Object{ - cluster, - createMicrovmCluster(testClusterName, testClusterNamespace), - tenantClusterNodes, - } - - client := createFakeClient(g, objects) - result, err := reconcileCluster(client) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(result.Requeue).To(BeFalse()) - g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) - - reconciled, err := getMicrovmCluster(context.TODO(), client, testClusterName, testClusterNamespace) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(reconciled.Status.Ready).To(BeTrue()) - g.Expect(reconciled.Status.FailureDomains).To(HaveLen(1)) - - c := conditions.Get(reconciled, infrav1.LoadBalancerAvailableCondition) - g.Expect(c).ToNot(BeNil()) - g.Expect(c.Status).To(Equal(corev1.ConditionTrue)) - - c = conditions.Get(reconciled, clusterv1.ReadyCondition) - g.Expect(c).ToNot(BeNil()) - g.Expect(c.Status).To(Equal(corev1.ConditionTrue)) -} - func TestClusterReconciliationWithMvmClusterEndpoint(t *testing.T) { g := NewWithT(t) mvmCluster := createMicrovmCluster(testClusterName, testClusterNamespace) - mvmCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{ - Host: "192.168.8.15", - Port: 6443, + mvmCluster.Spec.LoadBalancerRef = &corev1.ObjectReference{ + Kind: "ExternalLoadBalancerEndpoint", + Name: "tenant1-elb-endpoint", } - tenantClusterNodes := &corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, + endpoint := &infrav1.ExternalLoadBalancer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant1-elb-endpoint", + Namespace: "ns1", + }, + Spec: infrav1.ExternalLoadBalancerSpec{ + Endpoint: infrav1.ExternalLoadBalancerEndpoint{ + Host: "localhost", + Port: 6443, }, }, + Status: infrav1.ExternalLoadBalancerStatus{ + Ready: true, + }, } objects := []runtime.Object{ createCluster(testClusterName, testClusterNamespace), mvmCluster, - tenantClusterNodes, + endpoint, } client := createFakeClient(g, objects) @@ -140,19 +100,32 @@ func TestClusterReconciliationWithClusterEndpointAPIServerNotReady(t *testing.T) g := NewWithT(t) cluster := createCluster(testClusterName, testClusterNamespace) - cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{ - Host: "192.168.8.15", - Port: 6443, + mvmCluster := createMicrovmCluster(testClusterName, testClusterNamespace) + mvmCluster.Spec.LoadBalancerRef = &corev1.ObjectReference{ + Kind: "ExternalLoadBalancerEndpoint", + Name: "tenant1-elb-endpoint", } - tenantClusterNodes := &corev1.NodeList{ - Items: []corev1.Node{}, + endpoint := &infrav1.ExternalLoadBalancer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant1-elb-endpoint", + Namespace: "ns1", + }, + Spec: infrav1.ExternalLoadBalancerSpec{ + Endpoint: infrav1.ExternalLoadBalancerEndpoint{ + Host: "localhost", + Port: 6443, + }, + }, + Status: infrav1.ExternalLoadBalancerStatus{ + Ready: false, + }, } objects := []runtime.Object{ cluster, - createMicrovmCluster(testClusterName, testClusterNamespace), - tenantClusterNodes, + mvmCluster, + endpoint, } client := createFakeClient(g, objects) diff --git a/main.go b/main.go index 2a213d6..f0b701c 100644 --- a/main.go +++ b/main.go @@ -286,6 +286,15 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) error { return fmt.Errorf("unable to create microvm machine controller: %w", err) } + if err := (&controllers.ExternalLoadBalancerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("externalloadbalancer-controller"), + WatchFilterValue: watchFilterValue, + }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: microvmMachineConcurrency, RecoverPanic: true}); err != nil { + return fmt.Errorf("unable to create external loadbalancer controller: %w", err) + } + return nil } diff --git a/templates/cluster-template-cilium.yaml b/templates/cluster-template-cilium.yaml index 55549ef..57ccc1c 100644 --- a/templates/cluster-template-cilium.yaml +++ b/templates/cluster-template-cilium.yaml @@ -25,18 +25,29 @@ spec: name: "${CLUSTER_NAME}-control-plane" --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: ExternalLoadBalancer +metadata: + name: "${CLUSTER_NAME}-api-server" +spec: + clusterName: ${CLUSTER_NAME} + endpoint: + host: ${CONTROL_PLANE_VIP} + port: 6443 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: MicrovmCluster metadata: name: "${CLUSTER_NAME}" spec: - controlPlaneEndpoint: - host: "${CONTROL_PLANE_VIP}" - port: 6443 placement: staticPool: hosts: - endpoint: "${HOST_ENDPOINT:=127.0.0.1:9090}" controlplaneAllowed: true + loadBalancerRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ExternalLoadBalancer + name: "${CLUSTER_NAME}-api-server" --- kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 86138ba..b0c8c97 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -20,19 +20,31 @@ spec: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 name: "${CLUSTER_NAME}-control-plane" --- +# ExternalLoadBalancer Definition +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: ExternalLoadBalancer +metadata: + name: "${CLUSTER_NAME}-api-server" +spec: + clusterName: ${CLUSTER_NAME} + endpoint: + host: ${CONTROL_PLANE_VIP} + port: 6443 +--- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: MicrovmCluster metadata: name: "${CLUSTER_NAME}" spec: - controlPlaneEndpoint: - host: "${CONTROL_PLANE_VIP}" - port: 6443 placement: staticPool: hosts: - endpoint: "${HOST_ENDPOINT:=127.0.0.1:9090}" controlplaneAllowed: true + loadBalancerRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ExternalLoadBalancer + name: "${CLUSTER_NAME}-api-server" --- kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1