From 754ab637274e35c615faa3164a7c100d7b8d1628 Mon Sep 17 00:00:00 2001 From: Tom Weininger Date: Wed, 17 Apr 2024 12:47:30 +0000 Subject: [PATCH 1/5] Log if not all conditions are ready --- controllers/amphoracontroller_controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controllers/amphoracontroller_controller.go b/controllers/amphoracontroller_controller.go index 6455cd79..35008808 100644 --- a/controllers/amphoracontroller_controller.go +++ b/controllers/amphoracontroller_controller.go @@ -461,6 +461,8 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context if instance.Status.Conditions.AllSubConditionIsTrue() { instance.Status.Conditions.MarkTrue( condition.ReadyCondition, condition.ReadyMessage) + } else { + Log.Info("Not all conditions are ready for Amphora controller") } Log.Info("Reconciled Service successfully") return ctrl.Result{}, nil From 835ce7f3d55f7bd14e1f6b8b01207fbb26402195 Mon Sep 17 00:00:00 2001 From: Tom Weininger Date: Fri, 19 Apr 2024 11:32:05 +0000 Subject: [PATCH 2/5] Update Octavia Controller's ready condition Take ready conditions of other services into account as well. --- api/v1beta1/octavia_types.go | 18 ++++++++++++++---- controllers/amphoracontroller_controller.go | 1 - 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index 37acc5b2..ee7ef91d 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -302,11 +302,21 @@ func init() { SchemeBuilder.Register(&Octavia{}, &OctaviaList{}) } -// IsReady - returns true if service is ready to server requests +// IsReady - returns true if services are ready to serve requests func (instance Octavia) IsReady() bool { - ready := instance.Status.OctaviaAPIReadyCount > 0 - // TODO: add other ready counts - return ready + readyCounts := []int32{ + instance.Status.OctaviaAPIReadyCount, + instance.Status.OctaviaHealthManagerReadyCount, + instance.Status.OctaviaHousekeepingReadyCount, + instance.Status.OctaviaWorkerReadyCount, + } + + for _, readyCount := range readyCounts { + if readyCount < 1 { + return false + } + } + return true } // SetupDefaults - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) diff --git a/controllers/amphoracontroller_controller.go b/controllers/amphoracontroller_controller.go index 35008808..30e7c9a7 100644 --- a/controllers/amphoracontroller_controller.go +++ b/controllers/amphoracontroller_controller.go @@ -464,7 +464,6 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context } else { Log.Info("Not all conditions are ready for Amphora controller") } - Log.Info("Reconciled Service successfully") return ctrl.Result{}, nil } From 6aba325c6181b1f977ff1beeb18700d8f06b69b7 Mon Sep 17 00:00:00 2001 From: Tom Weininger Date: Wed, 24 Apr 2024 15:03:39 +0000 Subject: [PATCH 3/5] Add new OctaviaAmphoraCertsReadyCondition --- api/v1beta1/conditions.go | 11 +++++++++++ controllers/octavia_controller.go | 6 ++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index c3dba159..23a0df3c 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -30,6 +30,8 @@ const ( OctaviaHousekeepingReadyCondition condition.Type = "OctaviaHousekeepingReady" OctaviaWorkerReadyCondition condition.Type = "OctaviaWorkerReady" + + OctaviaAmphoraCertsReadyCondition condition.Type = "OctaviaAmphoraCertsReady" ) // Common Messages used by API objects @@ -69,4 +71,13 @@ const ( // OctaviaAmphoraControllerReadyErrorMessage OctaviaWorkerReadyErrorMessage = "OctaviaWorker error occured %s" + + // + // OctaviaAmphoraCertsReady condition messages + // + // OctaviaAmphoraCertsReadyInitMessage + OctaviaAmphoraCertsReadyInitMessage = "Octavia Amphora certificates are not initialized" + + // OctaviaAmphoraCertsReadyErrorMessage + OctaviaAmphoraCertsReadyErrorMessage = "Octavia Amphora certificates error occured %s" ) diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index d02aba00..48e7013f 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -187,6 +187,7 @@ func (r *OctaviaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(octaviav1.OctaviaAmphoraCertsReadyCondition, condition.InitReason, octaviav1.OctaviaAmphoraCertsReadyInitMessage), amphoraControllerInitCondition(octaviav1.HealthManager), amphoraControllerInitCondition(octaviav1.Housekeeping), amphoraControllerInitCondition(octaviav1.Worker), @@ -504,13 +505,14 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav err = octavia.EnsureAmphoraCerts(ctx, instance, helper, &Log) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - condition.ServiceConfigReadyCondition, + octaviav1.OctaviaAmphoraCertsReadyCondition, condition.ErrorReason, condition.SeverityWarning, - condition.ServiceConfigReadyErrorMessage, + octaviav1.OctaviaAmphoraCertsReadyErrorMessage, err.Error())) return ctrl.Result{}, err } + instance.Status.Conditions.MarkTrue(octaviav1.OctaviaAmphoraCertsReadyCondition, octaviav1.OctaviaAmphoraCertsReadyInitMessage) if err = octavia.EnsureQuotas(ctx, instance, &r.Log, helper); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( From fbcc68b936dbe5db7dc65c24e36638ba887208e2 Mon Sep 17 00:00:00 2001 From: Tom Weininger Date: Tue, 30 Apr 2024 11:00:16 +0000 Subject: [PATCH 4/5] Add more conditions to Octavia controller New conditions are for: - Quotas - SSH - Amphora Images --- api/v1beta1/conditions.go | 33 +++++++++++++++++++++++++++++++ controllers/octavia_controller.go | 29 +++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 23a0df3c..697a77bb 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -32,6 +32,12 @@ const ( OctaviaWorkerReadyCondition condition.Type = "OctaviaWorkerReady" OctaviaAmphoraCertsReadyCondition condition.Type = "OctaviaAmphoraCertsReady" + + OctaviaQuotasReadyCondition condition.Type = "OctaviaQuotasReady" + + OctaviaAmphoraSSHReadyCondition condition.Type = "OctaviaAmphoraSSHReady" + + OctaviaAmphoraImagesReadyCondition condition.Type = "OctaviaAmphoraImagesReady" ) // Common Messages used by API objects @@ -80,4 +86,31 @@ const ( // OctaviaAmphoraCertsReadyErrorMessage OctaviaAmphoraCertsReadyErrorMessage = "Octavia Amphora certificates error occured %s" + + // + // OctaviaQuotasReady condition messages + // + // OctaviaQuotasReadyInitMessage + OctaviaQuotasReadyInitMessage = "Octavia quotas are not initialized" + + // OctaviaAmphoraCertsReadyErrorMessage + OctaviaQuotasReadyErrorMessage = "Octavia quotas setup error occured %s" + + // + // OctaviaAmphoraSSHReady condition messages + // + // OctaviaAmphoraSSHReadyInitMessage + OctaviaAmphoraSSHReadyInitMessage = "Octavia Amphora SSH setup is not initialized" + + // OctaviaAmphoraSSHReadyErrorMessage + OctaviaAmphoraSSHReadyErrorMessage = "Octavia Amphora SSH setup error occured %s" + + // + // OctaviaAmphoraImagesReady condition messages + // + // OctaviaAmphoraImagesReadyInitMessage + OctaviaAmphoraImagesReadyInitMessage = "Octavia Amphora Images setup is not initialized" + + // OctaviaAmphoraImagesReadyErrorMessage + OctaviaAmphoraImagesReadyErrorMessage = "Octavia Amphora Images setup error occured %s" ) diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index 48e7013f..a645dddc 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -188,6 +188,9 @@ func (r *OctaviaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), condition.UnknownCondition(octaviav1.OctaviaAmphoraCertsReadyCondition, condition.InitReason, octaviav1.OctaviaAmphoraCertsReadyInitMessage), + condition.UnknownCondition(octaviav1.OctaviaQuotasReadyCondition, condition.InitReason, octaviav1.OctaviaQuotasReadyInitMessage), + condition.UnknownCondition(octaviav1.OctaviaAmphoraSSHReadyCondition, condition.InitReason, octaviav1.OctaviaAmphoraSSHReadyInitMessage), + condition.UnknownCondition(octaviav1.OctaviaAmphoraImagesReadyCondition, condition.InitReason, octaviav1.OctaviaAmphoraImagesReadyInitMessage), amphoraControllerInitCondition(octaviav1.HealthManager), amphoraControllerInitCondition(octaviav1.Housekeeping), amphoraControllerInitCondition(octaviav1.Worker), @@ -501,6 +504,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav condition.InputReadyWaitingMessage)) return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) err = octavia.EnsureAmphoraCerts(ctx, instance, helper, &Log) if err != nil { @@ -516,15 +520,14 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav if err = octavia.EnsureQuotas(ctx, instance, &r.Log, helper); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, + octaviav1.OctaviaQuotasReadyCondition, condition.ErrorReason, condition.SeverityWarning, - condition.InputReadyErrorMessage, + octaviav1.OctaviaQuotasReadyErrorMessage, err.Error())) return ctrl.Result{}, err } - - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + instance.Status.Conditions.MarkTrue(octaviav1.OctaviaQuotasReadyCondition, octaviav1.OctaviaQuotasReadyInitMessage) // // TODO check when/if Init, Update, or Upgrade should/could be skipped @@ -776,16 +779,34 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav // Amphora SSH key config for debugging err = octavia.EnsureAmpSSHConfig(ctx, instance, helper) if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + octaviav1.OctaviaAmphoraSSHReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + octaviav1.OctaviaAmphoraSSHReadyErrorMessage, + err.Error())) return ctrl.Result{}, err } + instance.Status.Conditions.MarkTrue( + octaviav1.OctaviaAmphoraSSHReadyCondition, + octaviav1.OctaviaAmphoraSSHReadyInitMessage) ctrlResult, err = r.reconcileAmphoraImages(ctx, instance, helper) if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil } if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + octaviav1.OctaviaAmphoraImagesReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + octaviav1.OctaviaAmphoraImagesReadyErrorMessage, + err.Error())) return ctrl.Result{}, err } + instance.Status.Conditions.MarkTrue( + octaviav1.OctaviaAmphoraImagesReadyCondition, + octaviav1.OctaviaAmphoraImagesReadyInitMessage) // create Deployment - end From 6bc2f29df048a92b599a37db6256c7e06c516eee Mon Sep 17 00:00:00 2001 From: Tom Weininger Date: Tue, 30 Apr 2024 11:56:46 +0000 Subject: [PATCH 5/5] Use specific complete message for new conditons --- api/v1beta1/conditions.go | 12 ++++++++++++ controllers/octavia_controller.go | 12 ++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 697a77bb..fe266fc6 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -87,6 +87,9 @@ const ( // OctaviaAmphoraCertsReadyErrorMessage OctaviaAmphoraCertsReadyErrorMessage = "Octavia Amphora certificates error occured %s" + // OctaviaAmphoraCertsReadyCompleteMessage + OctaviaAmphoraCertsReadyCompleteMessage = "Octavia Amphora certificates setup completed" + // // OctaviaQuotasReady condition messages // @@ -96,6 +99,9 @@ const ( // OctaviaAmphoraCertsReadyErrorMessage OctaviaQuotasReadyErrorMessage = "Octavia quotas setup error occured %s" + // OctaviaAmphoraSSHReadyCompleteMessage + OctaviaQuotasReadyCompleteMessage = "Octavia quotas setup completed" + // // OctaviaAmphoraSSHReady condition messages // @@ -105,6 +111,9 @@ const ( // OctaviaAmphoraSSHReadyErrorMessage OctaviaAmphoraSSHReadyErrorMessage = "Octavia Amphora SSH setup error occured %s" + // OctaviaAmphoraSSHReadyCompleteMessage + OctaviaAmphoraSSHReadyCompleteMessage = "Octavia Amphora SSH setup completed" + // // OctaviaAmphoraImagesReady condition messages // @@ -113,4 +122,7 @@ const ( // OctaviaAmphoraImagesReadyErrorMessage OctaviaAmphoraImagesReadyErrorMessage = "Octavia Amphora Images setup error occured %s" + + // OctaviaAmphoraImagesReadyCompleteMessage + OctaviaAmphoraImagesReadyCompleteMessage = "Octavia Amphora Images setup completed" ) diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index a645dddc..e780c740 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -516,7 +516,9 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav err.Error())) return ctrl.Result{}, err } - instance.Status.Conditions.MarkTrue(octaviav1.OctaviaAmphoraCertsReadyCondition, octaviav1.OctaviaAmphoraCertsReadyInitMessage) + instance.Status.Conditions.MarkTrue( + octaviav1.OctaviaAmphoraCertsReadyCondition, + octaviav1.OctaviaAmphoraCertsReadyCompleteMessage) if err = octavia.EnsureQuotas(ctx, instance, &r.Log, helper); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( @@ -527,7 +529,9 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav err.Error())) return ctrl.Result{}, err } - instance.Status.Conditions.MarkTrue(octaviav1.OctaviaQuotasReadyCondition, octaviav1.OctaviaQuotasReadyInitMessage) + instance.Status.Conditions.MarkTrue( + octaviav1.OctaviaQuotasReadyCondition, + octaviav1.OctaviaQuotasReadyCompleteMessage) // // TODO check when/if Init, Update, or Upgrade should/could be skipped @@ -789,7 +793,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav } instance.Status.Conditions.MarkTrue( octaviav1.OctaviaAmphoraSSHReadyCondition, - octaviav1.OctaviaAmphoraSSHReadyInitMessage) + octaviav1.OctaviaAmphoraSSHReadyCompleteMessage) ctrlResult, err = r.reconcileAmphoraImages(ctx, instance, helper) if (ctrlResult != ctrl.Result{}) { @@ -806,7 +810,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav } instance.Status.Conditions.MarkTrue( octaviav1.OctaviaAmphoraImagesReadyCondition, - octaviav1.OctaviaAmphoraImagesReadyInitMessage) + octaviav1.OctaviaAmphoraImagesReadyCompleteMessage) // create Deployment - end