From 862b4f70e6d7a4c4173292a6748b39a73972326a Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 18 Dec 2023 07:47:00 +1100 Subject: [PATCH] feat: support idling messages from core --- apis/lagoon/v1beta1/lagoonmessaging_types.go | 1 + apis/lagoon/v1beta1/lagoontask_types.go | 34 +++++- apis/lagoon/v1beta1/zz_generated.deepcopy.go | 32 ++++++ .../crd/bases/crd.lagoon.sh_lagoonbuilds.yaml | 2 + .../crd/bases/crd.lagoon.sh_lagoontasks.yaml | 17 +++ controllers/namespace/namespace.go | 104 ++++++++++++++++++ controllers/namespace/predicates.go | 38 +++++++ internal/messenger/consumer.go | 33 ++++++ internal/messenger/tasks_handler.go | 86 +++++++++++++++ main.go | 13 +++ 10 files changed, 354 insertions(+), 6 deletions(-) create mode 100644 controllers/namespace/namespace.go create mode 100644 controllers/namespace/predicates.go diff --git a/apis/lagoon/v1beta1/lagoonmessaging_types.go b/apis/lagoon/v1beta1/lagoonmessaging_types.go index f9fe6ec..c07c363 100644 --- a/apis/lagoon/v1beta1/lagoonmessaging_types.go +++ b/apis/lagoon/v1beta1/lagoonmessaging_types.go @@ -47,6 +47,7 @@ type LagoonMessage struct { Type string `json:"type,omitempty"` Namespace string `json:"namespace,omitempty"` Meta *LagoonLogMeta `json:"meta,omitempty"` + Idled bool `json:"idled,omitempty"` // BuildInfo *LagoonBuildInfo `json:"buildInfo,omitempty"` } diff --git a/apis/lagoon/v1beta1/lagoontask_types.go b/apis/lagoon/v1beta1/lagoontask_types.go index 3231e53..d05ffd8 100644 --- a/apis/lagoon/v1beta1/lagoontask_types.go +++ b/apis/lagoon/v1beta1/lagoontask_types.go @@ -75,12 +75,34 @@ func (b TaskType) String() string { type LagoonTaskSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - Key string `json:"key,omitempty"` - Task LagoonTaskInfo `json:"task,omitempty"` - Project LagoonTaskProject `json:"project,omitempty"` - Environment LagoonTaskEnvironment `json:"environment,omitempty"` - Misc *LagoonMiscInfo `json:"misc,omitempty"` - AdvancedTask *LagoonAdvancedTaskInfo `json:"advancedTask,omitempty"` + Key string `json:"key,omitempty"` + Task LagoonTaskInfo `json:"task,omitempty"` + Project LagoonTaskProject `json:"project,omitempty"` + Environment LagoonTaskEnvironment `json:"environment,omitempty"` + Misc *LagoonMiscInfo `json:"misc,omitempty"` + AdvancedTask *LagoonAdvancedTaskInfo `json:"advancedTask,omitempty"` + LagoonService LagoonServiceInfo `json:"lagoonService,omitempty"` + Idling LagoonIdling `json:"lagoonIdling,omitempty"` +} + +// TaskType const for the status type +type ServiceState string + +// These are valid conditions of a job. +const ( + StateStop ServiceState = "stop" + StateStart ServiceState = "start" + StateRestart ServiceState = "restart" +) + +type LagoonServiceInfo struct { + Name string `json:"name,omitempty"` + State ServiceState `json:"state,omitempty"` +} + +type LagoonIdling struct { + Idle bool `json:"idle,omitempty"` + ForceScale bool `json:"forceScale,omitempty"` } // LagoonTaskInfo defines what a task can use to communicate with Lagoon via SSH/API. diff --git a/apis/lagoon/v1beta1/zz_generated.deepcopy.go b/apis/lagoon/v1beta1/zz_generated.deepcopy.go index 842af61..6540fd0 100644 --- a/apis/lagoon/v1beta1/zz_generated.deepcopy.go +++ b/apis/lagoon/v1beta1/zz_generated.deepcopy.go @@ -198,6 +198,21 @@ func (in *LagoonBuildStatus) DeepCopy() *LagoonBuildStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonIdling) DeepCopyInto(out *LagoonIdling) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonIdling. +func (in *LagoonIdling) DeepCopy() *LagoonIdling { + if in == nil { + return nil + } + out := new(LagoonIdling) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LagoonLog) DeepCopyInto(out *LagoonLog) { *out = *in @@ -318,6 +333,21 @@ func (in *LagoonMiscInfo) DeepCopy() *LagoonMiscInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonServiceInfo) DeepCopyInto(out *LagoonServiceInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonServiceInfo. +func (in *LagoonServiceInfo) DeepCopy() *LagoonServiceInfo { + if in == nil { + return nil + } + out := new(LagoonServiceInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LagoonStatusMessages) DeepCopyInto(out *LagoonStatusMessages) { *out = *in @@ -499,6 +529,8 @@ func (in *LagoonTaskSpec) DeepCopyInto(out *LagoonTaskSpec) { *out = new(LagoonAdvancedTaskInfo) **out = **in } + out.LagoonService = in.LagoonService + out.Idling = in.Idling } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskSpec. diff --git a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml index bbc2dc2..505dd2b 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml @@ -302,6 +302,8 @@ spec: description: LagoonMessage is used for sending build info back to Lagoon messaging queue to update the environment or deployment properties: + idled: + type: boolean meta: description: LagoonLogMeta is the metadata that is used by logging in Lagoon. diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index f1394f9..dfb14ce 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -71,6 +71,21 @@ spec: description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster Important: Run "make" to regenerate code after modifying this file' type: string + lagoonIdling: + properties: + forceScale: + type: boolean + idle: + type: boolean + type: object + lagoonService: + properties: + name: + type: string + state: + description: TaskType const for the status type + type: string + type: object misc: description: LagoonMiscInfo defines the resource or backup information for a misc task. @@ -284,6 +299,8 @@ spec: description: LagoonMessage is used for sending build info back to Lagoon messaging queue to update the environment or deployment properties: + idled: + type: boolean meta: description: LagoonLogMeta is the metadata that is used by logging in Lagoon. diff --git a/controllers/namespace/namespace.go b/controllers/namespace/namespace.go new file mode 100644 index 0000000..63375e2 --- /dev/null +++ b/controllers/namespace/namespace.go @@ -0,0 +1,104 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namespace + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + + "github.com/go-logr/logr" + lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + "github.com/uselagoon/remote-controller/internal/messenger" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +// NamespaceReconciler reconciles idling +type NamespaceReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + EnableMQ bool + Messaging *messenger.Messenger + LagoonTargetName string +} + +func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("namespace", req.NamespacedName) + + var namespace corev1.Namespace + if err := r.Get(ctx, req.NamespacedName, &namespace); err != nil { + return ctrl.Result{}, ignoreNotFound(err) + } + + // this would be nice to be a lagoon label :) + if val, ok := namespace.ObjectMeta.Labels["idling.amazee.io/idled"]; ok { + idled, _ := strconv.ParseBool(val) + opLog.Info(fmt.Sprintf("environment %s idle state %t", namespace.Name, idled)) + if r.EnableMQ { + var projectName, environmentName string + if p, ok := namespace.ObjectMeta.Labels["lagoon.sh/project"]; ok { + projectName = p + } + if e, ok := namespace.ObjectMeta.Labels["lagoon.sh/environment"]; ok { + environmentName = e + } + msg := lagoonv1beta1.LagoonMessage{ + Type: "idling", + Namespace: namespace.Name, + Meta: &lagoonv1beta1.LagoonLogMeta{ + Environment: environmentName, + Project: projectName, + Cluster: r.LagoonTargetName, + }, + Idled: idled, + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because for some reason, log the error and move on + // this may result in the state being out of sync in lagoon but eventually will be consistent + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + return ctrl.Result{}, nil + } + } + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the watch on the namespace resource with an event filter (see predicates.go) +func (r *NamespaceReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Namespace{}). + WithEventFilter(NamespacePredicates{}). + Complete(r) +} + +// will ignore not found errors +func ignoreNotFound(err error) error { + if apierrors.IsNotFound(err) { + return nil + } + return err +} diff --git a/controllers/namespace/predicates.go b/controllers/namespace/predicates.go new file mode 100644 index 0000000..b0f3c42 --- /dev/null +++ b/controllers/namespace/predicates.go @@ -0,0 +1,38 @@ +package namespace + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// NamespacePredicates defines the funcs for predicates +type NamespacePredicates struct { + predicate.Funcs +} + +// Create is used when a creation event is received by the controller. +func (n NamespacePredicates) Create(e event.CreateEvent) bool { + return false +} + +// Delete is used when a deletion event is received by the controller. +func (n NamespacePredicates) Delete(e event.DeleteEvent) bool { + return false +} + +// Update is used when an update event is received by the controller. +func (n NamespacePredicates) Update(e event.UpdateEvent) bool { + if oldIdled, ok := e.ObjectOld.GetLabels()["idling.amazee.io/idled"]; ok { + if newIdled, ok := e.ObjectNew.GetLabels()["idling.amazee.io/idled"]; ok { + if oldIdled != newIdled { + return true + } + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (n NamespacePredicates) Generic(e event.GenericEvent) bool { + return false +} diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index a0b9085..94b5759 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -416,6 +416,39 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue return } + case "deploytarget:environment:idling": + opLog.Info( + fmt.Sprintf( + "Received environment idling request for project %s, environment %s - %s", + jobSpec.Project.Name, + jobSpec.Environment.Name, + namespace, + ), + ) + // idle or unidle an environment, optionally forcible scale it so it can't be unidled by the ingress + err := m.ScaleOrIdleEnvironment(ctx, opLog, namespace, jobSpec.Idling.Idle, jobSpec.Idling.ForceScale) + if err != nil { + //@TODO: send msg back to lagoon and update task to failed? + message.Ack(false) // ack to remove from queue + return + } + case "deploytarget:environment:service": + opLog.Info( + fmt.Sprintf( + "Received environment service request for project %s, environment %s service %s - %s", + jobSpec.Project.Name, + jobSpec.Environment.Name, + jobSpec.LagoonService.Name, + namespace, + ), + ) + // idle an environment, optionally forcible scale it so it can't be unidled by the ingress + err := m.EnvironmentServiceState(ctx, opLog, namespace, jobSpec.LagoonService.Name, jobSpec.LagoonService.State) + if err != nil { + //@TODO: send msg back to lagoon and update task to failed? + message.Ack(false) // ack to remove from queue + return + } default: // if we get something that we don't know about, spit out the entire message opLog.Info( diff --git a/internal/messenger/tasks_handler.go b/internal/messenger/tasks_handler.go index a5ff074..f1965c8 100644 --- a/internal/messenger/tasks_handler.go +++ b/internal/messenger/tasks_handler.go @@ -6,12 +6,14 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "strings" "time" "github.com/go-logr/logr" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -312,3 +314,87 @@ func createAdvancedTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec, } return nil } + +func (m *Messenger) ScaleOrIdleEnvironment(ctx context.Context, opLog logr.Logger, ns string, idle, forceScale bool) error { + namespace := &corev1.Namespace{} + err := m.Client.Get(ctx, types.NamespacedName{ + Name: ns, + }, namespace) + if err != nil { + return err + } + if idle { + if forceScale { + // this would be nice to be a lagoon label :) + namespace.ObjectMeta.Labels["idling.amazee.io/force-scaled"] = "true" + } else { + // this would be nice to be a lagoon label :) + namespace.ObjectMeta.Labels["idling.amazee.io/force-idled"] = "true" + } + } else { + // this would be nice to be a lagoon label :) + namespace.ObjectMeta.Labels["idling.amazee.io/unidle"] = "true" + } + if err := m.Client.Update(context.Background(), namespace); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update namespace %s to set idle state.", + ns, + ), + ) + return err + } + return nil +} + +func (m *Messenger) EnvironmentServiceState(ctx context.Context, opLog logr.Logger, ns, service string, state lagoonv1beta1.ServiceState) error { + deployment := &appsv1.Deployment{} + err := m.Client.Get(ctx, types.NamespacedName{ + Name: service, + Namespace: ns, + }, deployment) + if err != nil { + return err + } + update := false + switch state { + case lagoonv1beta1.StateRestart: + deployment.ObjectMeta.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339) + update = true + case lagoonv1beta1.StateStop: + if *deployment.Spec.Replicas > 0 { + // if the service has replicas, then save the replica count and scale it to 0 + deployment.ObjectMeta.Annotations["service.lagoon.sh/replicas"] = strconv.FormatInt(int64(*deployment.Spec.Replicas), 10) + replicas := int32(0) + deployment.Spec.Replicas = &replicas + update = true + } + case lagoonv1beta1.StateStart: + if *deployment.Spec.Replicas == 0 { + // if the service has no replicas, set it back to what the previous replica value was + prevReplicas, err := strconv.Atoi(deployment.ObjectMeta.Annotations["service.lagoon.sh/replicas"]) + if err != nil { + return err + } + replicas := int32(prevReplicas) + deployment.Spec.Replicas = &replicas + delete(deployment.ObjectMeta.Annotations, "service.lagoon.sh/replicas") + update = true + } + default: + // nothing to do + return nil + } + if update { + if err := m.Client.Update(ctx, deployment); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update deployment %s to change its state.", + ns, + ), + ) + return err + } + } + return nil +} diff --git a/main.go b/main.go index 7238b1a..64756b4 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ import ( "github.com/hashicorp/golang-lru/v2/expirable" k8upv1 "github.com/k8up-io/k8up/v2/api/v1" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + "github.com/uselagoon/remote-controller/controllers/namespace" lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/controllers/v1beta1" "github.com/uselagoon/remote-controller/internal/messenger" k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1" @@ -852,6 +853,18 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "LagoonTask") os.Exit(1) } + // start the namespace reconciler + if err = (&namespace.NamespaceReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("namespace").WithName("Namespace"), + Scheme: mgr.GetScheme(), + EnableMQ: enableMQ, + Messaging: messaging, + LagoonTargetName: lagoonTargetName, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Namespace") + os.Exit(1) + } // for now the namespace reconciler only needs to run if harbor is enabled so that we can watch the namespace for rotation label events if lffHarborEnabled {