Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Wrap event recorder #1584

Merged
merged 17 commits into from
Jun 3, 2024
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .golangci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ linters-settings:
alias: prometheusclient
- pkg: github.com/kyma-project/lifecycle-manager/pkg/testutils/skrcontextimpl
alias: testskrcontext
- pkg: github.com/kyma-project/lifecycle-manager/internal/controller/watcher
alias: watcherctrl
ireturn:
allow:
- anon
Expand Down
85 changes: 51 additions & 34 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,14 @@ import (
"github.com/kyma-project/lifecycle-manager/api/shared"
"github.com/kyma-project/lifecycle-manager/api/v1beta2"
"github.com/kyma-project/lifecycle-manager/internal"
"github.com/kyma-project/lifecycle-manager/internal/controller"
"github.com/kyma-project/lifecycle-manager/internal/controller/kyma"
"github.com/kyma-project/lifecycle-manager/internal/controller/mandatorymodule"
"github.com/kyma-project/lifecycle-manager/internal/controller/manifest"
"github.com/kyma-project/lifecycle-manager/internal/controller/purge"
watcherctrl "github.com/kyma-project/lifecycle-manager/internal/controller/watcher"
"github.com/kyma-project/lifecycle-manager/internal/crd"
"github.com/kyma-project/lifecycle-manager/internal/descriptor/provider"
"github.com/kyma-project/lifecycle-manager/internal/event"
"github.com/kyma-project/lifecycle-manager/internal/pkg/flags"
"github.com/kyma-project/lifecycle-manager/internal/pkg/metrics"
"github.com/kyma-project/lifecycle-manager/internal/remote"
Expand Down Expand Up @@ -82,12 +86,9 @@ var (
func registerSchemas(scheme *machineryruntime.Scheme) {
machineryutilruntime.Must(k8sclientscheme.AddToScheme(scheme))
machineryutilruntime.Must(api.AddToScheme(scheme))

machineryutilruntime.Must(apiextensionsv1.AddToScheme(scheme))
machineryutilruntime.Must(certmanagerv1.AddToScheme(scheme))

machineryutilruntime.Must(istioclientapiv1beta1.AddToScheme(scheme))

machineryutilruntime.Must(v1beta2.AddToScheme(scheme))
// +kubebuilder:scaffold:scheme
}
Expand Down Expand Up @@ -163,7 +164,8 @@ func setupManager(flagVar *flags.FlagVar, cacheOptions cache.Options, scheme *ma
kcpRestConfig := mgr.GetConfig()
remoteClientCache := remote.NewClientCache()
kcpClient := remote.NewClientWithConfig(mgr.GetClient(), kcpRestConfig)
skrContextProvider := remote.NewKymaSkrContextProvider(kcpClient, remoteClientCache)
eventRecorder := event.NewRecorderWrapper(mgr.GetEventRecorderFor(shared.OperatorName))
skrContextProvider := remote.NewKymaSkrContextProvider(kcpClient, remoteClientCache, eventRecorder)
var skrWebhookManager *watcher.SKRWebhookManifestManager
options := controllerOptionsFromFlagVar(flagVar)
if flagVar.EnableKcpWatcher {
Expand All @@ -178,13 +180,13 @@ func setupManager(flagVar *flags.FlagVar, cacheOptions cache.Options, scheme *ma
descriptorProvider := provider.NewCachedDescriptorProvider(nil)
kymaMetrics := metrics.NewKymaMetrics(sharedMetrics)
mandatoryModulesMetrics := metrics.NewMandatoryModulesMetrics()
setupKymaReconciler(mgr, descriptorProvider, skrContextProvider, flagVar, options, skrWebhookManager, kymaMetrics,
setupKymaReconciler(mgr, descriptorProvider, skrContextProvider, eventRecorder, flagVar, options, skrWebhookManager, kymaMetrics,
setupLog)
setupManifestReconciler(mgr, flagVar, options, sharedMetrics, mandatoryModulesMetrics, setupLog)
setupMandatoryModuleReconciler(mgr, descriptorProvider, flagVar, options, mandatoryModulesMetrics, setupLog)
setupMandatoryModuleDeletionReconciler(mgr, descriptorProvider, flagVar, options, setupLog)
setupMandatoryModuleDeletionReconciler(mgr, descriptorProvider, eventRecorder, flagVar, options, setupLog)
if flagVar.EnablePurgeFinalizer {
setupPurgeReconciler(mgr, skrContextProvider, flagVar, options, setupLog)
setupPurgeReconciler(mgr, skrContextProvider, eventRecorder, flagVar, options, setupLog)
}
if flagVar.EnableWebhooks {
enableWebhooks(mgr, setupLog)
Expand Down Expand Up @@ -274,15 +276,20 @@ func controllerOptionsFromFlagVar(flagVar *flags.FlagVar) ctrlruntime.Options {
}

func setupKymaReconciler(mgr ctrl.Manager,
descriptorProvider *provider.CachedDescriptorProvider, skrContextFactory remote.SkrContextProvider, flagVar *flags.FlagVar, options ctrlruntime.Options,
skrWebhookManager *watcher.SKRWebhookManifestManager, kymaMetrics *metrics.KymaMetrics, setupLog logr.Logger,
descriptorProvider *provider.CachedDescriptorProvider,
skrContextFactory remote.SkrContextProvider,
event event.Event,
flagVar *flags.FlagVar,
options ctrlruntime.Options,
skrWebhookManager *watcher.SKRWebhookManifestManager,
kymaMetrics *metrics.KymaMetrics,
setupLog logr.Logger,
) {
options.MaxConcurrentReconciles = flagVar.MaxConcurrentKymaReconciles

if err := (&kyma.Reconciler{
Client: mgr.GetClient(),
SkrContextFactory: skrContextFactory,
EventRecorder: mgr.GetEventRecorderFor(shared.OperatorName),
Event: event,
DescriptorProvider: descriptorProvider,
SyncRemoteCrds: remote.NewSyncCrdsUseCase(mgr.GetClient(), skrContextFactory, nil),
SKRWebhookManager: skrWebhookManager,
Expand All @@ -297,7 +304,7 @@ func setupKymaReconciler(mgr ctrl.Manager,
IsManagedKyma: flagVar.IsKymaManaged,
Metrics: kymaMetrics,
}).SetupWithManager(
mgr, options, kyma.ReconcilerSetupSettings{
mgr, options, kyma.SetupOptions{
ListenerAddr: flagVar.KymaListenerAddr,
EnableDomainNameVerification: flagVar.EnableDomainNameVerification,
IstioNamespace: flagVar.IstioNamespace,
Expand Down Expand Up @@ -356,13 +363,17 @@ func getWatcherImg(flagVar *flags.FlagVar) string {
return fmt.Sprintf("%s:%s", watcherRegProd, flagVar.WatcherImageTag)
}

func setupPurgeReconciler(mgr ctrl.Manager, skrContextProvider remote.SkrContextProvider, flagVar *flags.FlagVar,
options ctrlruntime.Options, setupLog logr.Logger,
func setupPurgeReconciler(mgr ctrl.Manager,
skrContextProvider remote.SkrContextProvider,
event event.Event,
flagVar *flags.FlagVar,
options ctrlruntime.Options,
setupLog logr.Logger,
) {
if err := (&controller.PurgeReconciler{
if err := (&purge.Reconciler{
Client: mgr.GetClient(),
SkrContextFactory: skrContextProvider,
EventRecorder: mgr.GetEventRecorderFor(shared.OperatorName),
Event: event,
PurgeFinalizerTimeout: flagVar.PurgeFinalizerTimeout,
SkipCRDs: matcher.CreateCRDMatcherFrom(flagVar.SkipPurgingFor),
IsManagedKyma: flagVar.IsKymaManaged,
Expand All @@ -383,11 +394,11 @@ func setupManifestReconciler(mgr ctrl.Manager, flagVar *flags.FlagVar, options c
options.RateLimiter = internal.ManifestRateLimiter(flagVar.FailureBaseDelay,
flagVar.FailureMaxDelay, flagVar.RateLimiterFrequency, flagVar.RateLimiterBurst)

if err := controller.SetupWithManager(
if err := manifest.SetupWithManager(
mgr, options, queue.RequeueIntervals{
Success: flagVar.ManifestRequeueSuccessInterval,
Busy: flagVar.KymaRequeueBusyInterval,
}, controller.SetupUpSetting{
}, manifest.SetupOptions{
ListenerAddr: flagVar.ManifestListenerAddr,
EnableDomainNameVerification: flagVar.EnableDomainNameVerification,
}, metrics.NewManifestMetrics(sharedMetrics), mandatoryModulesMetrics,
Expand All @@ -402,11 +413,11 @@ func setupKcpWatcherReconciler(mgr ctrl.Manager, options ctrlruntime.Options, fl
) {
options.MaxConcurrentReconciles = flagVar.MaxConcurrentWatcherReconciles

if err := (&controller.WatcherReconciler{
Client: mgr.GetClient(),
EventRecorder: mgr.GetEventRecorderFor(shared.OperatorName),
Scheme: mgr.GetScheme(),
RestConfig: mgr.GetConfig(),
if err := (&watcherctrl.Reconciler{
nesmabadr marked this conversation as resolved.
Show resolved Hide resolved
Client: mgr.GetClient(),
Event: event.NewRecorderWrapper(mgr.GetEventRecorderFor(shared.OperatorName)),
Scheme: mgr.GetScheme(),
RestConfig: mgr.GetConfig(),
RequeueIntervals: queue.RequeueIntervals{
Success: flagVar.WatcherRequeueSuccessInterval,
Busy: flags.DefaultKymaRequeueBusyInterval,
Expand All @@ -415,20 +426,22 @@ func setupKcpWatcherReconciler(mgr ctrl.Manager, options ctrlruntime.Options, fl
},
IstioGatewayNamespace: flagVar.IstioGatewayNamespace,
}).SetupWithManager(mgr, options); err != nil {
setupLog.Error(err, "unable to create controller", "controller", controller.WatcherControllerName)
setupLog.Error(err, "unable to create watcher controller")
os.Exit(bootstrapFailedExitCode)
}
}

func setupMandatoryModuleReconciler(mgr ctrl.Manager, descriptorProvider *provider.CachedDescriptorProvider,
flagVar *flags.FlagVar, options ctrlruntime.Options, metrics *metrics.MandatoryModulesMetrics,
func setupMandatoryModuleReconciler(mgr ctrl.Manager,
descriptorProvider *provider.CachedDescriptorProvider,
flagVar *flags.FlagVar,
options ctrlruntime.Options,
metrics *metrics.MandatoryModulesMetrics,
setupLog logr.Logger,
) {
options.MaxConcurrentReconciles = flagVar.MaxConcurrentMandatoryModuleReconciles

if err := (&controller.MandatoryModuleReconciler{
Client: mgr.GetClient(),
EventRecorder: mgr.GetEventRecorderFor(shared.OperatorName),
if err := (&mandatorymodule.InstallationReconciler{
Client: mgr.GetClient(),
RequeueIntervals: queue.RequeueIntervals{
Success: flagVar.MandatoryModuleRequeueSuccessInterval,
Busy: flagVar.KymaRequeueBusyInterval,
Expand All @@ -445,14 +458,18 @@ func setupMandatoryModuleReconciler(mgr ctrl.Manager, descriptorProvider *provid
}
}

func setupMandatoryModuleDeletionReconciler(mgr ctrl.Manager, descriptorProvider *provider.CachedDescriptorProvider,
flagVar *flags.FlagVar, options ctrlruntime.Options, setupLog logr.Logger,
func setupMandatoryModuleDeletionReconciler(mgr ctrl.Manager,
descriptorProvider *provider.CachedDescriptorProvider,
event event.Event,
flagVar *flags.FlagVar,
options ctrlruntime.Options,
setupLog logr.Logger,
) {
options.MaxConcurrentReconciles = flagVar.MaxConcurrentMandatoryModuleDeletionReconciles

if err := (&controller.MandatoryModuleDeletionReconciler{
if err := (&mandatorymodule.DeletionReconciler{
Client: mgr.GetClient(),
EventRecorder: mgr.GetEventRecorderFor(shared.OperatorName),
Event: event,
DescriptorProvider: descriptorProvider,
RequeueIntervals: queue.RequeueIntervals{
Success: flagVar.MandatoryModuleDeletionRequeueSuccessInterval,
Expand Down
6 changes: 3 additions & 3 deletions docs/technical-reference/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ To run, Lifecycle Manager uses the following workflow:
Apart from the custom resources, Lifecycle Manager uses also Kyma, Manifest, and Watcher controllers:

- [Kyma controller](../../internal/controller/kyma/controller.go) - reconciles the Kyma CR which means creating Manifest CRs for each Kyma module enabled in the Kyma CR and deleting them when modules are disabled in the Kyma CR. It is also responsible for synchronising ModuleTemplate CRs between KCP and Kyma runtimes.
- [Manifest controller](../../internal/controller/manifest_controller.go) - reconciles the Manifest CRs created by the Kyma controller, which means, installing components specified in the Manifest CR on the target SKR cluster and removing them when the Manifest CRs are flagged for deletion.
- [Purge controller](../../internal/controller/purge_controller.go) - reconciles the Kyma CRs which are marked for deletion longer than the grace period, which means purging all the resources deployed by the Lifecycle Manager on the target SKR cluster.
- [Watcher controller](../../internal/controller/watcher_controller.go) - reconciles the Watcher CR which means creating Istio Virtual Service resources on KCP when a Watcher CR is created and removing the same resources when it is deleted. This is done in order to configure the routing of the messages coming from the watcher agent installed on each Kyma runtime and going to a listener agent deployed on KCP.
- [Manifest controller](../../internal/controller/manifest/controller.go) - reconciles the Manifest CRs created by the Kyma controller, which means, installing components specified in the Manifest CR on the target SKR cluster and removing them when the Manifest CRs are flagged for deletion.
nesmabadr marked this conversation as resolved.
Show resolved Hide resolved
lindnerby marked this conversation as resolved.
Show resolved Hide resolved
- [Purge controller](../../internal/controller/purge/controller.go) - reconciles the Kyma CRs which are marked for deletion longer than the grace period, which means purging all the resources deployed by the Lifecycle Manager on the target SKR cluster.
lindnerby marked this conversation as resolved.
Show resolved Hide resolved
- [Watcher controller](../../internal/controller/watcher/controller.go) - reconciles the Watcher CR which means creating Istio Virtual Service resources on KCP when a Watcher CR is created and removing the same resources when it is deleted. This is done in order to configure the routing of the messages coming from the watcher agent installed on each Kyma runtime and going to a listener agent deployed on KCP.
lindnerby marked this conversation as resolved.
Show resolved Hide resolved

For more details about Lifecycle Manager controllers, read the [Controllers](controllers.md) document.

Expand Down
10 changes: 5 additions & 5 deletions docs/technical-reference/controllers.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,20 @@ The Kyma CR in Kyma Control Plane shows the initial specification and the curren

Lifecycle Manager uses two Mandatory Modules Controllers:

- [Mandatory modules installation controller](../../internal/controller/mandatory_modules_installation_controller.go) deals with the reconciliation of mandatory modules
- [Mandatory modules deletion controller](../../internal/controller/mandatory_modules_deletion_controller.go) deals with the deletion of mandatory modules
- [Mandatory modules installation controller](../../internal/controller/mandatorymodule/installation_controller.go) deals with the reconciliation of mandatory modules
- [Mandatory modules deletion controller](../../internal/controller/mandatorymodule/deletion_controller.go) deals with the deletion of mandatory modules

Since the channel concept does not apply to mandatory modules, the Mandatory Modules Installation Controller fetches all the Mandatory ModuleTemplate CRs without any channel filtering. It then translates the ModuleTemplate CR for the mandatory module to a [Manifest CR](../../api/v1beta2/manifest_types.go) with an OwnerReference to the Kyma CR. Similarly to the [Kyma Controller](../../internal/controller/kyma/controller.go),
it propagates changes from the ModuleTemplate CR to the Manifest CR. The mandatory ModuleTemplate CR is not synchronized to the remote cluster and the module status does not appear in the Kyma CR status. If a mandatory module needs to be removed from all clusters, the corresponding ModuleTemplate CR needs to be deleted. The Mandatory Module Deletion Controller picks this event up and marks all associated Manifest CRs for deletion. To ensure that the ModuleTemplate CR is not removed immediately, the controller adds a finalizer to the ModuleTemplate CR. Once all associated Manifest CRs are deleted, the finalizer is removed and the ModuleTemplate CR is deleted.

## Manifest Controller

[Manifest controller](../../internal/controller/manifest_controller.go) deals with the reconciliation and installation of data desired through a Manifest CR, a representation of a single module desired in a cluster.
[Manifest controller](../../internal/controller/manifest/controller.go) deals with the reconciliation and installation of data desired through a Manifest CR, a representation of a single module desired in a cluster.
Since it mainly is a delegation to the [declarative reconciliation library](../../internal/declarative/README.md) with certain [internal implementation additions](../../internal/manifest/README.md), please look at the respective documentation for these parts to understand them more.

## Purge Controller
[Purge controller](../../internal/controller/purge_controller.go) is responsible for handling the forced cleanup of deployed resources in a remote cluster when its Kyma CR is marked for deletion.
[Purge controller](../../internal/controller/purge/controller.go) is responsible for handling the forced cleanup of deployed resources in a remote cluster when its Kyma CR is marked for deletion.
Suppose a Kyma CR has been marked for deletion for longer than the grace period (default is 5 minutes). In that case, the controller resolves the remote client for the cluster, retrieves all relevant CRs deployed on the cluster, and removes finalizers, allowing the resources to be garbage collected. This ensures that all associated resources are properly purged, maintaining the integrity and cleanliness of the cluster.
## Watcher Controller

[Watcher controller](../../internal/controller/watcher_controller.go) deals with the update of VirtualService rules derived from the [Watcher CR](../../api/v1beta2/watcher_types.go). This is then used to initialize the Watcher CR from the Kyma Controller in each runtime, a small component initialized to propagate changes from the runtime (remote) clusters back to react to changes that can affect the Manifest CR integrity.
[Watcher controller](../../internal/controller/watcher/controller.go) deals with the update of VirtualService rules derived from the [Watcher CR](../../api/v1beta2/watcher_types.go). This is then used to initialize the Watcher CR from the Kyma Controller in each runtime, a small component initialized to propagate changes from the runtime (remote) clusters back to react to changes that can affect the Manifest CR integrity.
Loading
Loading