Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: tls policy status to state of the world tasks #885

Merged
merged 16 commits into from
Oct 9, 2024
Merged

refactor: tls policy status to state of the world tasks #885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
b6fe7bd
refactor: tls policy status to state of the world tasks
KevFan Sep 26, 2024
0fe2d41
refactor: improve linking functions
KevFan Sep 30, 2024
fdf9bd4
refactor: events from certificate owned by TLSPolicies only
KevFan Oct 1, 2024
270c7c5
test: re-add unit tests for enforced condition
KevFan Oct 1, 2024
33a80fc
feat: status for cert manager not installed in tls policies
KevFan Oct 2, 2024
e6a1e4f
fixup: use GroupKind for isObjectOwnedBy function
KevFan Oct 2, 2024
a6649f2
fixup: logs
KevFan Oct 2, 2024
bd3b3c1
refactor: align with some common conventions
KevFan Oct 3, 2024
fddf4ee
fixup: map[string]error for TLSPolicy validation & refine target not …
KevFan Oct 4, 2024
36081c4
refactor: link certs to listeners instead
KevFan Oct 7, 2024
1880122
fixup: subscribe to events for tls status
KevFan Oct 7, 2024
fef1ae4
refactor: check for isCertManagerInstalled only from boot
KevFan Oct 7, 2024
e974c4b
refactor: struct to store state of dependencies in initial boot
KevFan Oct 7, 2024
c83ca3d
refactor: bootOptionsBuilder
KevFan Oct 8, 2024
1093827
fixup: address comments
KevFan Oct 9, 2024
1a391b8
fixup: move getting hostname to outside loop
KevFan Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
234 changes: 180 additions & 54 deletions controllers/state_of_the_world.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package controllers

import (
"fmt"
"reflect"

certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
egv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1"
Expand All @@ -19,6 +20,8 @@ import (
"k8s.io/client-go/dynamic"
"k8s.io/utils/env"
ctrlruntime "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
ctrlruntimepredicate "sigs.k8s.io/controller-runtime/pkg/predicate"
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"

Expand All @@ -42,6 +45,7 @@ var (
//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=list;watch

func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.DynamicClient, logger logr.Logger) *controller.Controller {
// Base options
controllerOpts := []controller.ControllerOption{
controller.ManagedBy(manager),
controller.WithLogger(logger),
Expand Down Expand Up @@ -83,6 +87,7 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*corev1.ConfigMap]{}),
controller.FilterResourcesByLabel[*corev1.ConfigMap](fmt.Sprintf("%s=true", kuadrant.TopologyLabel)),
)),
// TODO: Move as boot options for Limitador and Authorino as there can be a possibility that the operators are not installed
controller.WithRunnable("limitador watcher", controller.Watch(
&limitadorv1alpha1.Limitador{},
kuadrantv1beta1.LimitadorsResource,
Expand Down Expand Up @@ -112,11 +117,56 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
),
}

ok, err := kuadrantgatewayapi.IsGatewayAPIInstalled(manager.GetRESTMapper())
if err != nil || !ok {
logger.Info("gateway api is not installed, skipping watches and reconcilers", "err", err)
// Boot options and reconciler based on detected dependencies
bootOptions := NewBootOptionsBuilder(manager, client, logger)
controllerOpts = append(controllerOpts, bootOptions.getOptions()...)
controllerOpts = append(controllerOpts, controller.WithReconcile(bootOptions.Reconciler()))

return controller.NewController(controllerOpts...)
}

// NewBootOptionsBuilder is used to return a list of controller.ControllerOption and a controller.ReconcileFunc that depend
// on if external dependent CRDs are installed at boot time
func NewBootOptionsBuilder(manager ctrlruntime.Manager, client *dynamic.DynamicClient, logger logr.Logger) *BootOptionsBuilder {
return &BootOptionsBuilder{
manager: manager,
client: client,
logger: logger,
}
}

type BootOptionsBuilder struct {
logger logr.Logger
manager ctrlruntime.Manager
client *dynamic.DynamicClient

// Internal configurations
isGatewayAPIInstalled bool
isEnvoyGatewayInstalled bool
isIstioInstalled bool
isCertManagerInstalled bool
isConsolePluginInstalled bool
}

func (b *BootOptionsBuilder) getOptions() []controller.ControllerOption {
var opts []controller.ControllerOption
opts = append(opts, b.getGatewayAPIOptions()...)
opts = append(opts, b.getIstioOptions()...)
opts = append(opts, b.getEnvoyGatewayOptions()...)
opts = append(opts, b.getCertManagerOptions()...)
opts = append(opts, b.getConsolePluginOptions()...)

return opts
}

func (b *BootOptionsBuilder) getGatewayAPIOptions() []controller.ControllerOption {
var opts []controller.ControllerOption
var err error
b.isGatewayAPIInstalled, err = kuadrantgatewayapi.IsGatewayAPIInstalled(b.manager.GetRESTMapper())
if err != nil || !b.isGatewayAPIInstalled {
b.logger.Info("gateway api is not installed, skipping watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
opts = append(opts,
controller.WithRunnable("gatewayclass watcher", controller.Watch(
&gwapiv1.GatewayClass{},
controller.GatewayClassesResource,
Expand All @@ -135,11 +185,17 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
)
}

isEnvoyGatewayInstalled, err := envoygateway.IsEnvoyGatewayInstalled(manager.GetRESTMapper())
if err != nil || !isEnvoyGatewayInstalled {
logger.Info("envoygateway is not installed, skipping related watches and reconcilers", "err", err)
return opts
}

func (b *BootOptionsBuilder) getEnvoyGatewayOptions() []controller.ControllerOption {
var opts []controller.ControllerOption
var err error
b.isEnvoyGatewayInstalled, err = envoygateway.IsEnvoyGatewayInstalled(b.manager.GetRESTMapper())
if err != nil || !b.isEnvoyGatewayInstalled {
b.logger.Info("envoygateway is not installed, skipping related watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
opts = append(opts,
controller.WithRunnable("envoypatchpolicy watcher", controller.Watch(
&egv1alpha1.EnvoyPatchPolicy{},
envoygateway.EnvoyPatchPoliciesResource,
Expand All @@ -165,11 +221,17 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
// TODO: add specific tasks to workflow
}

isIstioInstalled, err := istio.IsIstioInstalled(manager.GetRESTMapper())
if err != nil || !isIstioInstalled {
logger.Info("istio is not installed, skipping related watches and reconcilers", "err", err)
return opts
}

func (b *BootOptionsBuilder) getIstioOptions() []controller.ControllerOption {
var opts []controller.ControllerOption
var err error
b.isIstioInstalled, err = istio.IsIstioInstalled(b.manager.GetRESTMapper())
if err != nil || !b.isIstioInstalled {
b.logger.Info("istio is not installed, skipping related watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
opts = append(opts,
controller.WithRunnable("envoyfilter watcher", controller.Watch(
&istioclientnetworkingv1alpha3.EnvoyFilter{},
istio.EnvoyFiltersResource,
Expand All @@ -195,76 +257,125 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
// TODO: add istio specific tasks to workflow
}

isCertManagerInstalled, err := kuadrantgatewayapi.IsCertManagerInstalled(manager.GetRESTMapper(), logger)
if err != nil || !isCertManagerInstalled {
logger.Info("cert manager is not installed, skipping related watches and reconcilers", "err", err)
return opts
}

func (b *BootOptionsBuilder) getCertManagerOptions() []controller.ControllerOption {
var opts []controller.ControllerOption
var err error
b.isCertManagerInstalled, err = kuadrantgatewayapi.IsCertManagerInstalled(b.manager.GetRESTMapper(), b.logger)
if err != nil || !b.isCertManagerInstalled {
b.logger.Info("cert manager is not installed, skipping related watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
controller.WithRunnable("certificate watcher", controller.Watch(
&certmanagerv1.Certificate{},
CertManagerCertificatesResource,
metav1.NamespaceAll,
)),
controller.WithRunnable("issuers watcher", controller.Watch(
&certmanagerv1.Issuer{},
CertManagerIssuersResource,
metav1.NamespaceAll,
)),
controller.WithRunnable("clusterissuers watcher", controller.Watch(
&certmanagerv1.Certificate{},
CertMangerClusterIssuersResource,
metav1.NamespaceAll,
)),
controller.WithObjectKinds(
CertManagerCertificateKind,
CertManagerIssuerKind,
CertManagerClusterIssuerKind,
),
// TODO: add object links
)
// TODO: add tls policy specific tasks to workflow
opts = append(opts, certManagerControllerOpts()...)
}

isConsolePluginInstalled, err := openshift.IsConsolePluginInstalled(manager.GetRESTMapper())
if err != nil || !isConsolePluginInstalled {
logger.Info("console plugin is not installed, skipping related watches and reconcilers", "err", err)
return opts
}

func (b *BootOptionsBuilder) getConsolePluginOptions() []controller.ControllerOption {
var opts []controller.ControllerOption
var err error
b.isConsolePluginInstalled, err = openshift.IsConsolePluginInstalled(b.manager.GetRESTMapper())
if err != nil || !b.isConsolePluginInstalled {
b.logger.Info("console plugin is not installed, skipping related watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
opts = append(opts,
controller.WithRunnable("consoleplugin watcher", controller.Watch(
&consolev1.ConsolePlugin{}, openshift.ConsolePluginsResource, metav1.NamespaceAll,
controller.FilterResourcesByLabel[*consolev1.ConsolePlugin](fmt.Sprintf("%s=%s", consoleplugin.AppLabelKey, consoleplugin.AppLabelValue)))),
controller.WithObjectKinds(openshift.ConsolePluginGVK.GroupKind()),
)
}

controllerOpts = append(controllerOpts, controller.WithReconcile(buildReconciler(manager, client, isIstioInstalled, isEnvoyGatewayInstalled, isConsolePluginInstalled)))

return controller.NewController(controllerOpts...)
return opts
}

func buildReconciler(manager ctrlruntime.Manager, client *dynamic.DynamicClient, isIstioInstalled, isEnvoyGatewayInstalled, isConsolePluginInstalled bool) controller.ReconcileFunc {
func (b *BootOptionsBuilder) Reconciler() controller.ReconcileFunc {
mainWorkflow := &controller.Workflow{
Precondition: initWorkflow(client).Run,
Precondition: initWorkflow(b.client).Run,
Tasks: []controller.ReconcileFunc{
NewAuthorinoReconciler(client).Subscription().Reconcile,
NewLimitadorReconciler(client).Subscription().Reconcile,
NewAuthorinoReconciler(b.client).Subscription().Reconcile,
NewLimitadorReconciler(b.client).Subscription().Reconcile,
NewDNSWorkflow().Run,
NewTLSWorkflow().Run,
NewTLSWorkflow(b.client, b.isCertManagerInstalled).Run,
NewAuthWorkflow().Run,
NewRateLimitWorkflow().Run,
},
Postcondition: finalStepsWorkflow(client, isIstioInstalled, isEnvoyGatewayInstalled).Run,
Postcondition: finalStepsWorkflow(b.client, b.isIstioInstalled, b.isGatewayAPIInstalled).Run,
}

if isConsolePluginInstalled {
if b.isConsolePluginInstalled {
mainWorkflow.Tasks = append(mainWorkflow.Tasks,
NewConsolePluginReconciler(manager, operatorNamespace).Subscription().Reconcile,
NewConsolePluginReconciler(b.manager, operatorNamespace).Subscription().Reconcile,
)
}

return mainWorkflow.Run
}

func certManagerControllerOpts() []controller.ControllerOption {
isCertificateOwnedByTLSPolicy := func(c *certmanagerv1.Certificate) bool {
return isObjectOwnedByGroupKind(c, kuadrantv1alpha1.TLSPolicyGroupKind)
}

return []controller.ControllerOption{
controller.WithRunnable("certificate watcher", controller.Watch(
&certmanagerv1.Certificate{},
CertManagerCertificatesResource,
metav1.NamespaceAll,
controller.WithPredicates(ctrlruntimepredicate.TypedFuncs[*certmanagerv1.Certificate]{
CreateFunc: func(e event.TypedCreateEvent[*certmanagerv1.Certificate]) bool {
return isCertificateOwnedByTLSPolicy(e.Object)
},
UpdateFunc: func(e event.TypedUpdateEvent[*certmanagerv1.Certificate]) bool {
return isCertificateOwnedByTLSPolicy(e.ObjectNew)
},
DeleteFunc: func(e event.TypedDeleteEvent[*certmanagerv1.Certificate]) bool {
return isCertificateOwnedByTLSPolicy(e.Object)
},
GenericFunc: func(e event.TypedGenericEvent[*certmanagerv1.Certificate]) bool {
return isCertificateOwnedByTLSPolicy(e.Object)
},
})),
Comment on lines +327 to +340
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting approach. I wonder how it performs compared to labels.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I went with ownerReference instead since we already set the ownerRef to point to TLSPolicy since we want the associated Certificates deleted when the TLSPolicy is deleted.

Also we don't currently set a common label across all created Certificate's

kuadrant-operator/controllers/tlspolicy_certmanager_certificates.go

Lines 161 to 184 in 4793dc4

func commonTLSCertificateLabels(gwKey client.ObjectKey, p *v1alpha1.TLSPolicy) map[string]string {
common := map[string]string{}
for k, v := range policyTLSCertificateLabels(p) {
common[k] = v
}
for k, v := range gatewayTLSCertificateLabels(gwKey) {
common[k] = v
}
return common
}
func policyTLSCertificateLabels(p *v1alpha1.TLSPolicy) map[string]string {
return map[string]string{
p.DirectReferenceAnnotationName(): p.Name,
fmt.Sprintf("%s-namespace", p.DirectReferenceAnnotationName()): p.Namespace,
}
}
func gatewayTLSCertificateLabels(gwKey client.ObjectKey) map[string]string {
return map[string]string{
"gateway-namespace": gwKey.Namespace,
"gateway": gwKey.Name,
}
}

),
controller.WithRunnable("issuers watcher", controller.Watch(
&certmanagerv1.Issuer{},
CertManagerIssuersResource,
metav1.NamespaceAll,
controller.WithPredicates(ctrlruntimepredicate.TypedFuncs[*certmanagerv1.Issuer]{
UpdateFunc: func(e event.TypedUpdateEvent[*certmanagerv1.Issuer]) bool {
oldStatus := e.ObjectOld.GetStatus()
newStatus := e.ObjectOld.GetStatus()
return !reflect.DeepEqual(oldStatus, newStatus)
},
})),
),
controller.WithRunnable("clusterissuers watcher", controller.Watch(
&certmanagerv1.ClusterIssuer{},
CertMangerClusterIssuersResource,
metav1.NamespaceAll,
controller.WithPredicates(ctrlruntimepredicate.TypedFuncs[*certmanagerv1.ClusterIssuer]{
UpdateFunc: func(e event.TypedUpdateEvent[*certmanagerv1.ClusterIssuer]) bool {
oldStatus := e.ObjectOld.GetStatus()
newStatus := e.ObjectOld.GetStatus()
return !reflect.DeepEqual(oldStatus, newStatus)
},
})),
),
controller.WithObjectKinds(
CertManagerCertificateKind,
CertManagerIssuerKind,
CertManagerClusterIssuerKind,
),
controller.WithObjectLinks(
LinkListenerToCertificateFunc,
LinkGatewayToIssuerFunc,
LinkGatewayToClusterIssuerFunc,
),
}
}

func initWorkflow(client *dynamic.DynamicClient) *controller.Workflow {
return &controller.Workflow{
Precondition: NewEventLogger().Log,
Expand Down Expand Up @@ -319,3 +430,18 @@ func GetOldestKuadrant(kuadrants []*kuadrantv1beta1.Kuadrant) (*kuadrantv1beta1.
}
return oldest, nil
}

func isObjectOwnedByGroupKind(o client.Object, groupKind schema.GroupKind) bool {
for _, o := range o.GetOwnerReferences() {
oGV, err := schema.ParseGroupVersion(o.APIVersion)
if err != nil {
return false
}

if oGV.Group == groupKind.Group && o.Kind == groupKind.Kind {
return true
}
}

return false
}
Loading
Loading