Skip to content

Commit

Permalink
fix: revert again and test in cluster
Browse files Browse the repository at this point in the history 
Signed-off-by: Tom Plant <[email protected]>
  • Loading branch information
@pl4nty
pl4nty committed Jan 27, 2025
1 parent 6d0a715 commit c504cf6
Showing 1 changed file with 59 additions and 99 deletions.
158 changes: 59 additions & 99 deletions internal/controller/gateway_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/cloudflare/cloudflare-go/v2"
"github.com/cloudflare/cloudflare-go/v2/shared"
"github.com/cloudflare/cloudflare-go/v2/zero_trust"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -19,7 +20,6 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/tools/record"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
Expand All @@ -29,25 +29,20 @@ import (
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
)

const (
gatewayClassFinalizer = "cfargotunnel.com/finalizer"
gatewayFinalizer = "cfargotunnel.com/finalizer"
controllerName = "github.com/pl4nty/cloudflare-kubernetes-gateway"
)
const gatewayClassFinalizer = "cfargotunnel.com/finalizer"
const gatewayFinalizer = "cfargotunnel.com/finalizer"
const controllerName = "github.com/pl4nty/cloudflare-kubernetes-gateway"

// Definitions to manage status conditions
const (
// typeAvailableGateway represents the status of the Deployment reconciliation
typeAvailableGateway = string(gatewayv1.GatewayConditionAccepted)
// typeDegradedGateway represents the status used when the custom resource is deleted and the finalizer operations are yet to occur.
typeDegradedGateway = string(gatewayv1.GatewayConditionAccepted)
)

// GatewayReconciler reconciles a Gateway object
type GatewayReconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder record.EventRecorder
Scheme *runtime.Scheme

Check failure on line 44 in internal/controller/gateway_controller.go

View workflow job for this annotation

GitHub Actions / Run on Ubuntu 

File is not properly formatted (gofmt)
Recorder record.EventRecorder
}

// The following markers are used to generate the rules permissions (RBAC) on config/rbac using controller-gen
Expand All @@ -56,11 +51,11 @@ type GatewayReconciler struct {

// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=get;update
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/finalizers,verbs=update
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get;list;watch;update
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/status,verbs=update
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get;list;update;watch
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/finalizers,verbs=update
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/status,verbs=update
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;get;list;update;watch;delete
// +kubebuilder:rbac:groups=core,resources=events,verbs=create
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;delete
// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get
// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch

Expand All @@ -74,17 +69,17 @@ type GatewayReconciler struct {
// For further info:
// - About Operator Pattern: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/
// - About Controllers: https://kubernetes.io/docs/concepts/architecture/controller/
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
// nolint:gocyclo
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
//
//nolint:gocyclo
func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := log.FromContext(ctx)

// Fetch the Gateway instance
// The purpose is check if the Custom Resource for the Kind Gateway
// is applied on the cluster if not we return nil to stop the reconciliation
gateway := &gatewayv1.Gateway{}
err := r.Get(ctx, req.NamespacedName, gateway)
if err != nil {
if err := r.Get(ctx, req.NamespacedName, gateway); err != nil {
if apierrors.IsNotFound(err) {
// If the custom resource is not found then it usually means that it was deleted or not created
// In this way, we will stop the reconciliation
Expand All @@ -96,7 +91,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
return ctrl.Result{}, err
}

// check if parent GatewayClass is ours
// check if parent GatewayClass is ours and update finalizer
gatewayClass := &gatewayv1.GatewayClass{}
if err := r.Get(ctx, types.NamespacedName{Name: string(gateway.Spec.GatewayClassName)}, gatewayClass); err != nil {
if apierrors.IsNotFound(err) {
Expand Down Expand Up @@ -127,43 +122,19 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
}

account, api, err := InitCloudflareApi(ctx, r.Client, gatewayClass.Name)
if err != nil {
log.Error(err, "Failed to load Cloudflare API")
return ctrl.Result{}, err
}

// Let's just set the status as Unknown when no status is available
if len(gateway.Status.Conditions) == 0 {
meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: typeAvailableGateway, Status: metav1.ConditionUnknown, Reason: "Reconciling", Message: "Starting reconciliation"})
if err = r.Status().Update(ctx, gateway); err != nil {
log.Error(err, "Failed to update Gateway status")
return ctrl.Result{}, err
}

// Let's re-fetch the gateway Custom Resource after updating the status
// so that we have the latest state of the resource on the cluster and we will avoid
// raising the error "the object has been modified, please apply
// your changes to the latest version and try again" which would re-trigger the reconciliation
// if we try to update it again in the following operations
if err := r.Get(ctx, req.NamespacedName, gateway); err != nil {
log.Error(err, "Failed to re-fetch gateway")
return ctrl.Result{}, err
}
}

// Let's add a finalizer. Then, we can define some operations which should
// occur before the custom resource is deleted.
// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers
if !controllerutil.ContainsFinalizer(gateway, gatewayFinalizer) {
log.Info("Adding Finalizer for Gateway")
if ok := controllerutil.AddFinalizer(gateway, gatewayFinalizer); !ok {
err = fmt.Errorf("finalizer for Gateway was not added")
log.Error(err, "Failed to add finalizer for Gateway")
return ctrl.Result{}, err
log.Error(nil, "Failed to add finalizer into the Gateway")
return ctrl.Result{Requeue: true}, nil
}

if err = r.Update(ctx, gateway); err != nil {
log.Error(err, "Failed to update custom resource to add finalizer")
if err := r.Update(ctx, gateway); err != nil {
log.Error(err, "Failed to update Gateway to add finalizer")
return ctrl.Result{}, err
}
}
Expand All @@ -176,18 +147,18 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
log.Info("Performing Finalizer Operations for Gateway before delete CR")

// Let's add here a status "Downgrade" to reflect that this resource began its process to be terminated.
meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: typeDegradedGateway,
meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: string(gatewayv1.GatewayConditionAccepted),
Status: metav1.ConditionUnknown, Reason: string(gatewayv1.GatewayReasonPending), ObservedGeneration: gateway.Generation,
Message: fmt.Sprintf("Performing finalizer operations for the custom resource: %s ", gateway.Name)})
Message: fmt.Sprintf("Performing finalizer operations for the Gateway: %s ", gateway.Name)})

if err := r.Status().Update(ctx, gateway); err != nil {
log.Error(err, "Failed to update Gateway status")
log.Error(err, "Failed to update Gateway finalizer status")
return ctrl.Result{}, err
}

// Perform all operations required before removing the finalizer and allow
// the Kubernetes API to remove the custom resource.
if err := r.doFinalizerOperationsForGateway(gateway, ctx, gatewayClass, account, api); err != nil {
if err := r.doFinalizerOperationsForGateway(ctx, gatewayClass, gateway, account, api); err != nil {
log.Error(err, "Failed to complete finalizer operations for Gateway")
return ctrl.Result{Requeue: true}, nil
}
Expand All @@ -201,20 +172,19 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
return ctrl.Result{}, err
}

meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: typeDegradedGateway,
meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: string(gatewayv1.GatewayConditionAccepted),
Status: metav1.ConditionTrue, Reason: "Finalizing", ObservedGeneration: gateway.Generation,
Message: fmt.Sprintf("Finalizer operations for custom resource %s name were successfully accomplished", gateway.Name)})

if err := r.Status().Update(ctx, gateway); err != nil {
log.Error(err, "Failed to update Gateway status")
log.Error(err, "Failed to update Gateway finalizer status")
return ctrl.Result{}, err
}

log.Info("Removing Finalizer for Gateway after successfully perform the operations")
if ok := controllerutil.RemoveFinalizer(gateway, gatewayFinalizer); !ok {
err = fmt.Errorf("finalizer for Gateway was not removed")
log.Error(err, "Failed to remove finalizer for Gateway")
return ctrl.Result{}, err
return ctrl.Result{Requeue: true}, nil
}

if err := r.Update(ctx, gateway); err != nil {
Expand Down Expand Up @@ -333,14 +303,13 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: string(gatewayv1.GatewayConditionAccepted),
Status: metav1.ConditionFalse, Reason: string(gatewayv1.GatewayReasonListenersNotValid), ObservedGeneration: gateway.Generation,
Message: fmt.Sprintf("No valid listeners for gateway (%s)", gateway.Name)})
log.Info("No valid listeners for gateway", "gateway", gateway.Name)

if err := r.Status().Update(ctx, gateway); err != nil {
if strings.Contains(err.Error(), "apply your changes to the latest version and try again") {
log.Info("Conflict when updating Gateway status, retrying", "error", err.Error())
log.Info("Conflict when updating Gateway listener status, retrying")
return ctrl.Result{Requeue: true}, nil
} else {
log.Error(err, "Failed to update Gateway status")
log.Error(err, "Failed to update Gateway listener status")
return ctrl.Result{}, err
}
}
Expand All @@ -353,10 +322,10 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct

if err := r.Status().Update(ctx, gateway); err != nil {
if strings.Contains(err.Error(), "apply your changes to the latest version and try again") {
log.Info("Conflict when updating Gateway status, retrying", "error", err.Error())
log.Info("Conflict when updating Gateway listener status, retrying")
return ctrl.Result{Requeue: true}, nil
} else {
log.Error(err, "Failed to update Gateway status")
log.Error(err, "Failed to update Gateway listener status")
return ctrl.Result{}, err
}
}
Expand All @@ -368,7 +337,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
})
if err != nil {
if strings.Contains(err.Error(), "429 Too Many Requests") {
log.Info("Rate limited by Cloudflare, requeueing after 10 minutes")
log.Error(err, "Rate limited, requeueing after 10 minutes")
return ctrl.Result{
RequeueAfter: time.Minute * 10, // https://developers.cloudflare.com/fundamentals/api/reference/limits/
}, nil
Expand Down Expand Up @@ -439,19 +408,14 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
return ctrl.Result{}, err
}

if err := r.Get(ctx, req.NamespacedName, gateway); err != nil {
log.Error(err, "Failed to re-fetch gateway")
return ctrl.Result{}, err
}

// The following implementation will update the status
meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: string(gatewayv1.GatewayConditionProgrammed),
Status: metav1.ConditionTrue, Reason: string(gatewayv1.GatewayReasonProgrammed), ObservedGeneration: gateway.Generation,
Message: fmt.Sprintf("Tunnel and deployment for gateway (%s) reconciled successfully", gateway.Name)})

if err := r.Status().Update(ctx, gateway); err != nil {
if strings.Contains(err.Error(), "apply your changes to the latest version and try again") {
log.Info("Conflict when updating Gateway status, retrying", "error", err.Error())
log.Info("Conflict when updating Gateway listener status, retrying")
return ctrl.Result{Requeue: true}, nil
} else {
log.Error(err, "Failed to update Gateway status")
Expand Down Expand Up @@ -552,13 +516,13 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
}

// The following implementation will update the status
meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: typeAvailableGateway,
meta.SetStatusCondition(&gateway.Status.Conditions, metav1.Condition{Type: string(gatewayv1.GatewayConditionProgrammed),
Status: metav1.ConditionTrue, Reason: string(gatewayv1.GatewayReasonProgrammed), ObservedGeneration: gateway.Generation,
Message: fmt.Sprintf("Deployment for custom resource (%s) reconciled successfully", gateway.Name)})
Message: fmt.Sprintf("Tunnel and deployment for gateway (%s) reconciled successfully", gateway.Name)})

if err := r.Status().Update(ctx, gateway); err != nil {
if strings.Contains(err.Error(), "apply your changes to the latest version and try again") {
log.Info("Conflict when updating Gateway status, retrying", "error", err.Error())
log.Info("Conflict when updating Gateway listener status, retrying")
return ctrl.Result{Requeue: true}, nil
} else {
log.Error(err, "Failed to update Gateway status")
Expand All @@ -570,7 +534,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
}

// finalizeGateway will perform the required operations before delete the CR.
func (r *GatewayReconciler) doFinalizerOperationsForGateway(cr *gatewayv1.Gateway, ctx context.Context, gatewayClass *gatewayv1.GatewayClass, account string, api *cloudflare.Client) error {
func (r *GatewayReconciler) doFinalizerOperationsForGateway(ctx context.Context, gatewayClass *gatewayv1.GatewayClass, gateway *gatewayv1.Gateway, account string, api *cloudflare.Client) error {
// Note: It is not recommended to use finalizers with the purpose of deleting resources which are
// created and managed in the reconciliation. These ones, such as the Deployment created on this reconcile,
// are defined as dependent of the custom resource. See that we use the method ctrl.SetControllerReference.
Expand All @@ -582,7 +546,7 @@ func (r *GatewayReconciler) doFinalizerOperationsForGateway(cr *gatewayv1.Gatewa
tunnel, err := api.ZeroTrust.Tunnels.List(ctx, zero_trust.TunnelListParams{
AccountID: cloudflare.String(account),
IsDeleted: cloudflare.Bool(false),
Name: cloudflare.String(cr.Name),
Name: cloudflare.String(gateway.Name),
})
if err != nil {
log.Error(err, "Failed to get tunnel from Cloudflare API")
Expand Down Expand Up @@ -618,7 +582,7 @@ func (r *GatewayReconciler) doFinalizerOperationsForGateway(cr *gatewayv1.Gatewa
}

// if GatewayClass has no other Gateways, remove its finalizer
gateways := &gatewayv1.GatewayList{Items: []gatewayv1.Gateway{{Spec: gatewayv1.GatewaySpec{GatewayClassName: cr.Spec.GatewayClassName}}}}
gateways := &gatewayv1.GatewayList{Items: []gatewayv1.Gateway{{Spec: gatewayv1.GatewaySpec{GatewayClassName: gateway.Spec.GatewayClassName}}}}
if err := r.List(ctx, gateways); err != nil {
log.Error(err, "Failed to list Gateways")
return err
Expand All @@ -631,10 +595,10 @@ func (r *GatewayReconciler) doFinalizerOperationsForGateway(cr *gatewayv1.Gatewa
}

// The following implementation will raise an event
r.Recorder.Event(cr, "Warning", "Deleting",
fmt.Sprintf("Custom Resource %s is being deleted from the namespace %s",
cr.Name,
cr.Namespace))
r.Recorder.Event(gateway, "Warning", "Deleting",
fmt.Sprintf("Gateway %s is being deleted from the namespace %s",
gateway.Name,
gateway.Namespace))

return nil
}
Expand Down Expand Up @@ -689,7 +653,7 @@ func (r *GatewayReconciler) deploymentForGateway(
},
},
SecurityContext: &corev1.PodSecurityContext{
RunAsNonRoot: ptr.To(true),
RunAsNonRoot: &[]bool{true}[0],
// IMPORTANT: seccomProfile was introduced with Kubernetes 1.19
// If you are looking for to produce solutions to be supported
// on lower versions you must remove this option.
Expand All @@ -710,9 +674,9 @@ func (r *GatewayReconciler) deploymentForGateway(
// Ensure restrictive context for the container
// More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
SecurityContext: &corev1.SecurityContext{
RunAsNonRoot: ptr.To(true),
RunAsUser: ptr.To(int64(1001)),
AllowPrivilegeEscalation: ptr.To(false),
RunAsNonRoot: &[]bool{true}[0],
RunAsUser: &[]int64{1001}[0],
AllowPrivilegeEscalation: &[]bool{false}[0],
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{
"ALL",
Expand Down Expand Up @@ -742,10 +706,17 @@ func (r *GatewayReconciler) deploymentForGateway(
// labelsForGateway returns the labels for selecting the resources
// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
func labelsForGateway(name string) map[string]string {
// skip imageTag, to allow version updates to existing deployments
// var imageTag string
// image, err := imageForGateway()
// if err == nil {
// imageTag = strings.Split(image, ":")[1]
// }
return map[string]string{
"app.kubernetes.io/name": "cloudflare-kubernetes-gateway",
"cfargotunnel.com/name": name,
"app.kubernetes.io/name": "cloudflare-kubernetes-gateway",
// "app.kubernetes.io/version": imageTag,
"app.kubernetes.io/managed-by": "GatewayController",
"cfargotunnel.com/name": name,
}
}

Expand All @@ -755,30 +726,19 @@ func imageForGateway() (string, error) {
var imageEnvVar = "GATEWAY_IMAGE"
image, found := os.LookupEnv(imageEnvVar)
if !found {
return "", fmt.Errorf("Unable to find %s environment variable with the image", imageEnvVar)
return "", fmt.Errorf("unable to find %s environment variable with the image", imageEnvVar)
}
return image, nil
}

// SetupWithManager sets up the controller with the Manager.
// The whole idea is to be watching the resources that matter for the controller.
// When a resource that the controller is interested in changes, the Watch triggers
// the controller’s reconciliation loop, ensuring that the actual state of the resource
// matches the desired state as defined in the controller’s logic.
//
// Notice how we configured the Manager to monitor events such as the creation, update,
// or deletion of a Custom Resource (CR) of the Gateway kind, as well as any changes
// to the Deployment that the controller manages and owns.
// Note that the Deployment will be also watched in order to ensure its
// desirable state on the cluster
func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
pred := predicate.GenerationChangedPredicate{}
return ctrl.NewControllerManagedBy(mgr).
// Watch the Gateway CR(s) and trigger reconciliation whenever it
// is created, updated, or deleted
For(&gatewayv1.Gateway{}).
Named("cloudflare-gateway").
// Watch the Deployment managed by the GatewayReconciler. If any changes occur to the Deployment
// owned and managed by this controller, it will trigger reconciliation, ensuring that the cluster
// state aligns with the desired state. See that the ownerRef was set when the Deployment was created.
Owns(&appsv1.Deployment{}).
WithEventFilter(predicate.GenerationChangedPredicate{}).
WithEventFilter(pred).
Complete(r)
}

0 comments on commit c504cf6

Please sign in to comment.