First part implementation of shoot migration for firewalls (#308)

go.mod

@@ -35,6 +35,7 @@ require (
 	k8s.io/component-base v0.26.3
 	k8s.io/kubelet v0.26.3
 	sigs.k8s.io/controller-runtime v0.14.6
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
@@ -161,7 +162,6 @@ require (
 	sigs.k8s.io/controller-tools v0.11.3 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace (

pkg/controller/infrastructure/actuator_delete.go

@@ -16,9 +16,12 @@ import (
 	"github.com/metal-stack/metal-go/api/models"
 
 	extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller"
+	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	"github.com/gardener/gardener/pkg/controllerutils/reconciler"
 
-	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type networkDeleter struct {
@@ -70,6 +73,33 @@ func (a *actuator) Delete(ctx context.Context, logger logr.Logger, infrastructur
 		}
 	}
 
+	// the valuesprovider is unable to cleanup the mutating and validating webhooks
+	// because these are not namespaced and the names are determined at runtime
+	//
+	// so we clean it up here after control plane has terminated.
+
+	name := "firewall-controller-manager-" + cluster.ObjectMeta.Name
+
+	mwc := &admissionregistrationv1.MutatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	}
+	err = a.client.Delete(ctx, mwc)
+	if client.IgnoreNotFound(err) != nil {
+		return fmt.Errorf("unable to cleanup firewall-controller-manager mutating webhook")
+	}
+
+	vwc := &admissionregistrationv1.ValidatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	}
+	err = a.client.Delete(ctx, vwc)
+	if client.IgnoreNotFound(err) != nil {
+		return fmt.Errorf("unable to cleanup firewall-controller-manager validating webhook")
+	}
+
 	return nil
 }
 

pkg/controller/worker/actuator.go

@@ -2,6 +2,8 @@ package worker
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"github.com/gardener/gardener/extensions/pkg/util"
 
@@ -12,7 +14,11 @@ import (
 	apismetal "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal"
 	"github.com/metal-stack/gardener-extension-provider-metal/pkg/imagevector"
 	"github.com/metal-stack/gardener-extension-provider-metal/pkg/metal"
+	metalclient "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal/client"
 	metalv1alpha1 "github.com/metal-stack/machine-controller-manager-provider-metal/pkg/provider/migration/legacy-api/machine/v1alpha1"
+	metalgo "github.com/metal-stack/metal-go"
+	"github.com/metal-stack/metal-go/api/models"
+	"github.com/metal-stack/metal-lib/pkg/cache"
 
 	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	gardener "github.com/gardener/gardener/pkg/client/kubernetes"
@@ -26,140 +32,200 @@ import (
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
 )
 
-type delegateFactory struct {
-	logger logr.Logger
+type (
+	// actuator reconciles the cluster's worker nodes and the firewalls.
+	// we are wrapping gardener's worker actuator here because we need to intercept the migrate call from the actuator.
+	// unfortunately, there is no callback provided which we could use for this.
+	//
+	// why is the firewall reconciliation here and not in the controlplane controller?
+	//
+	// the controlplane controller deploys the firewall-controller-manager including validating and mutating webhooks
+	// this has to be running before we can create a firewall deployment because the mutating webhook is creating the userdata
+	// the worker controller acts after the controlplane controller, also the terms and responsibilities are pretty similar between machine-controller-manager and firewall-controller-manager,
+	// so this place seems to be a valid fit.
+	actuator struct {
+		controllerConfig config.ControllerConfiguration
+
+		workerActuator worker.Actuator
+
+		networkCache *cache.Cache[*cacheKey, *models.V1NetworkResponse]
+
+		restConfig *rest.Config
+		client     client.Client
+		scheme     *runtime.Scheme
+		decoder    runtime.Decoder
+	}
 
-	restConfig *rest.Config
+	delegateFactory struct {
+		controllerConfig config.ControllerConfiguration
 
-	client  client.Client
-	scheme  *runtime.Scheme
-	decoder runtime.Decoder
+		clientGetter func() (*rest.Config, client.Client, *runtime.Scheme, runtime.Decoder)
+		dataGetter   func(ctx context.Context, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) (*additionalData, error)
 
-	machineImageMapping []config.MachineImage
-	controllerConfig    config.ControllerConfiguration
-}
+		machineImageMapping []config.MachineImage
+	}
 
-// NewActuator creates a new Actuator that updates the status of the handled WorkerPoolConfigs.
-func NewActuator(machineImages []config.MachineImage, controllerConfig config.ControllerConfiguration) worker.Actuator {
-	delegateFactory := &delegateFactory{
-		logger:              log.Log.WithName("worker-actuator"),
-		machineImageMapping: machineImages,
-		controllerConfig:    controllerConfig,
+	workerDelegate struct {
+		logger logr.Logger
+
+		client  client.Client
+		scheme  *runtime.Scheme
+		decoder runtime.Decoder
+
+		machineImageMapping []config.MachineImage
+		seedChartApplier    gardener.ChartApplier
+		serverVersion       string
+
+		cluster *extensionscontroller.Cluster
+		worker  *extensionsv1alpha1.Worker
+
+		machineClasses     []map[string]interface{}
+		machineDeployments worker.MachineDeployments
+		machineImages      []apismetal.MachineImage
+
+		controllerConfig config.ControllerConfiguration
+		additionalData   *additionalData
 	}
-	return genericactuator.NewActuator(
-		delegateFactory,
-		metal.MachineControllerManagerName,
-		mcmChart,
-		mcmShootChart,
-		imagevector.ImageVector(),
-		extensionscontroller.ChartRendererFactoryFunc(util.NewChartRendererForShoot),
-	)
+)
+
+func (a *actuator) InjectFunc(f inject.Func) error {
+	return f(a.workerActuator)
 }
 
-func (d *delegateFactory) InjectScheme(scheme *runtime.Scheme) error {
+func (a *actuator) InjectScheme(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(metalv1alpha1.SchemeGroupVersion,
 		&metalv1alpha1.MetalMachineClass{},
 		&metalv1alpha1.MetalMachineClassList{},
 	)
-	d.scheme = scheme
-	d.decoder = serializer.NewCodecFactory(scheme).UniversalDecoder()
+	a.scheme = scheme
+	a.decoder = serializer.NewCodecFactory(scheme).UniversalDecoder()
 	return nil
 }
 
-func (d *delegateFactory) InjectConfig(restConfig *rest.Config) error {
-	d.restConfig = restConfig
+func (a *actuator) InjectClient(client client.Client) error {
+	a.client = client
 	return nil
 }
 
-func (d *delegateFactory) InjectClient(client client.Client) error {
-	d.client = client
+func (a *actuator) InjectConfig(restConfig *rest.Config) error {
+	a.restConfig = restConfig
 	return nil
 }
 
-func (d *delegateFactory) WorkerDelegate(ctx context.Context, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) (genericactuator.WorkerDelegate, error) {
-	clientset, err := kubernetes.NewForConfig(d.restConfig)
-	if err != nil {
-		return nil, err
+func NewActuator(machineImages []config.MachineImage, controllerConfig config.ControllerConfiguration) worker.Actuator {
+	a := &actuator{
+		controllerConfig: controllerConfig,
+		networkCache: cache.New(15*time.Minute, func(ctx context.Context, accessor *cacheKey) (*models.V1NetworkResponse, error) {
+			mclient, ok := ctx.Value(ClientKey).(metalgo.Client)
+			if !ok {
+				return nil, fmt.Errorf("no client passed in context")
+			}
+			privateNetwork, err := metalclient.GetPrivateNetworkFromNodeNetwork(ctx, mclient, accessor.projectID, accessor.nodeCIDR)
+			if err != nil {
+				return nil, err
+			}
+			return privateNetwork, nil
+		}),
 	}
 
-	serverVersion, err := clientset.Discovery().ServerVersion()
-	if err != nil {
-		return nil, err
+	delegateFactory := &delegateFactory{
+		clientGetter: func() (*rest.Config, client.Client, *runtime.Scheme, runtime.Decoder) {
+			return a.restConfig, a.client, a.scheme, a.decoder
+		},
+		dataGetter:          a.getAdditionalData,
+		controllerConfig:    controllerConfig,
+		machineImageMapping: machineImages,
 	}
 
-	seedChartApplier, err := gardener.NewChartApplierForConfig(d.restConfig)
+	a.workerActuator = genericactuator.NewActuator(
+		delegateFactory,
+		metal.MachineControllerManagerName,
+		mcmChart,
+		mcmShootChart,
+		imagevector.ImageVector(),
+		extensionscontroller.ChartRendererFactoryFunc(util.NewChartRendererForShoot),
+	)
+
+	return a
+}
+
+func (a *actuator) Reconcile(ctx context.Context, log logr.Logger, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) error {
+	err := a.firewallReconcile(ctx, log, worker, cluster)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	return NewWorkerDelegate(
-		d.logger,
-		d.client,
-		d.scheme,
-		d.decoder,
+	return a.workerActuator.Reconcile(ctx, log, worker, cluster)
+}
 
-		d.machineImageMapping,
-		seedChartApplier,
-		serverVersion.GitVersion,
+func (a *actuator) Delete(ctx context.Context, log logr.Logger, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) error {
+	err := a.workerActuator.Delete(ctx, log, worker, cluster)
+	if err != nil {
+		return err
+	}
 
-		worker,
-		cluster,
-		d.controllerConfig,
-	), nil
+	return a.firewallDelete(ctx, log, cluster)
 }
 
-type workerDelegate struct {
-	logger logr.Logger
+func (a *actuator) Migrate(ctx context.Context, log logr.Logger, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) error {
+	err := a.workerActuator.Migrate(ctx, log, worker, cluster)
+	if err != nil {
+		return err
+	}
 
-	client  client.Client
-	scheme  *runtime.Scheme
-	decoder runtime.Decoder
+	return a.firewallMigrate(ctx, log, cluster)
+}
 
-	machineImageMapping []config.MachineImage
-	seedChartApplier    gardener.ChartApplier
-	serverVersion       string
+func (a *actuator) Restore(ctx context.Context, log logr.Logger, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) error {
+	err := a.firewallRestore(ctx, log, worker, cluster)
+	if err != nil {
+		return err
+	}
 
-	cluster *extensionscontroller.Cluster
-	worker  *extensionsv1alpha1.Worker
+	return a.workerActuator.Restore(ctx, log, worker, cluster)
+}
 
-	machineClasses     []map[string]interface{}
-	machineDeployments worker.MachineDeployments
-	machineImages      []apismetal.MachineImage
+func (d *delegateFactory) WorkerDelegate(ctx context.Context, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) (genericactuator.WorkerDelegate, error) {
+	config, client, scheme, decoder := d.clientGetter()
 
-	controllerConfig config.ControllerConfiguration
-}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
 
-// NewWorkerDelegate creates a new context for a worker reconciliation.
-func NewWorkerDelegate(
-	logger logr.Logger,
-	client client.Client,
-	scheme *runtime.Scheme,
-	decoder runtime.Decoder,
+	serverVersion, err := clientset.Discovery().ServerVersion()
+	if err != nil {
+		return nil, err
+	}
 
-	machineImageMapping []config.MachineImage,
-	seedChartApplier gardener.ChartApplier,
-	serverVersion string,
+	seedChartApplier, err := gardener.NewChartApplierForConfig(config)
+	if err != nil {
+		return nil, err
+	}
 
-	worker *extensionsv1alpha1.Worker,
-	cluster *extensionscontroller.Cluster,
-	controllerConfig config.ControllerConfiguration,
+	additionalData, err := d.dataGetter(ctx, worker, cluster)
+	if err != nil {
+		return nil, err
+	}
 
-) genericactuator.WorkerDelegate {
 	return &workerDelegate{
-		logger:  logger,
+		logger: log.Log.WithName("metal-worker-delegate").WithValues("cluster", cluster.ObjectMeta.Name),
+
 		client:  client,
 		scheme:  scheme,
 		decoder: decoder,
 
-		machineImageMapping: machineImageMapping,
+		machineImageMapping: d.machineImageMapping,
 		seedChartApplier:    seedChartApplier,
-		serverVersion:       serverVersion,
+		serverVersion:       serverVersion.GitVersion,
 
 		cluster: cluster,
 		worker:  worker,
 
-		controllerConfig: controllerConfig,
-	}
+		controllerConfig: d.controllerConfig,
+		additionalData:   additionalData,
+	}, nil
 }