Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add feature to allow dry-run for change plans #89

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Add feature to allow dry-run for change plans #89

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
caea388
Add feature to allow dry-run for change plans
rsafonseca Sep 9, 2024
ce6238b
appease linter
rsafonseca Sep 9, 2024
6c4d618
appease linter
rsafonseca Sep 9, 2024
3023055
remove extraneous logs
rsafonseca Sep 9, 2024
c8023ab
fix description
rsafonseca Sep 10, 2024
e01ce12
Make objects passed into the context clearer
rsafonseca Sep 10, 2024
9056d77
plan paused clusters
rsafonseca Sep 10, 2024
9e7ce60
replace empty block with negated condition
rsafonseca Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 95 additions & 63 deletions controllers/controlplane/kopscontrolplane_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,13 @@ type KopsControlPlaneReconciler struct {
ControllerClass string
Recorder record.EventRecorder
TfExecPath string
DryRun bool
GetKopsClientSetFactory func(configBase string) (simple.Clientset, error)
BuildCloudFactory func(*kopsapi.Cluster) (fi.Cloud, error)
PopulateClusterSpecFactory func(ctx context.Context, kopsCluster *kopsapi.Cluster, kopsClientset simple.Clientset, cloud fi.Cloud) (*kopsapi.Cluster, error)
PrepareKopsCloudResourcesFactory func(ctx context.Context, kopsClientset simple.Clientset, kopsCluster *kopsapi.Cluster, terraformOutputDir string, cloud fi.Cloud) error
ApplyTerraformFactory func(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error
PlanTerraformFactory func(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error
DestroyTerraformFactory func(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error
KopsDeleteResourcesFactory func(ctx context.Context, cloud fi.Cloud, kopsClientset simple.Clientset, kopsCluster *kopsapi.Cluster) error
ValidateKopsClusterFactory func(kubeConfig *rest.Config, kopsCluster *kopsapi.Cluster, cloud fi.Cloud, igs *kopsapi.InstanceGroupList) (*validation.ValidationCluster, error)
Expand All @@ -114,6 +116,9 @@ type KopsControlPlaneReconciliation struct {
kcp *controlplanev1alpha1.KopsControlPlane
}

type KCPKey struct{}
rafatio marked this conversation as resolved.
Show resolved Hide resolved
type ClientKey struct{}

func init() {
// Set kops lib verbosity to ERROR
var log logr.Logger
Expand All @@ -136,6 +141,9 @@ func GetClusterStatus(kopsCluster *kopsapi.Cluster, cloud fi.Cloud) (*kopsapi.Cl
}

func (r *KopsControlPlaneReconciler) shouldDeleteCluster(kcp *controlplanev1alpha1.KopsControlPlane) bool {
if r.DryRun {
return false
}
if !kcp.ObjectMeta.DeletionTimestamp.IsZero() {
if kcp.Annotations[controlplanev1alpha1.ClusterDeleteProtectionAnnotation] == "true" {
r.Recorder.Eventf(kcp, corev1.EventTypeWarning, "ClusterDeleteProtectionEnabled", "cluster delete protection is enabled, skipping deletion")
Expand Down Expand Up @@ -571,8 +579,24 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req

initTime := time.Now()
kopsControlPlane := &controlplanev1alpha1.KopsControlPlane{}
if err := r.Get(ctx, req.NamespacedName, kopsControlPlane); err != nil {
return resultError, client.IgnoreNotFound(err)
var kubeReader client.Reader
if r.DryRun {
cli, ok := ctx.Value(ClientKey{}).(client.Reader)
if !ok {
return ctrl.Result{}, errors.New("failed to get kube reader client object")
}
kubeReader = cli
kcp, ok := ctx.Value(KCPKey{}).(controlplanev1alpha1.KopsControlPlane)
if !ok {
return ctrl.Result{}, errors.New("failed to get kcp object")
} else {
kopsControlPlane = &kcp
}
} else {
kubeReader = r.Client
if err := r.Get(ctx, req.NamespacedName, kopsControlPlane); err != nil {
return resultError, client.IgnoreNotFound(err)
}
}

reconciler := &KopsControlPlaneReconciliation{
Expand Down Expand Up @@ -600,7 +624,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToGetClusterMetadata", "could not get cluster with metadata: %s", err)
return resultError, err
}

var kmps []infrastructurev1alpha1.KopsMachinePool

terraformOutputDir := fmt.Sprintf("/tmp/%s", owner.GetName())
Expand Down Expand Up @@ -640,7 +663,7 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
}
}

if kopsControlPlane.Spec.TerraformConfig.CleanupTerraformDirectory {
if !r.DryRun && kopsControlPlane.Spec.TerraformConfig.CleanupTerraformDirectory {
err := utils.CleanupTerraformDirectory(terraformOutputDir)
if err != nil {
r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedCleanupTerraformDirectory", "failed to cleanup terraform directory from cluster: %s", err)
Expand All @@ -659,22 +682,20 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req

reconciler.log.Info(fmt.Sprintf("finished reconcile loop for %s, took %s", kopsControlPlane.ObjectMeta.GetName(), time.Since(initTime)))
}()

if annotations.HasPaused(owner) {
if !r.DryRun && annotations.HasPaused(owner) {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeNormal, "ClusterPaused", "reconciliation is paused since cluster %s is paused", owner.GetName())
kopsControlPlane.Status.Paused = true
return resultDefault, nil
}
kopsControlPlane.Status.Paused = false
kopsControlPlane.Status.Ready = false

awsCredentials, err := util.GetAWSCredentialsFromKopsControlPlaneSecret(ctx, r.Client, kopsControlPlane.Spec.IdentityRef.Name, kopsControlPlane.Spec.IdentityRef.Namespace)
awsCredentials, err := util.GetAWSCredentialsFromKopsControlPlaneSecret(ctx, kubeReader, kopsControlPlane.Spec.IdentityRef.Name, kopsControlPlane.Spec.IdentityRef.Namespace)
if err != nil {
reconciler.log.Error(err, "failed to get AWS credentials")
return resultError, err
}
reconciler.awsCredentials = *awsCredentials

kopsCluster := &kopsapi.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: owner.GetName(),
Expand All @@ -691,8 +712,7 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
return resultError, fmt.Errorf("error parsing NonMasqueradeCIDR %q: %v", kopsCluster.Spec.Networking.NonMasqueradeCIDR, err)
}
}

kmps, err = kopsutils.GetKopsMachinePoolsWithLabel(ctx, reconciler.Client, "cluster.x-k8s.io/cluster-name", kopsControlPlane.Name)
kmps, err = kopsutils.GetKopsMachinePoolsWithLabel(ctx, kubeReader, "cluster.x-k8s.io/cluster-name", kopsControlPlane.Name)
if err != nil {
return resultError, err
}
Expand Down Expand Up @@ -777,7 +797,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
if !controllerutil.ContainsFinalizer(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneFinalizer) {
controllerutil.AddFinalizer(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneFinalizer)
}

reconciler.Mux.Lock()
shouldUnlock = true

Expand Down Expand Up @@ -854,19 +873,22 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
return resultError, err
}

err = reconciler.createOrUpdateKopsCluster(ctx, kopsClientset, fullCluster, kopsControlPlane.Spec.SSHPublicKey, cloud)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToManageKopsState", "failed to manage Kops state: %s", err)
conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneStateReadyCondition, controlplanev1alpha1.KopsControlPlaneStateReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to create/update KopsCluster: %s", err.Error())
return resultError, err
}
conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneStateReadyCondition)
// Only Apply resources if DryRun isn't set from command line
if !r.DryRun {
err = reconciler.createOrUpdateKopsCluster(ctx, kopsClientset, fullCluster, kopsControlPlane.Spec.SSHPublicKey, cloud)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToManageKopsState", "failed to manage Kops state: %s", err)
conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneStateReadyCondition, controlplanev1alpha1.KopsControlPlaneStateReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to create/update KopsCluster: %s", err.Error())
return resultError, err
}
conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneStateReadyCondition)

err = reconciler.ReconcileKopsSecrets(ctx, kopsClientset, kopsCluster)
if err != nil {
conditions.MarkFalse(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition, controlplanev1alpha1.KopsControlPlaneSecretsReconciliationFailedReason, clusterv1.ConditionSeverityWarning, "failed to reconcile KopsSecrets: %s", err.Error())
err = reconciler.ReconcileKopsSecrets(ctx, kopsClientset, kopsCluster)
if err != nil {
conditions.MarkFalse(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition, controlplanev1alpha1.KopsControlPlaneSecretsReconciliationFailedReason, clusterv1.ConditionSeverityWarning, "failed to reconcile KopsSecrets: %s", err.Error())
}
conditions.MarkTrue(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition)
}
conditions.MarkTrue(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition)

var shouldIgnoreSG bool
if _, ok := owner.GetAnnotations()["kopscontrolplane.controlplane.wildlife.io/external-security-groups"]; ok {
Expand All @@ -889,57 +911,67 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
return resultError, err
}

// TODO: This is needed because we are using a method from kops lib
// we should check alternatives
kubeConfig, err := utils.GetKubeconfigFromKopsState(ctx, kopsCluster, kopsClientset)
if err != nil {
return resultError, err
}
// Only Apply resources if DryRun isn't set from command line
if r.DryRun {
reconciler.log.Info(fmt.Sprintf("planning Terraform for %s", kopsControlPlane.ObjectMeta.GetName()))
err = reconciler.PlanTerraformFactory(ctx, terraformOutputDir, r.TfExecPath, reconciler.awsCredentials)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToPlanTerraform", "failed to plan terraform: %s", err)
return resultError, err
}
} else {
// TODO: This is needed because we are using a method from kops lib
// we should check alternatives
kubeConfig, err := utils.GetKubeconfigFromKopsState(ctx, kopsCluster, kopsClientset)
if err != nil {
return resultError, err
}

err = reconciler.reconcileKubeconfig(ctx, kubeConfig, kopsCluster, owner)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToReconcileKubeconfig", "failed to reconcile kubeconfig: %s", err)
return resultError, err
}
err = reconciler.reconcileKubeconfig(ctx, kubeConfig, kopsCluster, owner)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToReconcileKubeconfig", "failed to reconcile kubeconfig: %s", err)
return resultError, err
}

reconciler.Mux.Unlock()
shouldUnlock = false
reconciler.Mux.Unlock()
shouldUnlock = false

reconciler.log.Info(fmt.Sprintf("applying Terraform for %s", kopsControlPlane.ObjectMeta.GetName()))
reconciler.log.Info(fmt.Sprintf("applying Terraform for %s", kopsControlPlane.ObjectMeta.GetName()))

err = reconciler.ApplyTerraformFactory(ctx, terraformOutputDir, r.TfExecPath, reconciler.awsCredentials)
if err != nil {
conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.TerraformApplyReadyCondition, controlplanev1alpha1.TerraformApplyReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to apply Terraform: %s", err.Error())
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToApplyTerraform", "failed to apply terraform: %s", err)
return resultError, err
}
conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.TerraformApplyReadyCondition)
err = reconciler.ApplyTerraformFactory(ctx, terraformOutputDir, r.TfExecPath, reconciler.awsCredentials)
if err != nil {
conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.TerraformApplyReadyCondition, controlplanev1alpha1.TerraformApplyReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to apply Terraform: %s", err.Error())
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToApplyTerraform", "failed to apply terraform: %s", err)
return resultError, err
}
conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.TerraformApplyReadyCondition)

reconciler.log.Info(fmt.Sprintf("Terraform applied for %s", kopsControlPlane.ObjectMeta.GetName()))
reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "TerraformApplied", "Terraform applied")
reconciler.log.Info(fmt.Sprintf("Terraform applied for %s", kopsControlPlane.ObjectMeta.GetName()))
reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "TerraformApplied", "Terraform applied")

err = reconciler.updateKopsMachinePoolWithProviderIDList(kopsControlPlane, kmps, &reconciler.awsCredentials)
if err != nil {
if apierrors.IsNotFound(err) {
return requeue1min, nil
err = reconciler.updateKopsMachinePoolWithProviderIDList(kopsControlPlane, kmps, &reconciler.awsCredentials)
if err != nil {
if apierrors.IsNotFound(err) {
return requeue1min, nil
}
return resultError, err
}
return resultError, err
}

igList, err := kopsClientset.InstanceGroupsFor(kopsCluster).List(ctx, metav1.ListOptions{})
if err != nil || len(igList.Items) == 0 {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToGetIGs", "cannot get InstanceGroups: %s", err)
return resultError, fmt.Errorf("cannot get InstanceGroups for %q: %w", kopsCluster.ObjectMeta.Name, err)
}
igList, err := kopsClientset.InstanceGroupsFor(kopsCluster).List(ctx, metav1.ListOptions{})
if err != nil || len(igList.Items) == 0 {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToGetIGs", "cannot get InstanceGroups: %s", err)
return resultError, fmt.Errorf("cannot get InstanceGroups for %q: %w", kopsCluster.ObjectMeta.Name, err)
}

val, err := reconciler.ValidateKopsClusterFactory(kubeConfig, kopsCluster, cloud, igList)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToValidateKubernetesCluster", "failed trying to validate Kubernetes cluster: %v", err)
return resultError, err
}
val, err := reconciler.ValidateKopsClusterFactory(kubeConfig, kopsCluster, cloud, igList)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToValidateKubernetesCluster", "failed trying to validate Kubernetes cluster: %v", err)
return resultError, err
}

kopsControlPlane.Status.Ready = utils.KopsClusterValidation(kopsControlPlane, r.Recorder, reconciler.log, val)
reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "ClusterReconciliationFinished", "cluster reconciliation finished")
kopsControlPlane.Status.Ready = utils.KopsClusterValidation(kopsControlPlane, r.Recorder, reconciler.log, val)
reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "ClusterReconciliationFinished", "cluster reconciliation finished")
}
return resultDefault, nil
}

Expand Down
18 changes: 13 additions & 5 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ require (
github.com/crossplane-contrib/provider-aws v0.30.1
github.com/go-logr/logr v1.4.2
github.com/hashicorp/go-version v1.6.0
github.com/hashicorp/hc-install v0.4.0
github.com/hashicorp/terraform-exec v0.17.3
github.com/hashicorp/hc-install v0.6.4
github.com/hashicorp/terraform-exec v0.21.0
github.com/onsi/ginkgo/v2 v2.20.0
github.com/onsi/gomega v1.34.1
github.com/pkg/errors v0.9.1
Expand Down Expand Up @@ -43,6 +43,9 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect
github.com/agext/levenshtein v1.2.1 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.8 // indirect
Expand All @@ -67,6 +70,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.29.1 // indirect
github.com/awslabs/operatorpkg v0.0.0-20240514175841-edb8fe5824b4 // indirect
github.com/cert-manager/cert-manager v1.15.0 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
Expand All @@ -88,6 +92,7 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/go-retryablehttp v0.7.6 // indirect
github.com/hashicorp/hcl/v2 v2.20.1 // indirect
github.com/hetznercloud/hcloud-go v1.56.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand All @@ -96,6 +101,7 @@ require (
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
github.com/moby/term v0.5.0 // indirect
Expand All @@ -121,6 +127,7 @@ require (
go.opentelemetry.io/otel/trace v1.27.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/mod v0.20.0 // indirect
golang.org/x/tools v0.24.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
gotest.tools/v3 v3.4.0 // indirect
Expand Down Expand Up @@ -170,7 +177,7 @@ require (
github.com/gophercloud/gophercloud v1.12.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
github.com/hashicorp/terraform-json v0.14.0 // indirect
github.com/hashicorp/terraform-json v0.22.1 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
Expand All @@ -190,14 +197,15 @@ require (
github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.53.0 // indirect
github.com/prometheus/procfs v0.15.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spotinst/spotinst-sdk-go v1.171.0 // indirect
github.com/zclconf/go-cty v1.12.1 // indirect
github.com/zclconf/go-cty v1.14.4 // indirect
go.mercari.io/hcledit v0.0.17
go.opencensus.io v0.24.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0
Expand Down
Loading
Loading