diff --git a/api/v1/kustomization_types.go b/api/v1/kustomization_types.go
index ea6c92ae..c000829d 100644
--- a/api/v1/kustomization_types.go
+++ b/api/v1/kustomization_types.go
@@ -167,6 +167,12 @@ type KustomizationSpec struct {
// Components specifies relative paths to specifications of other Components.
// +optional
Components []string `json:"components,omitempty"`
+
+ // PartialApply instructs the controller to apply the kustomization partially
+ // if there are errors during the apply phase.
+ // +kubebuilder:default:=false
+ // +optional
+ PartialApply bool `json:"partialApply,omitempty"`
}
// CommonMetadata defines the common labels and annotations.
diff --git a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml
index 16ed18b2..4f402708 100644
--- a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml
+++ b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml
@@ -231,6 +231,12 @@ spec:
maxLength: 200
minLength: 1
type: string
+ partialApply:
+ default: false
+ description: |-
+ PartialApply instructs the controller to apply the kustomization partially
+ if there are errors during the apply phase.
+ type: boolean
patches:
description: |-
Strategic merge and JSON patches, defined as inline YAML objects,
diff --git a/docs/api/v1/kustomize.md b/docs/api/v1/kustomize.md
index 7a2281ad..bc067d89 100644
--- a/docs/api/v1/kustomize.md
+++ b/docs/api/v1/kustomize.md
@@ -380,6 +380,19 @@ resources. When enabled, the HealthChecks are ignored. Defaults to false.
Components specifies relative paths to specifications of other Components.
+
+
+partialApply
+
+bool
+
+ |
+
+(Optional)
+ PartialApply instructs the controller to apply the kustomization partially
+if there are errors during the apply phase.
+ |
+
@@ -888,6 +901,19 @@ resources. When enabled, the HealthChecks are ignored. Defaults to false.
Components specifies relative paths to specifications of other Components.
+
+
+partialApply
+
+bool
+
+ |
+
+(Optional)
+ PartialApply instructs the controller to apply the kustomization partially
+if there are errors during the apply phase.
+ |
+
diff --git a/internal/controller/kustomization_controller.go b/internal/controller/kustomization_controller.go
index 830f4c22..34a7c261 100644
--- a/internal/controller/kustomization_controller.go
+++ b/internal/controller/kustomization_controller.go
@@ -29,6 +29,7 @@ import (
securejoin "github.com/cyphar/filepath-securejoin"
"github.com/fluxcd/pkg/ssa/normalize"
ssautil "github.com/fluxcd/pkg/ssa/utils"
+ "golang.org/x/sync/errgroup"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -420,7 +421,7 @@ func (r *KustomizationReconciler) reconcile(
}
// Validate and apply resources in stages.
- drifted, changeSet, err := r.apply(ctx, resourceManager, obj, revision, objects)
+ drifted, changeSet, err, errDuringPartialApply := r.apply(ctx, resourceManager, obj, revision, objects)
if err != nil {
conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", err)
return err
@@ -467,11 +468,15 @@ func (r *KustomizationReconciler) reconcile(
// Set last applied revision.
obj.Status.LastAppliedRevision = revision
- // Mark the object as ready.
- conditions.MarkTrue(obj,
- meta.ReadyCondition,
- meta.ReconciliationSucceededReason,
- fmt.Sprintf("Applied revision: %s", revision))
+ if errDuringPartialApply != nil {
+ conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, fmt.Sprintf("Applied revision: %s with, err: %s", revision, errDuringPartialApply))
+ } else {
+ // Mark the object as ready.
+ conditions.MarkTrue(obj,
+ meta.ReadyCondition,
+ meta.ReconciliationSucceededReason,
+ fmt.Sprintf("Applied revision: %s", revision))
+ }
return nil
}
@@ -657,11 +662,11 @@ func (r *KustomizationReconciler) apply(ctx context.Context,
manager *ssa.ResourceManager,
obj *kustomizev1.Kustomization,
revision string,
- objects []*unstructured.Unstructured) (bool, *ssa.ChangeSet, error) {
+ objects []*unstructured.Unstructured) (bool, *ssa.ChangeSet, error, error) {
log := ctrl.LoggerFrom(ctx)
if err := normalize.UnstructuredList(objects); err != nil {
- return false, nil, err
+ return false, nil, err, nil
}
if cmeta := obj.Spec.CommonMetadata; cmeta != nil {
@@ -751,7 +756,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context,
if decryptor.IsEncryptedSecret(u) {
return false, nil,
fmt.Errorf("%s is SOPS encrypted, configuring decryption is required for this secret to be reconciled",
- ssautil.FmtUnstructured(u))
+ ssautil.FmtUnstructured(u)), nil
}
switch {
@@ -771,7 +776,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context,
if len(defStage) > 0 {
changeSet, err := manager.ApplyAll(ctx, defStage, applyOpts)
if err != nil {
- return false, nil, err
+ return false, nil, err, nil
}
if changeSet != nil && len(changeSet.Entries) > 0 {
@@ -788,7 +793,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context,
Interval: 2 * time.Second,
Timeout: obj.GetTimeout(),
}); err != nil {
- return false, nil, err
+ return false, nil, err, nil
}
}
}
@@ -797,7 +802,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context,
if len(classStage) > 0 {
changeSet, err := manager.ApplyAll(ctx, classStage, applyOpts)
if err != nil {
- return false, nil, err
+ return false, nil, err, nil
}
if changeSet != nil && len(changeSet.Entries) > 0 {
@@ -814,25 +819,54 @@ func (r *KustomizationReconciler) apply(ctx context.Context,
Interval: 2 * time.Second,
Timeout: obj.GetTimeout(),
}); err != nil {
- return false, nil, err
+ return false, nil, err, nil
}
}
}
+ var errorDuringPartialApply error = nil
// sort by kind, validate and apply all the others objects
sort.Sort(ssa.SortableUnstructureds(resStage))
if len(resStage) > 0 {
- changeSet, err := manager.ApplyAll(ctx, resStage, applyOpts)
- if err != nil {
- return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String())
+ changeSet := ssa.NewChangeSet()
+ if obj.Spec.PartialApply {
+ collectedChanges := make([]ssa.ChangeSetEntry, len(resStage))
+ g := &errgroup.Group{}
+ g.SetLimit(r.ConcurrentSSA)
+
+ for i, resource := range resStage {
+ g.Go(func() error {
+ changeSetEntry, err := manager.Apply(ctx, resource, applyOpts)
+ if err != nil {
+ r.event(obj, revision, eventv1.EventSeverityError, fmt.Sprintf("error during apply for %v: %v", client.ObjectKeyFromObject(resource), err), nil)
+ return err
+ }
+ collectedChanges[i] = *changeSetEntry
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ errorDuringPartialApply = err
+ }
+ changeSet.Append(collectedChanges)
+ } else {
+ var err error
+ changeSet, err = manager.ApplyAll(ctx, resStage, applyOpts)
+ if err != nil {
+ return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String()), nil
+ }
}
if changeSet != nil && len(changeSet.Entries) > 0 {
resultSet.Append(changeSet.Entries)
- log.Info("server-side apply completed", "output", changeSet.ToMap(), "revision", revision)
+ if errorDuringPartialApply != nil {
+ log.Info("server-side partial apply completed with error", "output", changeSet.ToMap(), "revision", revision, "err", errorDuringPartialApply)
+ } else {
+ log.Info("server-side apply completed", "output", changeSet.ToMap(), "revision", revision)
+ }
for _, change := range changeSet.Entries {
- if HasChanged(change.Action) {
+ if change.Action != "" && HasChanged(change.Action) {
changeSetLog.WriteString(change.String() + "\n")
}
}
@@ -845,7 +879,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context,
r.event(obj, revision, eventv1.EventSeverityInfo, applyLog, nil)
}
- return applyLog != "", resultSet, nil
+ return applyLog != "", resultSet, nil, errorDuringPartialApply
}
func (r *KustomizationReconciler) checkHealth(ctx context.Context,
diff --git a/internal/controller/kustomization_partial_apply_test.go b/internal/controller/kustomization_partial_apply_test.go
new file mode 100644
index 00000000..f9c7a695
--- /dev/null
+++ b/internal/controller/kustomization_partial_apply_test.go
@@ -0,0 +1,182 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/fluxcd/pkg/apis/meta"
+ "github.com/fluxcd/pkg/testserver"
+ sourcev1 "github.com/fluxcd/source-controller/api/v1"
+ . "github.com/onsi/gomega"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ apimeta "k8s.io/apimachinery/pkg/api/meta"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
+)
+
+func TestKustomizationReconciler_PartialApply(t *testing.T) {
+ g := NewWithT(t)
+ id := "partial-apply-" + randStringRunes(5)
+ revision := "v1.0.0"
+
+ err := createNamespace(id)
+ g.Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+
+ err = createKubeConfigSecret(id)
+ g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
+
+ manifests := func(name string, data string) []testserver.File {
+ return []testserver.File{
+ {
+ Name: "custom-resource.yaml",
+ Body: fmt.Sprintf(`---
+apiVersion: example.com/v1
+kind: CustomResource
+metadata:
+ name: %[1]s
+spec:
+ exampleField: "%[2]s"
+`, name, data),
+ },
+ {
+ Name: "secret.yaml",
+ Body: fmt.Sprintf(`---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: %[1]s
+stringData:
+ key: "%[2]s"
+`, name, data),
+ },
+ }
+ }
+
+ artifact, err := testServer.ArtifactFromFiles(manifests(id, randStringRunes(5)))
+ g.Expect(err).NotTo(HaveOccurred(), "failed to create artifact from files")
+
+ repositoryName := types.NamespacedName{
+ Name: fmt.Sprintf("partial-apply-%s", randStringRunes(5)),
+ Namespace: id,
+ }
+
+ err = applyGitRepository(repositoryName, artifact, revision)
+ g.Expect(err).NotTo(HaveOccurred())
+
+ kustomizationKey := types.NamespacedName{
+ Name: fmt.Sprintf("partial-apply-%s", randStringRunes(5)),
+ Namespace: id,
+ }
+ kustomization := &kustomizev1.Kustomization{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: kustomizationKey.Name,
+ Namespace: kustomizationKey.Namespace,
+ },
+ Spec: kustomizev1.KustomizationSpec{
+ Interval: metav1.Duration{Duration: reconciliationInterval},
+ Path: "./",
+ KubeConfig: &meta.KubeConfigReference{
+ SecretRef: meta.SecretKeyReference{
+ Name: "kubeconfig",
+ },
+ },
+ SourceRef: kustomizev1.CrossNamespaceSourceReference{
+ Name: repositoryName.Name,
+ Namespace: repositoryName.Namespace,
+ Kind: sourcev1.GitRepositoryKind,
+ },
+ HealthChecks: []meta.NamespacedObjectKindReference{
+ {
+ APIVersion: "v1",
+ Kind: "Secret",
+ Name: id,
+ Namespace: id,
+ },
+ },
+ TargetNamespace: id,
+ PartialApply: false,
+ },
+ }
+
+ g.Expect(k8sClient.Create(context.Background(), kustomization)).To(Succeed())
+
+ resultK := &kustomizev1.Kustomization{}
+
+ t.Run("fails to apply resources", func(t *testing.T) {
+ artifact, err = testServer.ArtifactFromFiles(manifests(id, randStringRunes(5)))
+ g.Expect(err).NotTo(HaveOccurred())
+ revision = "v1.0.0"
+ err = applyGitRepository(repositoryName, artifact, revision)
+ g.Expect(err).NotTo(HaveOccurred())
+
+ g.Eventually(func() bool {
+ _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK)
+ return isReconcileFailure(resultK)
+ }, timeout, time.Second).Should(BeTrue())
+ logStatus(t, resultK)
+
+ kstatusCheck.CheckErr(ctx, resultK)
+
+ t.Run("emits dry-run error event", func(t *testing.T) {
+ events := getEvents(resultK.GetName(), map[string]string{"kustomize.toolkit.fluxcd.io/revision": revision})
+ g.Expect(len(events) > 0).To(BeTrue())
+ g.Expect(events[0].Type).To(BeIdenticalTo("Warning"))
+ g.Expect(events[0].Message).To(ContainSubstring("dry-run failed: no matches for kind"))
+ })
+ })
+
+ t.Run("partially applies secret", func(t *testing.T) {
+ artifact, err = testServer.ArtifactFromFiles(manifests(id, randStringRunes(5)))
+ g.Expect(err).NotTo(HaveOccurred())
+ revision = "v2.0.0"
+ err = applyGitRepository(repositoryName, artifact, revision)
+ g.Expect(err).NotTo(HaveOccurred())
+
+ g.Eventually(func() error {
+ _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK)
+ resultK.Spec.PartialApply = true
+ return k8sClient.Update(context.Background(), resultK)
+ }, timeout, time.Second).Should(BeNil())
+
+ g.Eventually(func() bool {
+ _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK)
+ hasCreatedSecret := !apierrors.IsNotFound(k8sClient.Get(context.Background(), types.NamespacedName{Name: id, Namespace: id}, &corev1.Secret{}))
+ return hasCreatedSecret && apimeta.IsStatusConditionFalse(resultK.Status.Conditions, meta.ReadyCondition)
+ }, timeout, time.Second).Should(BeTrue())
+ logStatus(t, resultK)
+
+ t.Run("emits partial apply error event", func(t *testing.T) {
+ events := getEvents(resultK.GetName(), map[string]string{"kustomize.toolkit.fluxcd.io/revision": revision})
+ g.Expect(len(events) > 0).To(BeTrue())
+ g.Expect(events[0].Type).To(BeIdenticalTo("Warning"))
+ g.Expect(events[0].Message).To(ContainSubstring("error during apply"))
+ g.Expect(events[0].Message).To(ContainSubstring("dry-run failed: no matches for kind"))
+ })
+
+ g.Expect(resultK.Status.Inventory.Entries).To(ContainElement(kustomizev1.ResourceRef{ID: fmt.Sprintf("%s_%s__Secret", id, id), Version: "v1"}))
+
+ kstatusCheck.CheckErr(ctx, resultK)
+ })
+}