Skip to content

Commit

Permalink
feat: use polling to ensure deleted and make waiting for deletion
Browse files Browse the repository at this point in the history 
optional.

* add another option to delete the resources in case they already exist
  • Loading branch information
@adityathebe
adityathebe committed Apr 16, 2024
1 parent 5974d2f commit ff2cec9
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 42 deletions.
6 changes: 6 additions & 0 deletions api/v1/checks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -863,6 +863,9 @@ type KubernetesResourceCheckWaitFor struct {
// Disable waiting for resources to get to their desired state.
Disable bool `json:"disable,omitempty"`

// Whether to wait for deletion or not
Delete bool `json:"delete,omitempty"`

// Timeout to wait for all static & non-static resources to be ready.
// Default: 10m
Timeout string `json:"timeout,omitempty"`
Expand Down Expand Up @@ -936,6 +939,9 @@ type KubernetesResourceCheck struct {
// Set initial delays and retry intervals for checks.
CheckRetries KubernetesResourceCheckRetries `json:"checkRetries,omitempty"`

// Ensure that the resources are deleted before creating them.
ClearResources bool `json:"clearResources,omitempty"`

// Kubeconfig is the kubeconfig or the path to the kubeconfig file.
Kubeconfig *types.EnvVar `yaml:"kubeconfig,omitempty" json:"kubeconfig,omitempty"`

Expand Down
80 changes: 38 additions & 42 deletions checks/kubernetes_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,15 @@ import (
"fmt"
"strconv"
"strings"
"sync"
"time"

"github.com/flanksource/gomplate/v3"
"github.com/samber/lo"
"github.com/sethvargo/go-retry"
"golang.org/x/sync/errgroup"
apiErrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/watch"

"github.com/flanksource/canary-checker/api/context"
v1 "github.com/flanksource/canary-checker/api/v1"
Expand Down Expand Up @@ -85,7 +84,7 @@ func (c *KubernetesResourceChecker) Check(ctx *context.Context, check v1.Kuberne
for i := range createdResources {
r := createdResources[i]
eg.Go(func() error {
return deleteResourceSync(ctx, r)
return deleteResource(ctx, r, check.WaitFor.Delete)
})
}
if err := eg.Wait(); err != nil {
Expand All @@ -96,9 +95,10 @@ func (c *KubernetesResourceChecker) Check(ctx *context.Context, check v1.Kuberne
for i := range check.Resources {
resource := check.Resources[i]

// attempt to delete the resource in case it already exist
if err := deleteResourceSync(ctx, resource); err != nil {
results.Failf(err.Error())
if check.ClearResources {
if err := deleteResource(ctx, resource, true); err != nil {
results.Failf(err.Error())
}
}

resource.SetAnnotations(map[string]string{annotationkey: ctx.Canary.ID()})
Expand Down Expand Up @@ -300,46 +300,20 @@ func (c *KubernetesResourceChecker) validate(ctx *context.Context, check v1.Kube
return nil
}

// deleteResourceSync deletes the given kubernetes resource and waits for it to be deleted.
func deleteResourceSync(ctx *context.Context, resource unstructured.Unstructured) error {
ctx.Logger.V(5).Infof("deleting resource (%s/%s/%s)", resource.GetKind(), resource.GetNamespace(), resource.GetName())
func deleteResource(ctx *context.Context, resource unstructured.Unstructured, waitForDelete bool) error {
ctx.Logger.V(4).Infof("deleting resource (%s/%s/%s)", resource.GetKind(), resource.GetNamespace(), resource.GetName())

rc, err := ctx.Kommons().GetRestClient(resource)
if err != nil {
return fmt.Errorf("failed to get rest client config for (%s/%s/%s): %w", resource.GetKind(), resource.GetNamespace(), resource.GetName(), err)
return fmt.Errorf("failed to get rest client for (%s/%s/%s): %w", resource.GetKind(), resource.GetNamespace(), resource.GetName(), err)
}

watcher, err := rc.Watch(resource.GetNamespace(), resource.GetAPIVersion(), &metav1.ListOptions{})
if err != nil {
return fmt.Errorf("failed to get watcher for (%s/%s/%s): %w", resource.GetKind(), resource.GetNamespace(), resource.GetName(), err)
namespace := utils.Coalesce(resource.GetNamespace(), ctx.Namespace)
deleteOpt := &metav1.DeleteOptions{
GracePeriodSeconds: lo.ToPtr(int64(0)),
PropagationPolicy: lo.ToPtr(metav1.DeletePropagationOrphan),
}
defer watcher.Stop()

var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case e := <-watcher.ResultChan():
if e.Object == nil {
// not sure why an empty event is fired.
// but this empty event kept coming through the channel
// resulting in high cpu usage.
return
}

if e.Type == watch.Deleted {
return
}

case <-ctx.Done():
return
}
}
}()

if err := ctx.Kommons().DeleteUnstructured(utils.Coalesce(resource.GetNamespace(), ctx.Namespace), &resource); err != nil {
if _, err := rc.DeleteWithOptions(namespace, resource.GetName(), deleteOpt); err != nil {
var statusErr *apiErrors.StatusError
if errors.As(err, &statusErr) {
switch statusErr.ErrStatus.Code {
Expand All @@ -351,6 +325,28 @@ func deleteResourceSync(ctx *context.Context, resource unstructured.Unstructured
return fmt.Errorf("failed to delete resource %s: %w", resource.GetName(), err)
}

wg.Wait()
return nil
if !waitForDelete {
return nil
}

// Poll the resource until it's deleted
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()

for {
select {
case <-ticker.C:
_, err = rc.Get(resource.GetNamespace(), resource.GetName())
if err != nil {
if apiErrors.IsNotFound(err) {
return nil
}
return fmt.Errorf("failed to get resource (%s/%s/%s) while polling for deletion: %w", resource.GetKind(), resource.GetNamespace(), resource.GetName(), err)
}
ctx.Logger.V(5).Infof("resource (%s/%s/%s) still exists, polling...", resource.GetKind(), resource.GetNamespace(), resource.GetName())

case <-ctx.Done():
return ctx.Err()
}
}
}
6 changes: 6 additions & 0 deletions config/deploy/crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5009,6 +5009,9 @@ spec:
type: object
type: array
x-kubernetes-preserve-unknown-fields: true
clearResources:
description: Deletes the resources, if they exist, before creating them.
type: boolean
description:
description: Description for the check
type: string
Expand Down Expand Up @@ -5141,6 +5144,9 @@ spec:
type: string
waitFor:
properties:
delete:
description: Whether to wait for deletion or not
type: boolean
disable:
description: Disable waiting for resources to get to their desired state.
type: boolean
Expand Down
6 changes: 6 additions & 0 deletions config/deploy/manifests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5008,6 +5008,9 @@ spec:
type: object
type: array
x-kubernetes-preserve-unknown-fields: true
clearResources:
description: Deletes the resources, if they exist, before creating them.
type: boolean
description:
description: Description for the check
type: string
Expand Down Expand Up @@ -5140,6 +5143,9 @@ spec:
type: string
waitFor:
properties:
delete:
description: Whether to wait for deletion or not
type: boolean
disable:
description: Disable waiting for resources to get to their desired state.
type: boolean
Expand Down
6 changes: 6 additions & 0 deletions config/schemas/canary.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2323,6 +2323,9 @@
"checkRetries": {
"$ref": "#/$defs/KubernetesResourceCheckRetries"
},
"clearResources": {
"type": "boolean"
},
"kubeconfig": {
"$ref": "#/$defs/EnvVar"
},
Expand Down Expand Up @@ -2363,6 +2366,9 @@
"disable": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"timeout": {
"type": "string"
},
Expand Down
6 changes: 6 additions & 0 deletions config/schemas/component.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2580,6 +2580,9 @@
"checkRetries": {
"$ref": "#/$defs/KubernetesResourceCheckRetries"
},
"clearResources": {
"type": "boolean"
},
"kubeconfig": {
"$ref": "#/$defs/EnvVar"
},
Expand Down Expand Up @@ -2620,6 +2623,9 @@
"disable": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"timeout": {
"type": "string"
},
Expand Down
6 changes: 6 additions & 0 deletions config/schemas/topology.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2550,6 +2550,9 @@
"checkRetries": {
"$ref": "#/$defs/KubernetesResourceCheckRetries"
},
"clearResources": {
"type": "boolean"
},
"kubeconfig": {
"$ref": "#/$defs/EnvVar"
},
Expand Down Expand Up @@ -2590,6 +2593,9 @@
"disable": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"timeout": {
"type": "string"
},
Expand Down

0 comments on commit ff2cec9

Please sign in to comment.