diff --git a/e2e/basic_test.go b/e2e/basic_test.go index 816bf32..7f539f5 100644 --- a/e2e/basic_test.go +++ b/e2e/basic_test.go @@ -145,6 +145,25 @@ var _ = Describe("Basic Suite", func() { }) }) }) + + Describe("Failure handling - ignore", func() { + var parentPod *v1.Pod + var childPod *v1.Pod + + BeforeEach(func() { + parentPod = DelayedPod("parent-pod", 15) + childPod = PodPause("child-pod") + }) + + Context("If failed parent is marked on-error:ignore", func() { + It("dependency must be followed", func() { + parentResDef := framework.WrapWithMetaAndCreate(parentPod, map[string]interface{}{"timeout": 5, "on-error": "ignore"}) + framework.Connect(parentResDef, framework.WrapAndCreate(childPod)) + framework.Run() + testutils.WaitForPod(framework.Clientset, framework.Namespace.Name, childPod.Name, v1.PodRunning) + }) + }) + }) }) func getKind(resdef *client.ResourceDefinition) string { diff --git a/examples/on-error-ignore/create.sh b/examples/on-error-ignore/create.sh new file mode 100755 index 0000000..1d0fbbd --- /dev/null +++ b/examples/on-error-ignore/create.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +source ../common.sh + +$KUBECTL_NAME create -f ../../manifests/appcontroller.yaml +wait-appcontroller + +$KUBECTL_NAME create -f deps.yaml + +cat timedout-pod.yaml | $KUBECTL_NAME create -f - +cat pod.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f - + +$KUBECTL_NAME exec k8s-appcontroller ac-run +$KUBECTL_NAME logs -f k8s-appcontroller diff --git a/examples/on-error-ignore/delete.sh b/examples/on-error-ignore/delete.sh new file mode 100755 index 0000000..1857ed2 --- /dev/null +++ b/examples/on-error-ignore/delete.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +source ../common.sh + +$KUBECTL_NAME delete -f deps.yaml + +cat timedout-pod.yaml | $KUBECTL_NAME delete -f - +cat pod.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME delete -f - + +$KUBECTL_NAME delete -f ../../manifests/appcontroller.yaml diff --git a/examples/on-error-ignore/deps.yaml b/examples/on-error-ignore/deps.yaml new file mode 100644 index 0000000..ad4137f --- /dev/null +++ b/examples/on-error-ignore/deps.yaml @@ -0,0 +1,6 @@ +apiVersion: appcontroller.k8s/v1alpha1 +kind: Dependency +metadata: + name: ce1b11dc-2850-1dad-a7dd-302038af20af +parent: pod/timed-out-pod +child: pod/eventually-alive-pod diff --git a/examples/on-error-ignore/pod.yaml b/examples/on-error-ignore/pod.yaml new file mode 100644 index 0000000..818a0cf --- /dev/null +++ b/examples/on-error-ignore/pod.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: eventually-alive-pod +spec: + containers: + - command: ["/bin/sh"] + args: + - -c + - echo ok > /tmp/health + image: gcr.io/google_containers/busybox + name: test-container + restartPolicy: Never diff --git a/examples/on-error-ignore/timedout-pod.yaml b/examples/on-error-ignore/timedout-pod.yaml new file mode 100644 index 0000000..b81f47a --- /dev/null +++ b/examples/on-error-ignore/timedout-pod.yaml @@ -0,0 +1,25 @@ +apiVersion: appcontroller.k8s/v1alpha1 +kind: Definition +metadata: + name: pod-timed-out-pod +meta: + timeout: 5 + on-error: ignore +pod: + apiVersion: v1 + kind: Pod + metadata: + name: timed-out-pod + spec: + containers: + - command: ["/bin/sh"] + args: + - -c + - sleep 30; echo ok > /tmp/health; sleep 60 + image: gcr.io/google_containers/busybox + readinessProbe: + exec: + command: + - /bin/cat + - /tmp/health + name: test-container diff --git a/pkg/resources/common.go b/pkg/resources/common.go index 61b1ecc..a6fc913 100644 --- a/pkg/resources/common.go +++ b/pkg/resources/common.go @@ -178,3 +178,20 @@ func GetIntMeta(r interfaces.BaseResource, paramName string, defaultValue int) i return int(intVal) } + +// GetStringMeta returns metadata value for parameter 'paramName', or 'defaultValue' +// if parameter is not set or is not a string value +func GetStringMeta(r interfaces.BaseResource, paramName string, defaultValue string) string { + value := r.Meta(paramName) + if value == nil { + return defaultValue + } + + strVal, ok := value.(string) + if !ok { + log.Printf("Metadata parameter '%s' for resource '%s' is set to '%v' but it does not seem to be a string, using default value %s", paramName, r.Key(), value, defaultValue) + return defaultValue + } + + return string(strVal) +} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 62f5110..4a9f9a7 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -80,6 +80,7 @@ type ScheduledResource struct { Requires []*ScheduledResource RequiredBy []*ScheduledResource Started bool + Ignored bool Error error status string interfaces.Resource @@ -167,7 +168,11 @@ func (sr *ScheduledResource) IsBlocked() bool { status, err := req.Status(meta) - if err != nil && !onErrorSet { + req.RLock() + ignored := req.Ignored + req.RUnlock() + + if err != nil && !onErrorSet && !ignored { return true } else if status == "ready" && onErrorSet { return true @@ -222,6 +227,7 @@ func NewScheduledResource(kind string, name string, func NewScheduledResourceFor(r interfaces.Resource) *ScheduledResource { return &ScheduledResource{ Started: false, + Ignored: false, Error: nil, Resource: r, Meta: map[string]map[string]string{}, @@ -344,6 +350,7 @@ func createResources(toCreate chan *ScheduledResource, finished chan string, ccL attempts := resources.GetIntMeta(r.Resource, "retry", 1) timeoutInSeconds := resources.GetIntMeta(r.Resource, "timeout", -1) + onError := resources.GetStringMeta(r.Resource, "on-error", "") waitTimeout := WaitTimeout if timeoutInSeconds > 0 { @@ -398,6 +405,15 @@ func createResources(toCreate chan *ScheduledResource, finished chan string, ccL } log.Printf("Resource %s was not created: %v", r.Key(), err) + + if attemptNo >= attempts { + if onError == "ignore" { + r.Lock() + r.Ignored = true + log.Printf("Resource %s failure ignored -- prooceeding as normal", r.Key()) + r.Unlock() + } + } } finished <- r.Key() // Release semaphor