Merge pull request #205 from gluke77/failure-handling-ignore

Implement 'on error - ignore this'

e2e/basic_test.go

@@ -145,6 +145,25 @@ var _ = Describe("Basic Suite", func() {
 			})
 		})
 	})
+
+	Describe("Failure handling - ignore", func() {
+		var parentPod *v1.Pod
+		var childPod *v1.Pod
+
+		BeforeEach(func() {
+			parentPod = DelayedPod("parent-pod", 15)
+			childPod = PodPause("child-pod")
+		})
+
+		Context("If failed parent is marked on-error:ignore", func() {
+			It("dependency must be followed", func() {
+				parentResDef := framework.WrapWithMetaAndCreate(parentPod, map[string]interface{}{"timeout": 5, "on-error": "ignore"})
+				framework.Connect(parentResDef, framework.WrapAndCreate(childPod))
+				framework.Run()
+				testutils.WaitForPod(framework.Clientset, framework.Namespace.Name, childPod.Name, v1.PodRunning)
+			})
+		})
+	})
 })
 
 func getKind(resdef *client.ResourceDefinition) string {

examples/on-error-ignore/create.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+source ../common.sh
+
+$KUBECTL_NAME create -f ../../manifests/appcontroller.yaml
+wait-appcontroller
+
+$KUBECTL_NAME create -f deps.yaml
+
+cat timedout-pod.yaml | $KUBECTL_NAME create -f -
+cat pod.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME create -f -
+
+$KUBECTL_NAME exec k8s-appcontroller ac-run
+$KUBECTL_NAME logs -f k8s-appcontroller

examples/on-error-ignore/delete.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+source ../common.sh
+
+$KUBECTL_NAME delete -f deps.yaml
+
+cat timedout-pod.yaml | $KUBECTL_NAME delete -f -
+cat pod.yaml | $KUBECTL_NAME exec -i k8s-appcontroller kubeac wrap | $KUBECTL_NAME delete -f -
+
+$KUBECTL_NAME delete -f ../../manifests/appcontroller.yaml

examples/on-error-ignore/deps.yaml

@@ -0,0 +1,6 @@
+apiVersion: appcontroller.k8s/v1alpha1
+kind: Dependency
+metadata:
+  name: ce1b11dc-2850-1dad-a7dd-302038af20af
+parent: pod/timed-out-pod
+child: pod/eventually-alive-pod

examples/on-error-ignore/pod.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: eventually-alive-pod
+spec:
+  containers:
+  - command: ["/bin/sh"]
+    args:
+    - -c
+    - echo ok > /tmp/health
+    image: gcr.io/google_containers/busybox
+    name: test-container
+  restartPolicy: Never

examples/on-error-ignore/timedout-pod.yaml

@@ -0,0 +1,25 @@
+apiVersion: appcontroller.k8s/v1alpha1
+kind: Definition
+metadata:
+  name: pod-timed-out-pod
+meta:
+  timeout: 5
+  on-error: ignore
+pod:
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    name: timed-out-pod
+  spec:
+    containers:
+    - command: ["/bin/sh"]
+      args:
+      - -c
+      - sleep 30; echo ok > /tmp/health; sleep 60
+      image: gcr.io/google_containers/busybox
+      readinessProbe:
+        exec:
+          command:
+          - /bin/cat
+          - /tmp/health
+      name: test-container

pkg/resources/common.go

@@ -178,3 +178,20 @@ func GetIntMeta(r interfaces.BaseResource, paramName string, defaultValue int) i
 
 	return int(intVal)
 }
+
+// GetStringMeta returns metadata value for parameter 'paramName', or 'defaultValue'
+// if parameter is not set or is not a string value
+func GetStringMeta(r interfaces.BaseResource, paramName string, defaultValue string) string {
+	value := r.Meta(paramName)
+	if value == nil {
+		return defaultValue
+	}
+
+	strVal, ok := value.(string)
+	if !ok {
+		log.Printf("Metadata parameter '%s' for resource '%s' is set to '%v' but it does not seem to be a string, using default value %s", paramName, r.Key(), value, defaultValue)
+		return defaultValue
+	}
+
+	return string(strVal)
+}

pkg/scheduler/scheduler.go

@@ -80,6 +80,7 @@ type ScheduledResource struct {
 	Requires   []*ScheduledResource
 	RequiredBy []*ScheduledResource
 	Started    bool
+	Ignored    bool
 	Error      error
 	status     string
 	interfaces.Resource
@@ -167,7 +168,11 @@ func (sr *ScheduledResource) IsBlocked() bool {
 
 		status, err := req.Status(meta)
 
-		if err != nil && !onErrorSet {
+		req.RLock()
+		ignored := req.Ignored
+		req.RUnlock()
+
+		if err != nil && !onErrorSet && !ignored {
 			return true
 		} else if status == "ready" && onErrorSet {
 			return true
@@ -222,6 +227,7 @@ func NewScheduledResource(kind string, name string,
 func NewScheduledResourceFor(r interfaces.Resource) *ScheduledResource {
 	return &ScheduledResource{
 		Started:  false,
+		Ignored:  false,
 		Error:    nil,
 		Resource: r,
 		Meta:     map[string]map[string]string{},
@@ -344,6 +350,7 @@ func createResources(toCreate chan *ScheduledResource, finished chan string, ccL
 
 			attempts := resources.GetIntMeta(r.Resource, "retry", 1)
 			timeoutInSeconds := resources.GetIntMeta(r.Resource, "timeout", -1)
+			onError := resources.GetStringMeta(r.Resource, "on-error", "")
 
 			waitTimeout := WaitTimeout
 			if timeoutInSeconds > 0 {
@@ -398,6 +405,15 @@ func createResources(toCreate chan *ScheduledResource, finished chan string, ccL
 				}
 
 				log.Printf("Resource %s was not created: %v", r.Key(), err)
+
+				if attemptNo >= attempts {
+					if onError == "ignore" {
+						r.Lock()
+						r.Ignored = true
+						log.Printf("Resource %s failure ignored -- prooceeding as normal", r.Key())
+						r.Unlock()
+					}
+				}
 			}
 			finished <- r.Key()
 			// Release semaphor