Merge branch 'develop' of github.com:cloudfoundry-incubator/kubo-rele…

…ase into HEAD

jobs/kubelet/spec

@@ -39,6 +39,21 @@ properties:
   kubectl-drain-timeout:
     description: "The length of time to wait before giving up draining a node, zero means infinite"
     default: "0s"
+  kubelet-drain-grace-period:
+    description: "Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used."
+    default: "10"
+  kubelet-drain-force:
+    description: "Continue drain even if there are pods not managed by a ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet."
+    default: true
+  kubelet-drain-ignore-daemonsets:
+    description: "Ignore DaemonSet-managed pods during drain"
+    default: true
+  kubelet-drain-delete-local-data:
+    description: "Continue drain even if there are pods using emptyDir (local data that will be deleted when the node is drained)"
+    default: true
+  kubelet-drain-force-node:
+    description: "Forcibly terminate pods if all the pods fail to drain before the timeout."
+    default: false
   k8s-args:
     description: "Pass-through options for Kubernetes runtime arguments. See docs https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ for reference."
     example: |

jobs/kubelet/templates/bin/drain.erb

@@ -21,7 +21,17 @@ main() {
   retry cordon_node
   k8s_disks=$(get_k8s_disks)
   echo "Mounted volumes: $k8s_disks"
-  retry drain_node
+  if ! drain_node ; then
+    <% if_p("kubelet-drain-force-node") do |prop| %>
+      <% if prop %>
+        force_kill_pods
+      <% else %>
+        echo "Unsuccessful function drain_node"
+        exit 1
+      <% end %>
+    <% end %>
+  fi
+
   if [[ -z "${k8s_disks}" ]]; then
     echo "No attached PVs"
   else
@@ -35,13 +45,45 @@ cordon_node() {
 }
 
 drain_node() {
-  kubectl --kubeconfig /var/vcap/jobs/kubelet/config/kubeconfig-drain drain \
-    -l "bosh.id=<%= spec.id %>" \
-    --grace-period 10 \
-    --force \
-    --delete-local-data \
-    --ignore-daemonsets \
-    --timeout <%= p("kubectl-drain-timeout") %>
+  kubectl_args=()
+  kubectl_args+=(--kubeconfig /var/vcap/jobs/kubelet/config/kubeconfig-drain drain)
+  kubectl_args+=(-l "bosh.id=<%= spec.id %>")
+  kubectl_args+=(--grace-period <%= p("kubelet-drain-grace-period") %>)
+  kubectl_args+=(--timeout <%= p("kubectl-drain-timeout") %>)
+
+  <% if_p("kubelet-drain-force") do |prop| %>
+    <% if prop %>
+      kubectl_args+=(--force)
+    <% end %>
+  <% end %>
+
+  <% if_p("kubelet-drain-ignore-daemonsets") do |prop| %>
+    <% if prop %>
+      kubectl_args+=(--ignore-daemonsets)
+    <% end %>
+  <% end %>
+
+  <% if_p("kubelet-drain-delete-local-data") do |prop| %>
+    <% if prop %>
+      kubectl_args+=(--delete-local-data)
+    <% end %>
+  <% end %>
+
+  kubectl "${kubectl_args[@]}"
+}
+
+force_kill_pods() {
+  echo "Forcefully draining the node"
+  node_name=$(kubectl --kubeconfig /var/vcap/jobs/kubelet/config/kubeconfig-drain get node -l "bosh.id=<%= spec.id %>" -o jsonpath="{.items[:].metadata.name}")
+  namespaces=( $(kubectl --kubeconfig /var/vcap/jobs/kubelet/config/kubeconfig-drain get pods --all-namespaces --field-selector spec.nodeName=${node_name} -o jsonpath="{.items[:].metadata.namespace}" | xargs -n1 | sort -u -) )
+  for namespace in "${namespaces[@]}"; do
+    echo $namespace;
+    podnames=( $(kubectl --kubeconfig /var/vcap/jobs/kubelet/config/kubeconfig-drain get pods -n=${namespace} --field-selector spec.nodeName=${node_name} -o jsonpath="{.items[:].metadata.name}") )
+    for pod_name in "${podnames[@]}"; do
+      echo "Force Deleting pod ${pod_name}"
+      kubectl --kubeconfig /var/vcap/jobs/kubelet/config/kubeconfig-drain delete pods ${pod_name}  --namespace ${namespace} --grace-period=0 --force
+    done
+  done
 }
 
 get_k8s_disks() {