From 4b3fe515a735dce1187d6ccdd0ebb64c3e3d1c2f Mon Sep 17 00:00:00 2001
From: Emma Lin <line@vmware.com>
Date: Thu, 6 Apr 2017 15:04:53 -0500
Subject: [PATCH] [specific ci=Group6-VIC-Machine.6-07-Create-Network] Retry on
 intermittent failure (#4597)

* Retry task for vsan invalid path and dvswitch disrupted error

  vSphere might throw these two errors for internal system issue,
  and can be fixed by itself, so retry in task is one simple workaround

* update comments

* add bugzilla bug link to track initial vsan bug
---
 pkg/vsphere/tasks/waiter.go      | 44 ++++++++++++++++++++++++++++----
 pkg/vsphere/tasks/waiter_test.go | 10 ++++----
 2 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/pkg/vsphere/tasks/waiter.go b/pkg/vsphere/tasks/waiter.go
index edca9352d7..804c8293a6 100644
--- a/pkg/vsphere/tasks/waiter.go
+++ b/pkg/vsphere/tasks/waiter.go
@@ -1,4 +1,4 @@
-// Copyright 2016 VMware, Inc. All Rights Reserved.
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -70,7 +70,7 @@ func WaitForResult(ctx context.Context, f func(context.Context) (Task, error)) (
 			}
 		}
 
-		if !isTaskInProgress(err) {
+		if !isRetryError(err) {
 			return info, err
 		}
 
@@ -89,11 +89,27 @@ func WaitForResult(ctx context.Context, f func(context.Context) (Task, error)) (
 	}
 }
 
-func isTaskInProgress(err error) bool {
+const (
+	vimFault  = "vim"
+	soapFault = "soap"
+	taskFault = "task"
+)
+
+// isRetryErrors will return true for vSphere errors, which can be fixed by retry.
+// Currently the error includes TaskInProgress, NetworkDisruptedAndConfigRolledBack and InvalidArgument
+// Retry on NetworkDisruptedAndConfigRolledBack is to workaround vSphere issue
+// Retry on InvalidArgument(invlid path) is to workaround vSAN bug: https://bugzilla.eng.vmware.com/show_bug.cgi?id=1770798. TODO: Should remove it after vSAN fixed the bug
+func isRetryError(err error) bool {
 	if soap.IsSoapFault(err) {
 		switch f := soap.ToSoapFault(err).VimFault().(type) {
 		case types.TaskInProgress:
 			return true
+		case *types.NetworkDisruptedAndConfigRolledBack:
+			logExpectedFault(soapFault, f)
+			return true
+		case *types.InvalidArgument:
+			logExpectedFault(soapFault, f)
+			return true
 		default:
 			logSoapFault(f)
 		}
@@ -103,6 +119,12 @@ func isTaskInProgress(err error) bool {
 		switch f := soap.ToVimFault(err).(type) {
 		case *types.TaskInProgress:
 			return true
+		case *types.NetworkDisruptedAndConfigRolledBack:
+			logExpectedFault(vimFault, f)
+			return true
+		case *types.InvalidArgument:
+			logExpectedFault(vimFault, f)
+			return true
 		default:
 			logFault(f)
 		}
@@ -110,10 +132,18 @@ func isTaskInProgress(err error) bool {
 
 	switch err := err.(type) {
 	case task.Error:
-		if _, ok := err.Fault().(*types.TaskInProgress); ok {
+		switch f := err.Fault().(type) {
+		case *types.TaskInProgress:
+			return true
+		case *types.NetworkDisruptedAndConfigRolledBack:
+			logExpectedFault(taskFault, f)
+			return true
+		case *types.InvalidArgument:
+			logExpectedFault(taskFault, f)
 			return true
+		default:
+			logFault(err.Fault())
 		}
-		logFault(err.Fault())
 	default:
 		if f, ok := err.(types.HasFault); ok {
 			logFault(f.Fault())
@@ -136,3 +166,7 @@ func logSoapFault(fault types.AnyType) {
 func logError(err error) {
 	log.Debugf("unexpected error on task retry : %#v", err)
 }
+
+func logExpectedFault(kind string, fault interface{}) {
+	log.Debugf("task retry on expected %s fault: %#v", kind, fault)
+}
diff --git a/pkg/vsphere/tasks/waiter_test.go b/pkg/vsphere/tasks/waiter_test.go
index 1ab00b72c1..409a338162 100644
--- a/pkg/vsphere/tasks/waiter_test.go
+++ b/pkg/vsphere/tasks/waiter_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 VMware, Inc. All Rights Reserved.
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -387,18 +387,18 @@ func TestSoapFaults(t *testing.T) {
 
 	// Test the task.Error path
 	res, err := task.WaitForResult(ctx, nil)
-	if !isTaskInProgress(err) {
+	if !isRetryError(err) {
 		t.Error(err)
 	}
 
 	// Test the soap.IsVimFault() path
-	if !isTaskInProgress(soap.WrapVimFault(res.Error.Fault)) {
+	if !isRetryError(soap.WrapVimFault(res.Error.Fault)) {
 		t.Errorf("fault=%#v", res.Error.Fault)
 	}
 
 	// Test the soap.IsSoapFault() path
 	err = vm.MarkAsTemplate(ctx)
-	if !isTaskInProgress(err) {
+	if !isRetryError(err) {
 		t.Error(err)
 	}
 
@@ -413,7 +413,7 @@ func TestSoapFaults(t *testing.T) {
 	if err == nil {
 		t.Error("expected error")
 	}
-	if isTaskInProgress(err) {
+	if isRetryError(err) {
 		t.Error(err)
 	}