[specific ci=Group6-VIC-Machine.6-07-Create-Network] Retry on intermi…

…ttent failure (#4597)

* Retry task for vsan invalid path and dvswitch disrupted error

  vSphere might throw these two errors for internal system issue,
  and can be fixed by itself, so retry in task is one simple workaround

* update comments

* add bugzilla bug link to track initial vsan bug

pkg/vsphere/tasks/waiter.go

@@ -1,4 +1,4 @@
-// Copyright 2016 VMware, Inc. All Rights Reserved.
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -70,7 +70,7 @@ func WaitForResult(ctx context.Context, f func(context.Context) (Task, error)) (
 			}
 		}
 
-		if !isTaskInProgress(err) {
+		if !isRetryError(err) {
 			return info, err
 		}
 
@@ -89,11 +89,27 @@ func WaitForResult(ctx context.Context, f func(context.Context) (Task, error)) (
 	}
 }
 
-func isTaskInProgress(err error) bool {
+const (
+	vimFault  = "vim"
+	soapFault = "soap"
+	taskFault = "task"
+)
+
+// isRetryErrors will return true for vSphere errors, which can be fixed by retry.
+// Currently the error includes TaskInProgress, NetworkDisruptedAndConfigRolledBack and InvalidArgument
+// Retry on NetworkDisruptedAndConfigRolledBack is to workaround vSphere issue
+// Retry on InvalidArgument(invlid path) is to workaround vSAN bug: https://bugzilla.eng.vmware.com/show_bug.cgi?id=1770798. TODO: Should remove it after vSAN fixed the bug
+func isRetryError(err error) bool {
 	if soap.IsSoapFault(err) {
 		switch f := soap.ToSoapFault(err).VimFault().(type) {
 		case types.TaskInProgress:
 			return true
+		case *types.NetworkDisruptedAndConfigRolledBack:
+			logExpectedFault(soapFault, f)
+			return true
+		case *types.InvalidArgument:
+			logExpectedFault(soapFault, f)
+			return true
 		default:
 			logSoapFault(f)
 		}
@@ -103,17 +119,31 @@ func isTaskInProgress(err error) bool {
 		switch f := soap.ToVimFault(err).(type) {
 		case *types.TaskInProgress:
 			return true
+		case *types.NetworkDisruptedAndConfigRolledBack:
+			logExpectedFault(vimFault, f)
+			return true
+		case *types.InvalidArgument:
+			logExpectedFault(vimFault, f)
+			return true
 		default:
 			logFault(f)
 		}
 	}
 
 	switch err := err.(type) {
 	case task.Error:
-		if _, ok := err.Fault().(*types.TaskInProgress); ok {
+		switch f := err.Fault().(type) {
+		case *types.TaskInProgress:
+			return true
+		case *types.NetworkDisruptedAndConfigRolledBack:
+			logExpectedFault(taskFault, f)
+			return true
+		case *types.InvalidArgument:
+			logExpectedFault(taskFault, f)
 			return true
+		default:
+			logFault(err.Fault())
 		}
-		logFault(err.Fault())
 	default:
 		if f, ok := err.(types.HasFault); ok {
 			logFault(f.Fault())
@@ -136,3 +166,7 @@ func logSoapFault(fault types.AnyType) {
 func logError(err error) {
 	log.Debugf("unexpected error on task retry : %#v", err)
 }
+
+func logExpectedFault(kind string, fault interface{}) {
+	log.Debugf("task retry on expected %s fault: %#v", kind, fault)
+}

pkg/vsphere/tasks/waiter_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 VMware, Inc. All Rights Reserved.
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -387,18 +387,18 @@ func TestSoapFaults(t *testing.T) {
 
 	// Test the task.Error path
 	res, err := task.WaitForResult(ctx, nil)
-	if !isTaskInProgress(err) {
+	if !isRetryError(err) {
 		t.Error(err)
 	}
 
 	// Test the soap.IsVimFault() path
-	if !isTaskInProgress(soap.WrapVimFault(res.Error.Fault)) {
+	if !isRetryError(soap.WrapVimFault(res.Error.Fault)) {
 		t.Errorf("fault=%#v", res.Error.Fault)
 	}
 
 	// Test the soap.IsSoapFault() path
 	err = vm.MarkAsTemplate(ctx)
-	if !isTaskInProgress(err) {
+	if !isRetryError(err) {
 		t.Error(err)
 	}
 
@@ -413,7 +413,7 @@ func TestSoapFaults(t *testing.T) {
 	if err == nil {
 		t.Error("expected error")
 	}
-	if isTaskInProgress(err) {
+	if isRetryError(err) {
 		t.Error(err)
 	}