From c1fdf22953d6138df2b03dce9565a780b37c1793 Mon Sep 17 00:00:00 2001 From: Max Ewing Date: Tue, 1 Feb 2022 16:52:38 +0000 Subject: [PATCH 1/6] fix: flow activation failures due to action metadata --- .../Services/ProcessDeploymentService.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs index a9805b3..3c9dd07 100644 --- a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs +++ b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs @@ -3,12 +3,12 @@ using System; using System.Collections.Generic; using System.Linq; + using System.Threading; using Capgemini.PowerApps.PackageDeployerTemplate.Adapters; using Microsoft.Crm.Sdk.Messages; using Microsoft.Extensions.Logging; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Query; - using Polly; /// /// Deployment functionality relating to processes. @@ -121,6 +121,8 @@ private void ExecuteSetStateRequests(IEnumerable requests, IEnumerable successfulResponses; IEnumerable failedResponses; + var attempt = 1; + var maxAttempts = 3; do { var timeout = 120 + (remainingRequests.Count * 10); @@ -136,8 +138,16 @@ private void ExecuteSetStateRequests(IEnumerable requests, remainingRequests = failedResponses .Select(r => remainingRequests[r.RequestIndex]) .ToList(); + + attempt = successfulResponses.Any() ? 1 : attempt++; + + if (!successfulResponses.Any() && attempt <= maxAttempts) + { + // Short delay to account for errors such as where metadata cannot yet be retrieved by flows for newly activated actions. + Thread.Sleep(1000); + } } - while (successfulResponses.Any() && remainingRequests.Count > 0); + while (successfulResponses.Any() && remainingRequests.Count > 0 && attempt <= maxAttempts); if (!successfulResponses.Any() && remainingRequests.Any()) { From 7102a3c5f9519924b9efb415a3c12982f92e2025 Mon Sep 17 00:00:00 2001 From: Max Ewing Date: Wed, 2 Feb 2022 18:22:13 +0000 Subject: [PATCH 2/6] fix: flow activation errors due to action metadata --- .../Services/ProcessDeploymentService.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs index 3c9dd07..b8cdee1 100644 --- a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs +++ b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs @@ -139,15 +139,16 @@ private void ExecuteSetStateRequests(IEnumerable requests, .Select(r => remainingRequests[r.RequestIndex]) .ToList(); + // Reset attempt number if there are any successful responses. attempt = successfulResponses.Any() ? 1 : attempt++; - if (!successfulResponses.Any() && attempt <= maxAttempts) + // Delay if any flow activations failed due to failures getting metadata for actions + if (failedResponses.Any(f => f.Fault.ErrorCode == -2147089305 && f.Fault.Message.Contains("GetMetadataFor")) && attempt <= maxAttempts) { - // Short delay to account for errors such as where metadata cannot yet be retrieved by flows for newly activated actions. - Thread.Sleep(1000); + Thread.Sleep(10000); } } - while (successfulResponses.Any() && remainingRequests.Count > 0 && attempt <= maxAttempts); + while (remainingRequests.Count > 0 && attempt <= maxAttempts); if (!successfulResponses.Any() && remainingRequests.Any()) { From f649f226f7cfb4055647c424285d33521caedf19 Mon Sep 17 00:00:00 2001 From: Max Ewing Date: Mon, 7 Feb 2022 14:42:52 +0000 Subject: [PATCH 3/6] fix: infinite retries causing OutOfMemoryException --- .../Services/ProcessDeploymentService.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs index b8cdee1..39efcb7 100644 --- a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs +++ b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs @@ -140,10 +140,11 @@ private void ExecuteSetStateRequests(IEnumerable requests, .ToList(); // Reset attempt number if there are any successful responses. - attempt = successfulResponses.Any() ? 1 : attempt++; + attempt = successfulResponses.Any() ? 1 : attempt + 1; - // Delay if any flow activations failed due to failures getting metadata for actions - if (failedResponses.Any(f => f.Fault.ErrorCode == -2147089305 && f.Fault.Message.Contains("GetMetadataFor")) && attempt <= maxAttempts) + // Delay if any process activation errors are due to a FlowServiceClientError response. + // This is needed due to issues such as where recently activated actions are not available to flows straight away. + if (failedResponses.Any(f => f.Fault.ErrorCode == -2147089305 /*FlowServiceClientError*/ && attempt <= maxAttempts)) { Thread.Sleep(10000); } From 4cb320bc54612abc106af3b08ea726693d3b29b7 Mon Sep 17 00:00:00 2001 From: Max Ewing Date: Mon, 7 Feb 2022 14:43:00 +0000 Subject: [PATCH 4/6] test: add unit tests for retry logic --- .../Services/ProcessDeploymentServiceTests.cs | 91 +++++++++++++++++-- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/tests/Capgemini.PowerApps.PackageDeployerTemplate.UnitTests/Services/ProcessDeploymentServiceTests.cs b/tests/Capgemini.PowerApps.PackageDeployerTemplate.UnitTests/Services/ProcessDeploymentServiceTests.cs index dd997d1..68e73d1 100644 --- a/tests/Capgemini.PowerApps.PackageDeployerTemplate.UnitTests/Services/ProcessDeploymentServiceTests.cs +++ b/tests/Capgemini.PowerApps.PackageDeployerTemplate.UnitTests/Services/ProcessDeploymentServiceTests.cs @@ -208,6 +208,7 @@ public void SetStates_WithUserParameter_ExecutesAsUser() { var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; this.MockSetStatesProcesses(foundProcesses); + var userToImpersonate = "licenseduser@domaincom"; this.MockExecuteMultipleResponse( null, @@ -235,17 +236,63 @@ public void SetStates_WithError_LogsError() { var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; this.MockSetStatesProcesses(foundProcesses); + + var failedResponse = GetNewExecuteMultipleResponse(isFaulted: true); var fault = new OrganizationServiceFault { Message = "Some error." }; - var response = new ExecuteMultipleResponse - { - Results = new ParameterCollection + failedResponse.Responses.Add(new ExecuteMultipleResponseItem { Fault = fault }); + this.MockExecuteMultipleResponse(failedResponse); + + this.processDeploymentSvc.SetStates( + new List { - { "Responses", new ExecuteMultipleResponseItemCollection() }, - { "IsFaulted", true }, + foundProcesses.First().GetAttributeValue(Constants.Workflow.Fields.Name), }, - }; - response.Responses.Add(new ExecuteMultipleResponseItem { Fault = fault }); - this.MockExecuteMultipleResponse(response); + Enumerable.Empty()); + + this.loggerMock.VerifyLog(l => l.LogError(It.Is(s => s.Contains(fault.Message)))); + } + + [Fact] + public void SetStates_WithError_RetriesUpToThreeTimes() + { + var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; + this.MockSetStatesProcesses(foundProcesses); + + var failedResponse = GetNewExecuteMultipleResponse(isFaulted: true); + failedResponse.Responses.Add(new ExecuteMultipleResponseItem { Fault = new OrganizationServiceFault { Message = "Flow error." } }); + + var successfulResponse = GetNewExecuteMultipleResponse(isFaulted: false); + successfulResponse.Responses.Add(new ExecuteMultipleResponseItem()); + + this.MockExecuteMultipleResponses(failedResponse, failedResponse, successfulResponse); + + this.processDeploymentSvc.SetStates( + new List + { + foundProcesses.First().GetAttributeValue(Constants.Workflow.Fields.Name), + }, + Enumerable.Empty()); + + this.crmServiceAdapterMock.Verify( + svc => svc.ExecuteMultiple( + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(3)); + } + + [Fact] + public void SetStates_WithError_LogsErrorAfterThirdRetry() + { + var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; + this.MockSetStatesProcesses(foundProcesses); + + var failedResponse = GetNewExecuteMultipleResponse(isFaulted: true); + var fault = new OrganizationServiceFault { Message = "Some error." }; + failedResponse.Responses.Add(new ExecuteMultipleResponseItem { Fault = fault }); + + this.MockExecuteMultipleResponses(failedResponse, failedResponse, failedResponse); this.processDeploymentSvc.SetStates( new List @@ -257,6 +304,18 @@ public void SetStates_WithError_LogsError() this.loggerMock.VerifyLog(l => l.LogError(It.Is(s => s.Contains(fault.Message)))); } + private static ExecuteMultipleResponse GetNewExecuteMultipleResponse(bool isFaulted) + { + return new ExecuteMultipleResponse + { + Results = new ParameterCollection + { + { "Responses", new ExecuteMultipleResponseItemCollection() }, + { "IsFaulted", isFaulted }, + }, + }; + } + private static Entity GetProcess(int stateCode) { return new Entity(Constants.Workflow.LogicalName, Guid.NewGuid()) @@ -300,6 +359,22 @@ private void MockExecuteMultipleResponse(ExecuteMultipleResponse response = null } } + private void MockExecuteMultipleResponses(params ExecuteMultipleResponse[] responses) + { + var setup = this.crmServiceAdapterMock + .SetupSequence(svc => + svc.ExecuteMultiple( + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny())); + + foreach (var response in responses) + { + setup.Returns(response); + } + } + private void MockSetStatesProcesses(IList processes) { this.crmServiceAdapterMock.Setup( From 7a0ad0726ca9fa765849dc3313bc5021c3d6d678 Mon Sep 17 00:00:00 2001 From: Max Ewing Date: Tue, 8 Feb 2022 06:50:16 +0000 Subject: [PATCH 5/6] fix: increase delay Tests have shown that a maximum of 30 seconds is not sufficient --- .../Services/ProcessDeploymentService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs index 39efcb7..32eddfd 100644 --- a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs +++ b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs @@ -146,7 +146,7 @@ private void ExecuteSetStateRequests(IEnumerable requests, // This is needed due to issues such as where recently activated actions are not available to flows straight away. if (failedResponses.Any(f => f.Fault.ErrorCode == -2147089305 /*FlowServiceClientError*/ && attempt <= maxAttempts)) { - Thread.Sleep(10000); + Thread.Sleep(30000); } } while (remainingRequests.Count > 0 && attempt <= maxAttempts); From 03e5dac6464ba9205149756b78d0bfa6d77673fb Mon Sep 17 00:00:00 2001 From: Max Ewing Date: Tue, 15 Feb 2022 14:11:08 +0000 Subject: [PATCH 6/6] refactor: revert to individual update requests --- .../Services/ProcessDeploymentService.cs | 118 +++++------ .../Services/ProcessDeploymentServiceTests.cs | 187 +++++------------- 2 files changed, 112 insertions(+), 193 deletions(-) diff --git a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs index 32eddfd..0d8c0e4 100644 --- a/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs +++ b/src/Capgemini.PowerApps.PackageDeployerTemplate/Services/ProcessDeploymentService.cs @@ -3,11 +3,11 @@ using System; using System.Collections.Generic; using System.Linq; - using System.Threading; + using System.ServiceModel; using Capgemini.PowerApps.PackageDeployerTemplate.Adapters; - using Microsoft.Crm.Sdk.Messages; using Microsoft.Extensions.Logging; using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Messages; using Microsoft.Xrm.Sdk.Query; /// @@ -114,87 +114,91 @@ private void SetStates(IEnumerable processes, IEnumerable proces this.ExecuteSetStateRequests(requests, user); } - private void ExecuteSetStateRequests(IEnumerable requests, string user = null) + private void ExecuteSetStateRequests(IDictionary requests, string user = null) { - // Due to unpredictable process dependencies we should retry failed requests until there are zero successful responses. - var remainingRequests = new List(requests); - IEnumerable successfulResponses; - IEnumerable failedResponses; + var remainingRequests = requests; + + IDictionary failedRequests; + IDictionary successfulRequests; + IDictionary> errors; - var attempt = 1; - var maxAttempts = 3; do { - var timeout = 120 + (remainingRequests.Count * 10); - var executeMultipleRes = string.IsNullOrEmpty(user) ? - this.crmSvc.ExecuteMultiple(remainingRequests, true, true, timeout) : this.crmSvc.ExecuteMultiple(remainingRequests, user, true, true, timeout); - - successfulResponses = executeMultipleRes.Responses - .Where(r => r.Fault == null) - .ToList(); - failedResponses = executeMultipleRes.Responses - .Except(successfulResponses) - .ToList(); - remainingRequests = failedResponses - .Select(r => remainingRequests[r.RequestIndex]) - .ToList(); - - // Reset attempt number if there are any successful responses. - attempt = successfulResponses.Any() ? 1 : attempt + 1; - - // Delay if any process activation errors are due to a FlowServiceClientError response. - // This is needed due to issues such as where recently activated actions are not available to flows straight away. - if (failedResponses.Any(f => f.Fault.ErrorCode == -2147089305 /*FlowServiceClientError*/ && attempt <= maxAttempts)) + failedRequests = new Dictionary(); + successfulRequests = new Dictionary(); + errors = new Dictionary>(); + + foreach (var req in remainingRequests) { - Thread.Sleep(30000); + if (req.Value == null) + { + this.logger.LogInformation($"Process {req.Key[Constants.Workflow.Fields.Name]} already has desired state. Skipping."); + continue; + } + + if (req.Value.Target.GetAttributeValue(Constants.Workflow.Fields.StateCode).Value == 1) + { + this.logger.LogInformation($"Activating {req.Key[Constants.Workflow.Fields.Name]}."); + } + else + { + this.logger.LogInformation($"Deactivating {req.Key[Constants.Workflow.Fields.Name]}."); + } + + try + { + var response = string.IsNullOrEmpty(user) ? + (UpdateResponse)this.crmSvc.Execute(req.Value) : this.crmSvc.Execute(req.Value, user, true); + + successfulRequests.Add(req.Key, req.Value); + } + catch (FaultException ex) + { + failedRequests.Add(req.Key, req.Value); + errors.Add(req.Key, ex); + } } + + remainingRequests = failedRequests; } - while (remainingRequests.Count > 0 && attempt <= maxAttempts); + while (remainingRequests.Count > 0 && successfulRequests.Any()); - if (!successfulResponses.Any() && remainingRequests.Any()) + foreach (var error in errors) { - foreach (var failedResponse in failedResponses) - { - var failedRequest = (SetStateRequest)remainingRequests[failedResponse.RequestIndex]; - this.logger.LogError($"Failed to set state for process {failedRequest.EntityMoniker.Name} with the following error: {failedResponse.Fault.Message}."); - } + this.logger.LogError($"Failed to set state for process {error.Key[Constants.Workflow.Fields.Name]} with the following error: {error.Value.Message}."); } } - private List GetSetStateRequests(IEnumerable processes, IEnumerable processesToDeactivate) + private IDictionary GetSetStateRequests(IEnumerable processes, IEnumerable processesToDeactivate) { - var requests = new List(); - - foreach (var deployedProcess in processes) + return processes.ToDictionary(p => p, p => { var stateCode = new OptionSetValue(Constants.Workflow.StateCodeActive); var statusCode = new OptionSetValue(Constants.Workflow.StatusCodeActive); - if (processesToDeactivate != null && processesToDeactivate.Contains(deployedProcess[Constants.Workflow.Fields.Name])) + if (processesToDeactivate != null && processesToDeactivate.Contains(p[Constants.Workflow.Fields.Name])) { stateCode.Value = Constants.Workflow.StateCodeInactive; statusCode.Value = Constants.Workflow.StatusCodeInactive; } - if (stateCode.Value == deployedProcess.GetAttributeValue(Constants.Workflow.Fields.StateCode).Value) + if (stateCode.Value == p.GetAttributeValue(Constants.Workflow.Fields.StateCode).Value) { - this.logger.LogInformation($"Process {deployedProcess[Constants.Workflow.Fields.Name]} already has desired state. Skipping."); - continue; + return null; } - this.logger.LogInformation($"Setting process status for {deployedProcess[Constants.Workflow.Fields.Name]} with statecode {stateCode.Value} and statuscode {statusCode.Value}"); - - // SetStateRequest is supposedly deprecated but UpdateRequest doesn't work for deactivating active flows - requests.Add( - new SetStateRequest + return new UpdateRequest + { + Target = new Entity(p.LogicalName, p.Id) { - EntityMoniker = deployedProcess.ToEntityReference(), - State = stateCode, - Status = statusCode, - }); - } - - return requests; + Attributes = + { + { "statecode", stateCode }, + { "statuscode", statusCode }, + }, + }, + }; + }); } private EntityCollection RetrieveProcesses(IEnumerable names) diff --git a/tests/Capgemini.PowerApps.PackageDeployerTemplate.UnitTests/Services/ProcessDeploymentServiceTests.cs b/tests/Capgemini.PowerApps.PackageDeployerTemplate.UnitTests/Services/ProcessDeploymentServiceTests.cs index 68e73d1..8cbe86b 100644 --- a/tests/Capgemini.PowerApps.PackageDeployerTemplate.UnitTests/Services/ProcessDeploymentServiceTests.cs +++ b/tests/Capgemini.PowerApps.PackageDeployerTemplate.UnitTests/Services/ProcessDeploymentServiceTests.cs @@ -3,11 +3,10 @@ using System; using System.Collections.Generic; using System.Linq; - using System.Linq.Expressions; + using System.ServiceModel; using Capgemini.PowerApps.PackageDeployerTemplate.Adapters; using Capgemini.PowerApps.PackageDeployerTemplate.Services; using FluentAssertions; - using Microsoft.Crm.Sdk.Messages; using Microsoft.Extensions.Logging; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Messages; @@ -61,20 +60,10 @@ public void SetStatesBySolution_ProcessInComponentsToDeactivateList_DeactivatesP { var solutionProcesses = new List { GetProcess(Constants.Workflow.StateCodeActive) }; this.MockBySolutionProcesses(solutionProcesses); - this.MockExecuteMultipleResponse( - null, - svc => svc.ExecuteMultiple( - It.Is>( - reqs => reqs.Cast().Any( - req => - req.EntityMoniker.LogicalName == Constants.Workflow.LogicalName && - req.EntityMoniker.Id == solutionProcesses.First().Id && - req.State.Value == Constants.Workflow.StateCodeInactive && - req.Status.Value == Constants.Workflow.StatusCodeInactive)), - It.IsAny(), - It.IsAny(), - It.IsAny()), - true); + this.crmServiceAdapterMock.Setup( + crmSvc => crmSvc.Execute( + It.Is(u => u.Target.GetAttributeValue(Constants.Workflow.Fields.StateCode).Value == Constants.Workflow.StateCodeInactive))) + .Verifiable(); this.processDeploymentSvc.SetStatesBySolution( Solutions, @@ -95,14 +84,13 @@ public void SetStatesBySolution_WithUserParameter_ExecutesAsUser() GetProcess(Constants.Workflow.StateCodeInactive), }; this.MockBySolutionProcesses(solutionProcesses); - this.MockExecuteMultipleResponse( - null, - svc => svc.ExecuteMultiple( - It.IsAny>(), + + this.crmServiceAdapterMock.Setup( + crmSvc => crmSvc.Execute( + It.IsAny(), userToImpersonate, - It.IsAny(), - It.IsAny(), - It.IsAny())); + It.IsAny())) + .Verifiable(); this.processDeploymentSvc.SetStatesBySolution( Solutions, user: userToImpersonate); @@ -165,7 +153,10 @@ public void SetStates_ProcessInComponentsToActivateFound_ActivatesProcess() { var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; this.MockSetStatesProcesses(foundProcesses); - this.MockExecuteMultipleResponse(); + this.crmServiceAdapterMock.Setup( + crmSvc => crmSvc.Execute( + It.Is(u => u.Target.GetAttributeValue(Constants.Workflow.Fields.StateCode).Value == Constants.Workflow.StateCodeActive))) + .Verifiable(); this.processDeploymentSvc.SetStates(new List { @@ -180,20 +171,10 @@ public void SetStates_ProcessInComponentsToDeactivateFound_DectivatesProcess() { var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeActive) }; this.MockSetStatesProcesses(foundProcesses); - this.MockExecuteMultipleResponse( - null, - svc => svc.ExecuteMultiple( - It.Is>( - reqs => reqs.Cast().Any( - req => - req.EntityMoniker.LogicalName == Constants.Workflow.LogicalName && - req.EntityMoniker.Id == foundProcesses.First().Id && - req.State.Value == Constants.Workflow.StateCodeInactive && - req.Status.Value == Constants.Workflow.StatusCodeInactive)), - It.IsAny(), - It.IsAny(), - It.IsAny()), - true); + this.crmServiceAdapterMock.Setup( + crmSvc => crmSvc.Execute( + It.Is(u => u.Target.GetAttributeValue(Constants.Workflow.Fields.StateCode).Value == Constants.Workflow.StateCodeInactive))) + .Verifiable(); this.processDeploymentSvc.SetStates(Enumerable.Empty(), new List { @@ -208,17 +189,13 @@ public void SetStates_WithUserParameter_ExecutesAsUser() { var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; this.MockSetStatesProcesses(foundProcesses); - var userToImpersonate = "licenseduser@domaincom"; - this.MockExecuteMultipleResponse( - null, - svc => svc.ExecuteMultiple( - It.IsAny>(), - userToImpersonate, - It.IsAny(), - It.IsAny(), - It.IsAny()), - true); + this.crmServiceAdapterMock.Setup( + crmSvc => crmSvc.Execute( + It.IsAny(), + userToImpersonate, + It.IsAny())) + .Verifiable(); this.processDeploymentSvc.SetStates( new List @@ -236,11 +213,10 @@ public void SetStates_WithError_LogsError() { var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; this.MockSetStatesProcesses(foundProcesses); - - var failedResponse = GetNewExecuteMultipleResponse(isFaulted: true); - var fault = new OrganizationServiceFault { Message = "Some error." }; - failedResponse.Responses.Add(new ExecuteMultipleResponseItem { Fault = fault }); - this.MockExecuteMultipleResponse(failedResponse); + var fault = new FaultException(new OrganizationServiceFault(), "Some error."); + this.crmServiceAdapterMock + .Setup(crmSvc => crmSvc.Execute(It.IsAny())) + .Throws(fault); this.processDeploymentSvc.SetStates( new List @@ -253,33 +229,25 @@ public void SetStates_WithError_LogsError() } [Fact] - public void SetStates_WithError_RetriesUpToThreeTimes() + public void SetStates_WithError_RetriesWhenOtherRequestsAreSuccessful() { - var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; + var foundProcesses = new List + { + GetProcess(Constants.Workflow.StateCodeInactive), + GetProcess(Constants.Workflow.StateCodeInactive), + }; this.MockSetStatesProcesses(foundProcesses); - - var failedResponse = GetNewExecuteMultipleResponse(isFaulted: true); - failedResponse.Responses.Add(new ExecuteMultipleResponseItem { Fault = new OrganizationServiceFault { Message = "Flow error." } }); - - var successfulResponse = GetNewExecuteMultipleResponse(isFaulted: false); - successfulResponse.Responses.Add(new ExecuteMultipleResponseItem()); - - this.MockExecuteMultipleResponses(failedResponse, failedResponse, successfulResponse); + var fault = new FaultException(new OrganizationServiceFault()); + this.crmServiceAdapterMock + .SetupSequence(crmSvc => crmSvc.Execute(It.IsAny())) + .Throws(fault) + .Returns(new UpdateResponse()); this.processDeploymentSvc.SetStates( - new List - { - foundProcesses.First().GetAttributeValue(Constants.Workflow.Fields.Name), - }, + foundProcesses.Select(p => p.GetAttributeValue(Constants.Workflow.Fields.Name)).ToList(), Enumerable.Empty()); - this.crmServiceAdapterMock.Verify( - svc => svc.ExecuteMultiple( - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny()), - Times.Exactly(3)); + this.crmServiceAdapterMock.Verify(svc => svc.Execute(It.IsAny()), Times.Exactly(3)); } [Fact] @@ -288,11 +256,13 @@ public void SetStates_WithError_LogsErrorAfterThirdRetry() var foundProcesses = new List { GetProcess(Constants.Workflow.StateCodeInactive) }; this.MockSetStatesProcesses(foundProcesses); - var failedResponse = GetNewExecuteMultipleResponse(isFaulted: true); - var fault = new OrganizationServiceFault { Message = "Some error." }; - failedResponse.Responses.Add(new ExecuteMultipleResponseItem { Fault = fault }); - - this.MockExecuteMultipleResponses(failedResponse, failedResponse, failedResponse); + var faultException = new FaultException( + new OrganizationServiceFault { Message = "Some message." }); + this.crmServiceAdapterMock + .SetupSequence(crmSvc => crmSvc.Execute(It.IsAny())) + .Throws(faultException) + .Throws(faultException) + .Returns(new UpdateResponse()); this.processDeploymentSvc.SetStates( new List @@ -301,19 +271,7 @@ public void SetStates_WithError_LogsErrorAfterThirdRetry() }, Enumerable.Empty()); - this.loggerMock.VerifyLog(l => l.LogError(It.Is(s => s.Contains(fault.Message)))); - } - - private static ExecuteMultipleResponse GetNewExecuteMultipleResponse(bool isFaulted) - { - return new ExecuteMultipleResponse - { - Results = new ParameterCollection - { - { "Responses", new ExecuteMultipleResponseItemCollection() }, - { "IsFaulted", isFaulted }, - }, - }; + this.loggerMock.VerifyLog(l => l.LogError(It.Is(s => s.Contains(faultException.Message)))); } private static Entity GetProcess(int stateCode) @@ -323,7 +281,7 @@ private static Entity GetProcess(int stateCode) Attributes = { { - Constants.Workflow.Fields.Name, "Process" + Constants.Workflow.Fields.Name, $"Process {Guid.NewGuid()}" }, { Constants.Workflow.Fields.StateCode, new OptionSetValue(stateCode) @@ -332,49 +290,6 @@ private static Entity GetProcess(int stateCode) }; } - private void MockExecuteMultipleResponse(ExecuteMultipleResponse response = null, Expression> expression = null, bool verifiable = false) - { - if (expression == null) - { - expression = svc => svc.ExecuteMultiple( - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny()); - } - - if (response == null) - { - response = new ExecuteMultipleResponse(); - response.Results["Responses"] = new ExecuteMultipleResponseItemCollection(); - } - - var returnResult = this.crmServiceAdapterMock - .Setup(expression) - .Returns(response); - - if (verifiable) - { - returnResult.Verifiable(); - } - } - - private void MockExecuteMultipleResponses(params ExecuteMultipleResponse[] responses) - { - var setup = this.crmServiceAdapterMock - .SetupSequence(svc => - svc.ExecuteMultiple( - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny())); - - foreach (var response in responses) - { - setup.Returns(response); - } - } - private void MockSetStatesProcesses(IList processes) { this.crmServiceAdapterMock.Setup(