From 6ab3f80ac8e044ab7bcf9e5c6382e42fe6ca1518 Mon Sep 17 00:00:00 2001 From: David Justo Date: Tue, 16 Apr 2024 19:28:01 -0700 Subject: [PATCH 1/5] manually track popReceipt --- src/DurableTask.AzureStorage/Storage/Queue.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.AzureStorage/Storage/Queue.cs b/src/DurableTask.AzureStorage/Storage/Queue.cs index 46b4115d8..27b068788 100644 --- a/src/DurableTask.AzureStorage/Storage/Queue.cs +++ b/src/DurableTask.AzureStorage/Storage/Queue.cs @@ -27,12 +27,14 @@ class Queue readonly AzureStorageClient azureStorageClient; readonly AzureStorageOrchestrationServiceStats stats; readonly QueueClient queueClient; + readonly Dictionary messageIdPopReceipts; public Queue(AzureStorageClient azureStorageClient, QueueServiceClient queueServiceClient, string queueName) { this.azureStorageClient = azureStorageClient; this.stats = this.azureStorageClient.Stats; this.queueClient = queueServiceClient.GetQueueClient(queueName); + this.messageIdPopReceipts = new Dictionary(); } public string Name => this.queueClient.Name; @@ -61,15 +63,21 @@ await this.queueClient public async Task UpdateMessageAsync(QueueMessage queueMessage, TimeSpan visibilityTimeout, Guid? clientRequestId = null, CancellationToken cancellationToken = default) { + // TODO: handle case where popREceipt cannot be found + this.messageIdPopReceipts.TryGetValue(queueMessage.MessageId, out string popReceipt); using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId); - await this.queueClient + Response response = await this.queueClient .UpdateMessageAsync( queueMessage.MessageId, - queueMessage.PopReceipt, + popReceipt, visibilityTimeout: visibilityTimeout, cancellationToken: cancellationToken) .DecorateFailure(); + this.messageIdPopReceipts[queueMessage.MessageId] = response.Value.PopReceipt; + + UpdateReceipt receipt = response.Value; + this.stats.MessagesUpdated.Increment(); } @@ -82,6 +90,7 @@ await this.queueClient queueMessage.PopReceipt, cancellationToken) .DecorateFailure(); + this.messageIdPopReceipts.Remove(queueMessage.MessageId); } public async Task GetMessageAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken = default) @@ -93,6 +102,7 @@ await this.queueClient return null; } + this.messageIdPopReceipts.Add(message.MessageId, message.PopReceipt); this.stats.MessagesRead.Increment(); return message; } From e62f2a7e0a48ddae105ebcc2f0cf7720448dd4d2 Mon Sep 17 00:00:00 2001 From: David Justo Date: Wed, 17 Apr 2024 18:52:31 -0700 Subject: [PATCH 2/5] refactor, add comments --- src/DurableTask.AzureStorage/Storage/Queue.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.AzureStorage/Storage/Queue.cs b/src/DurableTask.AzureStorage/Storage/Queue.cs index 27b068788..331dfd056 100644 --- a/src/DurableTask.AzureStorage/Storage/Queue.cs +++ b/src/DurableTask.AzureStorage/Storage/Queue.cs @@ -27,6 +27,7 @@ class Queue readonly AzureStorageClient azureStorageClient; readonly AzureStorageOrchestrationServiceStats stats; readonly QueueClient queueClient; + // TODO: I don't love this map. I think we should use our own AzStorage Queue class to keep track of popReceipts instead. We may need to change quite a few interfaces in response though, so they take our Queue class instead of the Az Storage SDK Queue abstraction readonly Dictionary messageIdPopReceipts; public Queue(AzureStorageClient azureStorageClient, QueueServiceClient queueServiceClient, string queueName) @@ -63,8 +64,13 @@ await this.queueClient public async Task UpdateMessageAsync(QueueMessage queueMessage, TimeSpan visibilityTimeout, Guid? clientRequestId = null, CancellationToken cancellationToken = default) { - // TODO: handle case where popREceipt cannot be found - this.messageIdPopReceipts.TryGetValue(queueMessage.MessageId, out string popReceipt); + string popReceipt = queueMessage.PopReceipt; // default case + if (this.messageIdPopReceipts.TryGetValue(queueMessage.MessageId, out string foundReceipt)) + { + // TODO: we should log something if we cannot find a pop receipt + popReceipt = foundReceipt; + } + using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId); Response response = await this.queueClient .UpdateMessageAsync( From 05d473ee925e7ced69e74e44c58c48ff166a00a6 Mon Sep 17 00:00:00 2001 From: David Justo Date: Wed, 17 Apr 2024 18:53:28 -0700 Subject: [PATCH 3/5] add versionSuffix for custom build --- src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index b99bbab9a..9afd47b55 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -24,7 +24,7 @@ 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) - preview.5 + preview.5.popReceiptfix.1 $(VersionPrefix).0 $(VersionPrefix).$(FileVersionRevision) From 04ef9b06e755e16c042c0c284b6c0f381206a4e9 Mon Sep 17 00:00:00 2001 From: David Justo Date: Wed, 17 Apr 2024 19:23:45 -0700 Subject: [PATCH 4/5] csproj remove vulns --- src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj | 1 - src/DurableTask.Core/DurableTask.Core.csproj | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj index 9afd47b55..e8ba285b9 100644 --- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj +++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj @@ -45,7 +45,6 @@ - diff --git a/src/DurableTask.Core/DurableTask.Core.csproj b/src/DurableTask.Core/DurableTask.Core.csproj index 802e5e13f..2c4e48416 100644 --- a/src/DurableTask.Core/DurableTask.Core.csproj +++ b/src/DurableTask.Core/DurableTask.Core.csproj @@ -32,7 +32,7 @@ - + From 61d8fecd783fd707746d74e39f6fa090d2749adb Mon Sep 17 00:00:00 2001 From: David Justo Date: Wed, 17 Apr 2024 20:37:35 -0700 Subject: [PATCH 5/5] minimize release temporarily --- azure-pipelines-release.yml | 109 ------------------------------------ 1 file changed, 109 deletions(-) diff --git a/azure-pipelines-release.yml b/azure-pipelines-release.yml index aa2f0cfec..c14684390 100644 --- a/azure-pipelines-release.yml +++ b/azure-pipelines-release.yml @@ -17,10 +17,6 @@ steps: verbosityRestore: Minimal projects: | src/DurableTask.AzureStorage/DurableTask.AzureStorage.sln - src/DurableTask.Emulator/DurableTask.Emulator.csproj - src/DurableTask.ServiceBus/DurableTask.ServiceBus.csproj - src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj - src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj # Build the filtered solution in release mode, specifying the continuous integration flag. - task: VSBuild@1 @@ -32,43 +28,6 @@ steps: configuration: Release msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true -- task: VSBuild@1 - displayName: 'Build (ApplicationInsights)' - inputs: - solution: 'src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj' - vsVersion: '16.0' - logFileVerbosity: minimal - configuration: Release - msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true - -- task: VSBuild@1 - displayName: 'Build (Emulator)' - inputs: - solution: 'src/DurableTask.Emulator/DurableTask.Emulator.csproj' - vsVersion: '16.0' - logFileVerbosity: minimal - configuration: Release - msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true - -- task: VSBuild@1 - displayName: 'Build (ServiceBus)' - inputs: - solution: 'src/DurableTask.ServiceBus/DurableTask.ServiceBus.csproj' - vsVersion: '16.0' - logFileVerbosity: minimal - configuration: Release - msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true - -- task: VSBuild@1 - displayName: 'Build (AzureServiceFabric)' - inputs: - solution: 'src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj' - vsVersion: '16.0' - logFileVerbosity: minimal - configuration: Release - platform: x64 - msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true - - task: UseDotNet@2 displayName: 'Use the .NET Core 2.1 SDK (required for build signing)' inputs: @@ -127,74 +86,6 @@ steps: packDirectory: $(build.artifactStagingDirectory) packagesToPack: 'src/DurableTask.AzureStorage/DurableTask.AzureStorage.sln' -- task: DotNetCoreCLI@2 - displayName: Generate nuget packages - inputs: - command: pack - verbosityPack: Minimal - configuration: Release - nobuild: true - packDirectory: $(build.artifactStagingDirectory) - packagesToPack: 'src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj' - -- task: DotNetCoreCLI@2 - displayName: Generate nuget packages - inputs: - command: pack - verbosityPack: Minimal - configuration: Release - nobuild: true - packDirectory: $(build.artifactStagingDirectory) - packagesToPack: 'src/DurableTask.Emulator/DurableTask.Emulator.csproj' - -- task: DotNetCoreCLI@2 - displayName: Generate nuget packages - inputs: - command: pack - verbosityPack: Minimal - configuration: Release - nobuild: true - packDirectory: $(build.artifactStagingDirectory) - packagesToPack: 'src/DurableTask.ServiceBus/DurableTask.ServiceBus.csproj' - -- task: DotNetCoreCLI@2 - displayName: Generate nuget packages - inputs: - command: pack - verbosityPack: Minimal - configuration: Release - nobuild: true - packDirectory: $(build.artifactStagingDirectory) - packagesToPack: 'src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj' - buildProperties: 'Platform=x64' - -# Digitally sign all the nuget packages with the Microsoft certificate. -# This appears to be an in-place signing job, which is convenient. -- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'ESRP CodeSigning: Nupkg' - inputs: - ConnectedServiceName: 'ESRP Service' - FolderPath: $(build.artifactStagingDirectory) - Pattern: '*.nupkg' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode": "CP-401405", - "OperationCode": "NuGetSign", - "Parameters": {}, - "ToolName": "sign", - "ToolVersion": "1.0" - }, - { - "KeyCode": "CP-401405", - "OperationCode": "NuGetVerify", - "Parameters": {}, - "ToolName": "sign", - "ToolVersion": "1.0" - } - ] - # Make the nuget packages available for download in the ADO portal UI - publish: $(build.artifactStagingDirectory) displayName: 'Publish nuget packages to Artifacts'