[Cosmos DB] Query max concurrency flag (#3669)

* Extending queries to use max concurrency parallelism
* Add new Action Filter for Query Latency Over Efficiency
* Adding new tests to ensure that FhirController has the expected attributes. Adding more tests on top of the new filter. Limiting the execution of the filter to Azure API for FHIR only.

src/Microsoft.Health.Fhir.Core/Features/KnownHeaders.cs

@@ -30,6 +30,7 @@ public static class KnownHeaders
         public const string ProfileValidation = "x-ms-profile-validation";
         public const string CustomAuditHeaderPrefix = "X-MS-AZUREFHIR-AUDIT-";
         public const string FhirUserHeader = "x-ms-fhiruser";
+        public const string QueryLatencyOverEfficiency = "x-ms-query-latency-over-efficiency";
 
         // #conditionalQueryParallelism - Header used to activate parallel conditional-query processing.
         public const string ConditionalQueryProcessingLogic = "x-conditionalquery-processing-logic";

src/Microsoft.Health.Fhir.Core/Registration/AzureApiForFhirRuntimeConfiguration.cs

@@ -18,5 +18,7 @@ public class AzureApiForFhirRuntimeConfiguration : IFhirRuntimeConfiguration
         public bool IsCustomerKeyValidationBackgroundWorkerSupported => false;
 
         public bool IsTransactionSupported => false;
+
+        public bool IsLatencyOverEfficiencySupported => true;
     }
 }

src/Microsoft.Health.Fhir.Core/Registration/AzureHealthDataServicesRuntimeConfiguration.cs

@@ -18,5 +18,7 @@ public class AzureHealthDataServicesRuntimeConfiguration : IFhirRuntimeConfigura
         public bool IsCustomerKeyValidationBackgroundWorkerSupported => true;
 
         public bool IsTransactionSupported => true;
+
+        public bool IsLatencyOverEfficiencySupported => false;
     }
 }

src/Microsoft.Health.Fhir.Core/Registration/IFhirRuntimeConfiguration.cs

@@ -28,5 +28,10 @@ public interface IFhirRuntimeConfiguration
         /// Support to transactions.
         /// </summary>
         bool IsTransactionSupported { get; }
+
+        /// <summary>
+        /// Supports the 'latency-over-efficiency' HTTP header.
+        /// </summary>
+        bool IsLatencyOverEfficiencySupported { get; }
     }
 }

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs

@@ -0,0 +1,49 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Linq;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Health.Api.Features.Audit;
+using Microsoft.Health.Fhir.Api.Controllers;
+using Microsoft.Health.Fhir.Api.Features.Filters;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Api.UnitTests.Controllers
+{
+    [Trait(Traits.OwningTeam, OwningTeam.Fhir)]
+    [Trait(Traits.Category, Categories.Web)]
+    public sealed class FhirControllerTests
+    {
+        [Fact]
+        public void WhenProviderAFhirController_CheckIfAllExpectedServiceFilterAttributesArePresent()
+        {
+            Type[] expectedCustomAttributes = new Type[]
+            {
+                typeof(AuditLoggingFilterAttribute),
+                typeof(OperationOutcomeExceptionFilterAttribute),
+                typeof(ValidateFormatParametersAttribute),
+                typeof(QueryLatencyOverEfficiencyFilterAttribute),
+            };
+
+            ServiceFilterAttribute[] serviceFilterAttributes = Attribute.GetCustomAttributes(typeof(FhirController), typeof(ServiceFilterAttribute))
+                .Select(a => a as ServiceFilterAttribute)
+                .ToArray();
+
+            foreach (Type expectedCustomAttribute in expectedCustomAttributes)
+            {
+                bool attributeWasFound = serviceFilterAttributes.Any(s => s.ServiceType == expectedCustomAttribute);
+
+                if (!attributeWasFound)
+                {
+                    string errorMessage = $"The custom attribute '{expectedCustomAttribute.ToString()}' is not assigned to '{nameof(FhirController)}'.";
+                    Assert.Fail(errorMessage);
+                }
+            }
+        }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/QueryLatencyOverEfficiencyFilterAttributeTests.cs

@@ -0,0 +1,100 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.Abstractions;
+using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.Health.Core.Features.Context;
+using Microsoft.Health.Fhir.Api.Features.Filters;
+using Microsoft.Health.Fhir.Core.Features;
+using Microsoft.Health.Fhir.Core.Features.Context;
+using Microsoft.Health.Fhir.Core.Registration;
+using Microsoft.Health.Fhir.Core.UnitTests.Features.Context;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
+using NSubstitute;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Api.UnitTests.Features.Filters
+{
+    [Trait(Traits.OwningTeam, OwningTeam.Fhir)]
+    [Trait(Traits.Category, Categories.Web)]
+    public sealed class QueryLatencyOverEfficiencyFilterAttributeTests
+    {
+        private readonly IFhirRuntimeConfiguration _azureApiForFhirConfiguration = new AzureApiForFhirRuntimeConfiguration();
+        private readonly IFhirRuntimeConfiguration _azureHealthDataServicesFhirConfiguration = new AzureHealthDataServicesRuntimeConfiguration();
+
+        [Fact]
+        public void GivenAValidHttpContextForAzureApiForFhir_WhenItContainsALatencyOverEfficiencyFlag_ThenFhirContextIsDecorated()
+        {
+            var httpRequest = GetFakeHttpContext(isLatencyOverEfficiencyEnabled: true);
+
+            var filter = new QueryLatencyOverEfficiencyFilterAttribute(httpRequest.RequestContext, _azureApiForFhirConfiguration);
+            filter.OnActionExecuting(httpRequest.ActionContext);
+
+            var fhirContextPropertyBag = httpRequest.RequestContext.RequestContext.Properties;
+
+            Assert.True(fhirContextPropertyBag.ContainsKey(KnownQueryParameterNames.OptimizeConcurrency));
+            Assert.Equal(true, fhirContextPropertyBag[KnownQueryParameterNames.OptimizeConcurrency]);
+        }
+
+        [Fact]
+        public void GivenAValidHttpContextForAzureHealthDataService_WhenItContainsALatencyOverEfficiencyFlag_ThenFhirContextIsNotDecorated()
+        {
+            // The latency-over-efficiency flag is only applicable to Azure API for FHIR.
+
+            var httpRequest = GetFakeHttpContext(isLatencyOverEfficiencyEnabled: true);
+
+            var filter = new QueryLatencyOverEfficiencyFilterAttribute(httpRequest.RequestContext, _azureHealthDataServicesFhirConfiguration);
+            filter.OnActionExecuting(httpRequest.ActionContext);
+
+            var fhirContextPropertyBag = httpRequest.RequestContext.RequestContext.Properties;
+
+            Assert.False(fhirContextPropertyBag.ContainsKey(KnownQueryParameterNames.OptimizeConcurrency));
+        }
+
+        [Fact]
+        public void GivenAValidHttpContext_WhenItDoesNotContainALatencyOverEfficiencyFlag_ThenFhirContextIsClean()
+        {
+            var httpRequest = GetFakeHttpContext(isLatencyOverEfficiencyEnabled: false);
+
+            var filter = new QueryLatencyOverEfficiencyFilterAttribute(httpRequest.RequestContext, _azureApiForFhirConfiguration);
+            filter.OnActionExecuting(httpRequest.ActionContext);
+
+            var fhirContextPropertyBag = httpRequest.RequestContext.RequestContext.Properties;
+
+            Assert.False(fhirContextPropertyBag.ContainsKey(KnownQueryParameterNames.OptimizeConcurrency));
+        }
+
+        private static (RequestContextAccessor<IFhirRequestContext> RequestContext, ActionExecutingContext ActionContext) GetFakeHttpContext(bool isLatencyOverEfficiencyEnabled)
+        {
+            var httpContext = new DefaultHttpContext();
+
+            if (isLatencyOverEfficiencyEnabled)
+            {
+                httpContext.Request.Headers[KnownHeaders.QueryLatencyOverEfficiency] = "true";
+            }
+
+            ActionExecutingContext context = new ActionExecutingContext(
+                new ActionContext(
+                    httpContext,
+                    new RouteData(),
+                    new ActionDescriptor()),
+                new List<IFilterMetadata>(),
+                actionArguments: new Dictionary<string, object>(),
+                FilterTestsHelper.CreateMockFhirController());
+
+            DefaultFhirRequestContext fhirRequestContext = new DefaultFhirRequestContext();
+
+            var fhirRequestContextAccessor = Substitute.For<RequestContextAccessor<IFhirRequestContext>>();
+            fhirRequestContextAccessor.RequestContext.Returns(fhirRequestContext);
+
+            return (fhirRequestContextAccessor, context);
+        }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Headers/HttpContextExtensionsTests.cs

@@ -0,0 +1,153 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Health.Fhir.Api.Features.Headers;
+using Microsoft.Health.Fhir.Api.Features.Resources;
+using Microsoft.Health.Fhir.Api.Features.Resources.Bundle;
+using Microsoft.Health.Fhir.Core.Features;
+using Microsoft.Health.Fhir.Core.Features.Context;
+using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Api.UnitTests.Features.Headers
+{
+    [Trait(Traits.OwningTeam, OwningTeam.Fhir)]
+    [Trait(Traits.Category, Categories.Web)]
+    public sealed class HttpContextExtensionsTests
+    {
+        [Fact]
+        public void WhenHttpContextDoesNotHaveCustomHeaders_ReturnDefaultValues()
+        {
+            HttpContext httpContext = GetFakeHttpContext();
+
+            bool isLatencyOverEfficiencyEnabled = httpContext.IsLatencyOverEfficiencyEnabled();
+            Assert.False(isLatencyOverEfficiencyEnabled);
+
+            BundleProcessingLogic bundleProcessingLogic = httpContext.GetBundleProcessingLogic();
+            Assert.Equal(BundleProcessingLogic.Sequential, bundleProcessingLogic);
+
+            // #conditionalQueryParallelism
+            ConditionalQueryProcessingLogic conditionalQueryProcessingLogic = httpContext.GetConditionalQueryProcessingLogic();
+            Assert.Equal(ConditionalQueryProcessingLogic.Sequential, conditionalQueryProcessingLogic);
+        }
+
+        [Theory]
+        [InlineData("", false)]
+        [InlineData(null, false)]
+        [InlineData("false", false)]
+        [InlineData("falsE", false)]
+        [InlineData("FALSE", false)]
+        [InlineData("2112", false)]
+        [InlineData("true", true)]
+        [InlineData("true ", true)]
+        [InlineData("TRUE", true)]
+        [InlineData(" TRUE ", true)]
+        [InlineData("   tRuE   ", true)]
+        public void WhenHttpContextHasCustomHeaders_ReturnIfLatencyOverEfficiencyIsEnabled(string value, bool isEnabled)
+        {
+            var httpHeaders = new Dictionary<string, string>() { { KnownHeaders.QueryLatencyOverEfficiency, value } };
+            HttpContext httpContext = GetFakeHttpContext(httpHeaders);
+
+            bool isLatencyOverEfficiencyEnabled = httpContext.IsLatencyOverEfficiencyEnabled();
+
+            Assert.Equal(isEnabled, isLatencyOverEfficiencyEnabled);
+        }
+
+        [Theory]
+        [InlineData("", ConditionalQueryProcessingLogic.Sequential)]
+        [InlineData(null, ConditionalQueryProcessingLogic.Sequential)]
+        [InlineData("sequential", ConditionalQueryProcessingLogic.Sequential)]
+        [InlineData("sequential ", ConditionalQueryProcessingLogic.Sequential)]
+        [InlineData("Sequential", ConditionalQueryProcessingLogic.Sequential)]
+        [InlineData("2112", ConditionalQueryProcessingLogic.Sequential)]
+        [InlineData("red barchetta", ConditionalQueryProcessingLogic.Sequential)]
+        [InlineData("parallel", ConditionalQueryProcessingLogic.Parallel)]
+        [InlineData("parallel  ", ConditionalQueryProcessingLogic.Parallel)]
+        [InlineData("Parallel", ConditionalQueryProcessingLogic.Parallel)]
+        [InlineData(" pArAllEl  ", ConditionalQueryProcessingLogic.Parallel)]
+        [InlineData("PARALLEL", ConditionalQueryProcessingLogic.Parallel)]
+        public void WhenHttpContextHasCustomHeaders_ReturnIfConditionalQueryProcessingLogicIsSet(string value, ConditionalQueryProcessingLogic processingLogic)
+        {
+            // #conditionalQueryParallelism
+
+            var httpHeaders = new Dictionary<string, string>() { { KnownHeaders.ConditionalQueryProcessingLogic, value } };
+            HttpContext httpContext = GetFakeHttpContext(httpHeaders);
+
+            ConditionalQueryProcessingLogic conditionalQueryProcessingLogic = httpContext.GetConditionalQueryProcessingLogic();
+
+            Assert.Equal(processingLogic, conditionalQueryProcessingLogic);
+        }
+
+        [Theory]
+        [InlineData("", BundleProcessingLogic.Sequential)]
+        [InlineData(null, BundleProcessingLogic.Sequential)]
+        [InlineData("sequential", BundleProcessingLogic.Sequential)]
+        [InlineData("sequential ", BundleProcessingLogic.Sequential)]
+        [InlineData("Sequential", BundleProcessingLogic.Sequential)]
+        [InlineData("2112", BundleProcessingLogic.Sequential)]
+        [InlineData("red barchetta", BundleProcessingLogic.Sequential)]
+        [InlineData("parallel", BundleProcessingLogic.Parallel)]
+        [InlineData("parallel  ", BundleProcessingLogic.Parallel)]
+        [InlineData("Parallel", BundleProcessingLogic.Parallel)]
+        [InlineData(" pArAllEl  ", BundleProcessingLogic.Parallel)]
+        [InlineData("PARALLEL", BundleProcessingLogic.Parallel)]
+        public void WhenHttpContextHasCustomHeaders_ReturnIfBundleProcessingLogicIsSet(string value, BundleProcessingLogic processingLogic)
+        {
+            var httpHeaders = new Dictionary<string, string>() { { BundleOrchestratorNamingConventions.HttpHeaderBundleProcessingLogic, value } };
+            HttpContext httpContext = GetFakeHttpContext(httpHeaders);
+
+            BundleProcessingLogic bundleProcessingLogic = httpContext.GetBundleProcessingLogic();
+
+            Assert.Equal(processingLogic, bundleProcessingLogic);
+        }
+
+        [Fact]
+        public void WhenProvidedAFhirRequestContext_ThenDecorateItWithOptimizeConcurrency()
+        {
+            // #conditionalQueryParallelism
+
+            IFhirRequestContext fhirRequestContext = new Core.UnitTests.Features.Context.DefaultFhirRequestContext()
+            {
+                BaseUri = new Uri("https://localhost/"),
+                CorrelationId = Guid.NewGuid().ToString(),
+                ResponseHeaders = new HeaderDictionary(),
+                RequestHeaders = new HeaderDictionary(),
+            };
+
+            fhirRequestContext.DecorateRequestContextWithOptimizedConcurrency();
+
+            Assert.True(fhirRequestContext.Properties.ContainsKey(KnownQueryParameterNames.OptimizeConcurrency));
+            Assert.Equal(true, fhirRequestContext.Properties[KnownQueryParameterNames.OptimizeConcurrency]);
+        }
+
+        private static HttpContext GetFakeHttpContext(IReadOnlyDictionary<string, string> optionalHttpHeaders = default)
+        {
+            var httpContext = new DefaultHttpContext()
+            {
+                Request =
+                {
+                    Scheme = "https",
+                    Host = new HostString("localhost"),
+                    PathBase = new PathString("/"),
+                },
+            };
+
+            if (optionalHttpHeaders != null)
+            {
+                foreach (var header in optionalHttpHeaders)
+                {
+                    httpContext.Request.Headers.Append(header.Key, header.Value);
+                }
+            }
+
+            return httpContext;
+        }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerEdgeCaseTests.cs

@@ -11,6 +11,7 @@
 using System.IO;
 using System.Linq;
 using System.Threading;
+using Azure;
 using Hl7.Fhir.Model;
 using Hl7.Fhir.Serialization;
 using Hl7.Fhir.Specification.Navigation;
@@ -87,9 +88,9 @@ public void GivenABundle_WhenProcessedWithConditionalQueryMaxParallelism_TheFhir
             // 4 - If the created HTTP request does not contain the header "x-conditionalquery-processing-logic" set as "parallel", then the key "_optimizeConcurrency"
             // is not expected in the FHIR Request Context property bag.
 
-            var bundleHandlerComponents = GetBundleHandlerComponents(new BundleRequestOptions() { MaxParallelism = maxParallelism });
+            var requestContext = CreateRequestContextForBundleHandlerProcessing(new BundleRequestOptions() { MaxParallelism = maxParallelism });
 
-            var fhirContextPropertyBag = bundleHandlerComponents.FhirRequestContext.Properties;
+            var fhirContextPropertyBag = requestContext.Properties;
 
             if (maxParallelism)
             {
@@ -102,7 +103,7 @@ public void GivenABundle_WhenProcessedWithConditionalQueryMaxParallelism_TheFhir
             }
         }
 
-        private (IRouter Router, BundleConfiguration BundleConfiguration, IMediator Mediator, BundleHandler BundleHandler, IFhirRequestContext FhirRequestContext) GetBundleHandlerComponents(BundleRequestOptions options)
+        private IFhirRequestContext CreateRequestContextForBundleHandlerProcessing(BundleRequestOptions options)
         {
             IRouter router = Substitute.For<IRouter>();
 
@@ -147,6 +148,11 @@ public void GivenABundle_WhenProcessedWithConditionalQueryMaxParallelism_TheFhir
                 httpContext.Request.Headers[KnownHeaders.ConditionalQueryProcessingLogic] = new StringValues("parallel");
             }
 
+            if (options.QueryLatencyOverEfficiency)
+            {
+                httpContext.Request.Headers[KnownHeaders.QueryLatencyOverEfficiency] = new StringValues("true");
+            }
+
             httpContextAccessor.HttpContext.Returns(httpContext);
 
             var transactionHandler = Substitute.For<ITransactionHandler>();
@@ -174,7 +180,7 @@ public void GivenABundle_WhenProcessedWithConditionalQueryMaxParallelism_TheFhir
                 mediator,
                 NullLogger<BundleHandler>.Instance);
 
-            return (router, bundleConfiguration, mediator, bundleHandler, fhirRequestContextAccessor.RequestContext);
+            return fhirRequestContextAccessor.RequestContext;
         }
 
         private IFeatureCollection CreateFeatureCollection(IRouter router)
@@ -214,6 +220,8 @@ private IFeatureCollection CreateFeatureCollection(IRouter router)
         private sealed class BundleRequestOptions()
         {
             public bool MaxParallelism { get; set; } = false;
+
+            public bool QueryLatencyOverEfficiency { get; set; } = false;
         }
     }
 }