KA issue 32 - optimize CQL for Lantana (#352)

* Update GlycemicControlHypoglycemicInitialPopulation.cql

* SupplementalDataElements changes

* MATGlobalCommonFunctions R4 changes

* MATGlobalCommonFunctions R3 changes

* MATGlobalCommonFunctions QICore R4 changes

* Fixes / optimizations

* Revert non-CaseRepresentation content

* Restore additional content

---------

Co-authored-by: JP <[email protected]>
Co-authored-by: Jonathan Percival <[email protected]>

cqf-fhir-benchmark/src/test/java/org/opencds/cqf/fhir/benchmark/Measures.java

@@ -25,8 +25,8 @@
 public class Measures {
     private When when;
 
-    @Setup(Level.Iteration)
-    public void setupIteration() throws Exception {
+    @Setup(Level.Trial)
+    public void setupTrial() throws Exception {
         var evaluationOptions = MeasureEvaluationOptions.defaultOptions();
         evaluationOptions.getEvaluationSettings().setLibraryCache(new HashMap<>());
         this.when = Measure.given()

cqf-fhir-benchmark/src/test/java/org/opencds/cqf/fhir/benchmark/MeasuresAdditionalData.java

@@ -28,8 +28,8 @@
 public class MeasuresAdditionalData {
     private When when;
 
-    @Setup(Level.Iteration)
-    public void setupIteration() throws Exception {
+    @Setup(Level.Trial)
+    public void setupTrial() throws Exception {
         var evaluationOptions = MeasureEvaluationOptions.defaultOptions();
         evaluationOptions.getEvaluationSettings().setLibraryCache(new HashMap<>());
 

cqf-fhir-benchmark/src/test/java/org/opencds/cqf/fhir/benchmark/PlanDefinitions.java

@@ -34,8 +34,8 @@ public class PlanDefinitions {
 
     private Apply apply;
 
-    @Setup(Level.Iteration)
-    public void setupIteration() throws Exception {
+    @Setup(Level.Trial)
+    public void setupTrial() throws Exception {
         this.apply = PlanDefinition.Assert.that(
                         "ANCDT17",
                         "Patient/5946f880-b197-400b-9caa-a3c661d23041",

cqf-fhir-benchmark/src/test/java/org/opencds/cqf/fhir/benchmark/Questionnaires.java

@@ -35,8 +35,8 @@ public class Questionnaires {
 
     private QuestionnaireResult result;
 
-    @Setup(Level.Iteration)
-    public void setupIteration() throws Exception {
+    @Setup(Level.Trial)
+    public void setupTrial() throws Exception {
         this.result = TestQuestionnaire.Assert.that(new IdType("Questionnaire", "ASLPA1"), "positive")
                 .withRepository(REPOSITORY)
                 .withParameters(parameters(
@@ -47,7 +47,7 @@ public void setupIteration() throws Exception {
 
     @Benchmark
     @Fork(warmups = 1, value = 1)
-    @Measurement(iterations = 2, timeUnit = TimeUnit.SECONDS)
+    @Measurement(iterations = 10, timeUnit = TimeUnit.SECONDS)
     @OutputTimeUnit(TimeUnit.SECONDS)
     public void test(Blackhole bh) throws Exception {
         // The Blackhole ensures that the compiler doesn't optimize

cqf-fhir-benchmark/src/test/java/org/opencds/cqf/fhir/benchmark/TerminologyProviders.java

@@ -41,24 +41,24 @@ public class TerminologyProviders {
             .withId("http://cts.nlm.nih.gov/fhir/ValueSet/2.16.840.1.113762.1.4.1190.58")
             .withVersion("20220304");
 
-    @Setup(Level.Iteration)
-    public void setupIteration() throws Exception {
+    @Setup(Level.Trial)
+    public void setupTrial() throws Exception {
         var repository = TestRepositoryFactory.createRepository(
                 FhirContext.forR4Cached(), MeasureProcessorEvaluateTest.class, "CaseRepresentation101");
         this.terminologyProvider = new RepositoryTerminologyProvider(repository);
     }
 
     @Benchmark
     @Fork(warmups = 1, value = 1)
-    @Measurement(iterations = 3, timeUnit = TimeUnit.SECONDS)
+    @Measurement(iterations = 10, timeUnit = TimeUnit.MILLISECONDS)
     @OutputTimeUnit(TimeUnit.SECONDS)
     public void testSmall(Blackhole bh) throws Exception {
         bh.consume(this.terminologyProvider.in(smallCode, smallValueSet));
     }
 
     @Benchmark
     @Fork(warmups = 1, value = 1)
-    @Measurement(iterations = 3, timeUnit = TimeUnit.SECONDS)
+    @Measurement(iterations = 10, timeUnit = TimeUnit.MILLISECONDS)
     @OutputTimeUnit(TimeUnit.SECONDS)
     public void testLarge(Blackhole bh) throws Exception {
         // NOTE: There's an issue with this valueSet in that it does not contain many duplicate codes.

cqf-fhir-cql/src/main/java/org/opencds/cqf/fhir/cql/engine/retrieve/RetrieveProvider.java

@@ -313,9 +313,16 @@ public void populateTerminologySearchParams(
         var sp = this.resolver.getSearchParameterDefinition(dataType, codePath, RestSearchParameterTypeEnum.TOKEN);
         if (codes != null) {
             List<IQueryParameterType> codeList = new ArrayList<>();
-            for (Code code : codes) {
-                codeList.add(new TokenParam(
-                        new InternalCodingDt().setSystem(code.getSystem()).setCode(code.getCode())));
+            // TODO: Fix up the RetrieveProvider API in the engine
+            // This is stupid hacky
+            for (Object code : codes) {
+                if (code instanceof Code) {
+                    var c = (Code) code;
+                    codeList.add(new TokenParam(
+                            new InternalCodingDt().setSystem(c.getSystem()).setCode(c.getCode())));
+                } else if (code != null) {
+                    codeList.add(new TokenParam(code.toString()));
+                }
             }
             searchParams.put(sp.getName(), codeList);
         } else if (valueSet != null) {

cqf-fhir-cr/src/test/resources/org/opencds/cqf/fhir/cr/measure/r4/CaseRepresentation101/cql/GlycemicControlHypoglycemicInitialPopulation.cql

@@ -82,7 +82,7 @@ define function "GetMedicationCode"(choice Choice<FHIR.CodeableConcept, FHIR.Ref
 
 define function "GetMedication"(reference Reference ):
   singleton from (
-    [Medication] M where M.id = Global.GetId(reference.reference)
+    [Medication: id in {Global.GetId(reference.reference)}]
   )
 
 define function "HospitalizationWithObservationOrEmergency"(TheEncounter FHIR.Encounter ):
@@ -102,7 +102,7 @@ define function "HospitalizationWithObservationOrEmergency"(TheEncounter FHIR.En
 
 define function "GetSpecimen"(reference Reference):
   singleton from (
-    [Specimen] S where S.id = Global.GetId(reference.reference)
+    [Specimen: id in {Global.GetId(reference.reference)}]
   )
 
 define function "GetIdExtensions"(domainResource DomainResource):

cqf-fhir-cr/src/test/resources/org/opencds/cqf/fhir/cr/measure/r4/CaseRepresentation101/cql/MATGlobalCommonFunctionsFHIR4.cql

@@ -269,20 +269,20 @@ define function "GetId"(uri String ):
 /*Returns the Condition resources referenced by the diagnosis element of the Encounter*/
 define function "EncounterDiagnosis"(Encounter Encounter ):
   Encounter.diagnosis D
-    return singleton from ([Condition] C where C.id = "GetId"(D.condition.reference))
+    return singleton from ([Condition: id in {"GetId"(D.condition.reference)}])
 
   // Returns the condition that is specified as the principal diagnosis for the encounter
   // TODO: BTR 2019-07-30: Shouldn't need the FHIRHelpers reference here, investigate
 
 define function "PrincipalDiagnosis"(Encounter Encounter ):
   (singleton from (Encounter.diagnosis D where FHIRHelpers.ToInteger(D.rank) = 1)) PD
-  		return singleton from ([Condition] C where C.id = "GetId"(PD.condition.reference))
+  		return singleton from ([Condition: id in {"GetId"(PD.condition.reference)}])
   // Returns the location for the given location reference
 
 /*Returns the Location resource specified by the given reference*/
 define function "GetLocation"(reference Reference ):
   singleton from (
-    [Location] L where L.id = GetId(reference.reference)
+    [Location: id in {GetId(reference.reference)}]
   )
 
 /*NOTE: Extensions are not the preferred approach, but are used as a way to access
@@ -341,7 +341,7 @@ define function "GetMedicationCode"(request MedicationRequest ):
   if request.medication is CodeableConcept then
   	  request.medication as CodeableConcept
   	else
-  	  (singleton from ([Medication] M where M.id = GetId((request.medication as Reference).reference))).code
+  	  (singleton from ([Medication: id in {GetId((request.medication as Reference).reference)}])).code
 
 /*Given an interval, return true if the interval has a starting boundary specified (i.e. the start of the interval is not null and not the minimum DateTime value)*/
 define function "HasStart"(period Interval<DateTime> ):

cqf-fhir-utility/src/main/java/org/opencds/cqf/fhir/utility/matcher/ResourceMatcher.java

@@ -120,10 +120,9 @@ default boolean matches(String name, List<IQueryParameterType> params, IBaseReso
                 } else if (param instanceof DateParam) {
                     match = isMatchDate((DateParam) param, r);
                 } else if (param instanceof TokenParam) {
-                    var codes = getCodes(r);
-                    if (codes == null) {
-                        match = isMatchToken((TokenParam) param, r);
-                    } else {
+                    match = isMatchToken((TokenParam) param, r);
+                    if (!match) {
+                        var codes = getCodes(r);
                         match = isMatchCoding((TokenParam) param, r, codes);
                     }
                 } else if (param instanceof UriParam) {
@@ -214,6 +213,10 @@ default boolean isMatchToken(TokenParam param, IBase pathResult) {
     }
 
     default boolean isMatchCoding(TokenParam param, IBase pathResult, List<BaseCodingDt> codes) {
+        if (codes == null || codes.isEmpty()) {
+            return false;
+        }
+
         // in value set
         if (param.getModifier() == TokenParamModifier.IN) {
             return inValueSet(codes);

cqf-fhir-utility/src/main/java/org/opencds/cqf/fhir/utility/matcher/ResourceMatcherDSTU3.java

@@ -61,9 +61,8 @@ public List<BaseCodingDt> getCodes(IBase codeElement) {
                     .getCoding().stream()
                             .map(code -> new InternalCodingDt(code.getSystem(), code.getCode()))
                             .collect(Collectors.toList());
-        } else {
-            return null;
         }
+
         return resolvedCodes;
     }
 

cqf-fhir-utility/src/main/java/org/opencds/cqf/fhir/utility/matcher/ResourceMatcherR4.java

@@ -62,9 +62,8 @@ public List<BaseCodingDt> getCodes(IBase codeElement) {
                     .getCoding().stream()
                             .map(code -> new InternalCodingDt(code.getSystem(), code.getCode()))
                             .collect(Collectors.toList());
-        } else {
-            return null;
         }
+
         return resolvedCodes;
     }
 

cqf-fhir-utility/src/main/java/org/opencds/cqf/fhir/utility/matcher/ResourceMatcherR5.java

@@ -62,9 +62,8 @@ public List<BaseCodingDt> getCodes(IBase codeElement) {
                     .getCoding().stream()
                             .map(code -> new InternalCodingDt(code.getSystem(), code.getCode()))
                             .collect(Collectors.toList());
-        } else {
-            return null;
         }
+
         return resolvedCodes;
     }
 

cqf-fhir-utility/src/main/java/org/opencds/cqf/fhir/utility/repository/IGFileStructureRepository.java

@@ -7,6 +7,7 @@
 import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.api.MethodOutcome;
+import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.rest.server.exceptions.UnclassifiedServerFailureException;
 import ca.uhn.fhir.util.BundleBuilder;
@@ -20,6 +21,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -29,6 +31,7 @@
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.opencds.cqf.fhir.api.Repository;
+import org.opencds.cqf.fhir.utility.Ids;
 import org.opencds.cqf.fhir.utility.matcher.ResourceMatcher;
 
 /**
@@ -193,9 +196,9 @@ protected <T extends IBaseResource> MethodOutcome writeLocation(T resource, Stri
         return new MethodOutcome(resource.getIdElement());
     }
 
-    protected <T extends IBaseResource> List<T> readLocation(Class<T> resourceClass) {
+    protected <T extends IBaseResource> Map<IIdType, T> readLocation(Class<T> resourceClass) {
         var location = this.directoryForResource(resourceClass);
-        List<T> resources = new ArrayList<>();
+        var resources = new HashMap<IIdType, T>();
         var inputDir = new File(location);
         if (inputDir.isDirectory()) {
             for (var file : inputDir.listFiles()) {
@@ -205,7 +208,7 @@ protected <T extends IBaseResource> List<T> readLocation(Class<T> resourceClass)
                     try {
                         var r = this.readLocation(resourceClass, file.getPath());
                         if (r.fhirType().equals(resourceClass.getSimpleName())) {
-                            resources.add(r);
+                            resources.put(r.getIdElement().toUnqualifiedVersionless(), r);
                         }
                     } catch (RuntimeException e) {
                         // intentionally empty
@@ -308,24 +311,49 @@ public <B extends IBaseBundle, T extends IBaseResource> B search(
             Map<String, String> headers) {
         BundleBuilder builder = new BundleBuilder(this.fhirContext);
 
-        var resourceList = readLocation(resourceType);
+        var resourceIdMap = readLocation(resourceType);
         if (searchParameters == null || searchParameters.isEmpty()) {
-            resourceList.forEach(builder::addCollectionEntry);
-        } else {
-            for (var resource : resourceList) {
-                boolean include = true;
-                for (var nextEntry : searchParameters.entrySet()) {
-                    var paramName = nextEntry.getKey();
-                    if (!resourceMatcher.matches(paramName, nextEntry.getValue(), resource)) {
-                        include = false;
-                        break;
-                    }
+            resourceIdMap.values().forEach(builder::addCollectionEntry);
+            builder.setType("searchset");
+            return (B) builder.getBundle();
+        }
+
+        Collection<T> candidates;
+        if (searchParameters.containsKey("_id")) {
+            // We are consuming the _id parameter in this if statement
+            var idQueries = searchParameters.get("_id");
+            searchParameters.remove("_id");
+
+            var idResources = new ArrayList<T>(idQueries.size());
+            for (var idQuery : idQueries) {
+                var idToken = (TokenParam) idQuery;
+                // Need to construct the equivalent "UnqualifiedVersionless" id that the map is
+                // indexed by. If an id has a version it won't match. Need apples-to-apples Ids types
+                var id = Ids.newId(fhirContext, resourceType.getSimpleName(), idToken.getValue());
+                var r = resourceIdMap.get(id);
+                if (r != null) {
+                    idResources.add(r);
                 }
+            }
 
-                if (include) {
-                    builder.addCollectionEntry(resource);
+            candidates = idResources;
+        } else {
+            candidates = resourceIdMap.values();
+        }
+
+        for (var resource : candidates) {
+            boolean include = true;
+            for (var nextEntry : searchParameters.entrySet()) {
+                var paramName = nextEntry.getKey();
+                if (!resourceMatcher.matches(paramName, nextEntry.getValue(), resource)) {
+                    include = false;
+                    break;
                 }
             }
+
+            if (include) {
+                builder.addCollectionEntry(resource);
+            }
         }
 
         builder.setType("searchset");

cqf-fhir-utility/src/main/java/org/opencds/cqf/fhir/utility/repository/InMemoryFhirRepository.java

@@ -122,12 +122,11 @@ public <B extends IBaseBundle, T extends IBaseResource> B search(
             Map<String, List<IQueryParameterType>> searchParameters,
             Map<String, String> headers) {
         BundleBuilder builder = new BundleBuilder(this.context);
-        builder.setType("searchset");
-
         var resourceIdMap = resourceMap.computeIfAbsent(resourceType.getSimpleName(), r -> new HashMap<>());
 
         if (searchParameters == null || searchParameters.isEmpty()) {
             resourceIdMap.values().forEach(builder::addCollectionEntry);
+            builder.setType("searchset");
             return (B) builder.getBundle();
         }
 
@@ -172,6 +171,7 @@ public <B extends IBaseBundle, T extends IBaseResource> B search(
             }
         }
 
+        builder.setType("searchset");
         return (B) builder.getBundle();
     }