Entity Framework Dynamic LINQ support (#79)

* Working on EF SQL adapter

* Progress

* Small progress

* Some progress on SQL translation

* Progress

* Progress

* More progress

* Fix test

* Fix tests

* More ways to get entitytype

* Params

* More refactoring

* More refactoring

* More refactor

* Make copy of cached fields so they don't get polluted

* Add entity type filter

* Change to navigation filter

* Add date support

* Add default field search support

* Adding support for junction tables. Better recursion detection.

* Don't exit on collections

* Fix build warnings

* Fix collection field support on default fields

* Add support for money data type

* PR feedback

* Date math and fixed missing and exists

* Fix double validation

* Allow controlling what includes are called

* Allow changing include name

* Add checks for navigation collections (#80)

* test(sql): add tests for using navigations

* fix: correct tests to highlight navigation bug

* fix: add check if skip navigation is collection

* test: add checks to confirm skip navigations

* fix: respect ISkipNavigation#IsCollection

* Update deps

* Dummy change

---------

Co-authored-by: Luke Gordon <[email protected]>

.gitignore

@@ -51,4 +51,5 @@ _NCrunch_*
 # Rider auto-generates .iml files, and contentModel.xml
 **/.idea/**/*.iml
 **/.idea/**/contentModel.xml
-**/.idea/**/modules.xml
+**/.idea/**/modules.xml
+**/.idea/copilot/chatSessions/

Foundatio.Parsers.sln

@@ -22,6 +22,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Foundatio.Parsers.ElasticQueries.Tests", "tests\Foundatio.Parsers.ElasticQueries.Tests\Foundatio.Parsers.ElasticQueries.Tests.csproj", "{C353E601-874C-4855-9B01-2980BC624BC4}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foundatio.Parsers.SqlQueries", "src\Foundatio.Parsers.SqlQueries\Foundatio.Parsers.SqlQueries.csproj", "{53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foundatio.Parsers.SqlQueries.Tests", "tests\Foundatio.Parsers.SqlQueries.Tests\Foundatio.Parsers.SqlQueries.Tests.csproj", "{B30934E7-C69F-465B-BACF-61AAEC7DA775}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -44,6 +48,14 @@ Global
 		{C353E601-874C-4855-9B01-2980BC624BC4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C353E601-874C-4855-9B01-2980BC624BC4}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C353E601-874C-4855-9B01-2980BC624BC4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B30934E7-C69F-465B-BACF-61AAEC7DA775}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B30934E7-C69F-465B-BACF-61AAEC7DA775}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B30934E7-C69F-465B-BACF-61AAEC7DA775}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B30934E7-C69F-465B-BACF-61AAEC7DA775}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

README.md

@@ -32,12 +32,12 @@ Debug.WriteLine(DebugQueryVisitor.Run(result));
 Here is the parse result as shown from the `DebugQueryVisitor`
 ```
 Group:
-  Left - Term: 
+  Left - Term:
       TermMax: 2
       TermMin: 1
       MinInclusive: True
       MaxInclusive: True
-      Field: 
+      Field:
           Name: field
 ```
 
@@ -80,7 +80,7 @@ System.Diagnostics.Debug.Assert(query == generatedQuery);
   - Automatically resolves non-analyzed keyword sub-fields for sorting and aggregations
   - Aliases can be defined right on your NEST mappings
     - Supports both root and inner field name aliases
-
-## Thanks to all the people who have contributed
 
+## Thanks to all the people who have contributed
+
 [![contributors](https://contributors-img.web.app/image?repo=FoundatioFx/Foundatio.Parsers)](https://github.com/FoundatioFx/Foundatio.Parsers/graphs/contributors)

build/common.props

@@ -1,7 +1,7 @@
 <Project>
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0</TargetFrameworks>
+    <TargetFrameworks>netstandard2.0;</TargetFrameworks>
     <Product>Foundatio.Parsers</Product>
     <Description>A lucene style query parser that is extensible and allows additional syntax features.</Description>
     <PackageProjectUrl>https://github.com/FoundatioFx/Foundatio.Parsers</PackageProjectUrl>

docker-compose.yml

@@ -22,6 +22,27 @@ services:
     networks:
       - foundatio
 
+  sqlserver:
+    image: mcr.microsoft.com/azure-sql-edge:1.0.7
+    ports:
+      - "1433:1433" # login with sa:P@ssword1
+    environment:
+      - "ACCEPT_EULA=Y"
+      - "SA_PASSWORD=P@ssword1"
+      - "MSSQL_PID=Developer"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "/opt/mssql-tools/bin/sqlcmd",
+          "-Usa",
+          "-PP@ssword1",
+          "-Q",
+          "select 1",
+        ]
+      interval: 1s
+      retries: 20
+
   ready:
     image: andrewlock/wait-for-dependencies
     command: elasticsearch:9200

src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj

@@ -3,8 +3,8 @@
   <ItemGroup>
     <PackageReference Include="Exceptionless.DateTimeExtensions" Version="3.4.3" />
     <PackageReference Include="NEST" Version="7.17.5" />
-    <PackageReference Include="System.Text.Json" Version="6.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0" />
+    <PackageReference Include="System.Text.Json" Version="8.0.4" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Foundatio.Parsers.LuceneQueries\Foundatio.Parsers.LuceneQueries.csproj" />

src/Foundatio.Parsers.LuceneQueries/Visitors/IncludeVisitor.cs

@@ -13,15 +13,17 @@ public class IncludeVisitor : ChainableMutatingQueryVisitor
 {
     private readonly LuceneQueryParser _parser = new();
     private readonly ShouldSkipIncludeFunc _shouldSkipInclude;
+    private readonly string _includeName;
 
-    public IncludeVisitor(ShouldSkipIncludeFunc shouldSkipInclude = null)
+    public IncludeVisitor(ShouldSkipIncludeFunc shouldSkipInclude = null, string includeName = "include")
     {
         _shouldSkipInclude = shouldSkipInclude;
+        _includeName = includeName;
     }
 
     public override async Task<IQueryNode> VisitAsync(TermNode node, IQueryVisitorContext context)
     {
-        if (node.Field != "@include" || (_shouldSkipInclude != null && _shouldSkipInclude(node, context)))
+        if (node.Field != "@" + _includeName || (_shouldSkipInclude != null && _shouldSkipInclude(node, context)))
             return node;
 
         var includeResolver = context.GetIncludeResolver();
@@ -34,7 +36,7 @@ public override async Task<IQueryNode> VisitAsync(TermNode node, IQueryVisitorCo
         var includeStack = context.GetIncludeStack();
         if (includeStack.Contains(node.Term))
         {
-            context.AddValidationError($"Recursive include ({node.Term})");
+            context.AddValidationError($"Recursive {_includeName} ({node.Term})");
             return node;
         }
 
@@ -43,7 +45,7 @@ public override async Task<IQueryNode> VisitAsync(TermNode node, IQueryVisitorCo
             string includedQuery = await includeResolver(node.Term).ConfigureAwait(false);
             if (includedQuery == null)
             {
-                context.AddValidationError($"Unresolved include ({node.Term})");
+                context.AddValidationError($"Unresolved {_includeName} ({node.Term})");
                 context.GetValidationResult().UnresolvedIncludes.Add(node.Term);
             }
 
@@ -62,7 +64,7 @@ public override async Task<IQueryNode> VisitAsync(TermNode node, IQueryVisitorCo
         }
         catch (Exception ex)
         {
-            context.AddValidationError($"Error in include resolver callback when resolving include ({node.Term}): {ex.Message}");
+            context.AddValidationError($"Error in {_includeName} resolver callback when resolving {_includeName} ({node.Term}): {ex.Message}");
             context.GetValidationResult().UnresolvedIncludes.Add(node.Term);
 
             return node;

src/Foundatio.Parsers.LuceneQueries/Visitors/ValidationVisitor.cs

@@ -33,7 +33,7 @@ public override void Visit(TermNode node, IQueryVisitorContext context)
         AddOperation(validationResult, node.GetOperationType(), node.Field);
 
         var validationOptions = context.GetValidationOptions();
-        if (validationOptions != null && !validationOptions.AllowLeadingWildcards && node.Term != null && (node.Term.StartsWith("*") || node.Term.StartsWith("?")))
+        if (validationOptions is { AllowLeadingWildcards: false } && node.Term != null && (node.Term.StartsWith("*") || node.Term.StartsWith("?")))
             context.AddValidationError("Terms must not start with a wildcard: " + node.Term);
     }
 
@@ -72,7 +72,7 @@ private void AddField(QueryValidationResult validationResult, IFieldQueryNode no
         }
         else
         {
-            var fields = node.GetDefaultFields(context.DefaultFields);
+            string[] fields = node.GetDefaultFields(context.DefaultFields);
             if (fields == null || fields.Length == 0)
                 validationResult.ReferencedFields.Add("");
             else
@@ -120,11 +120,14 @@ internal async Task ApplyQueryRestrictions(IQueryVisitorContext context)
 
         if (options.AllowedFields.Count > 0 && result.ReferencedFields.Count > 0)
         {
-            var nonAllowedFields = result.ReferencedFields.Where(f => !String.IsNullOrEmpty(f)).Distinct().ToList();
-            foreach (var field in options.AllowedFields)
+            var nonAllowedFields = new List<string>();
+            foreach (var field in result.ReferencedFields)
             {
-                if (nonAllowedFields.Any(f => !String.IsNullOrEmpty(f) && field.Equals(f)))
-                    nonAllowedFields.Remove(field);
+                if (String.IsNullOrWhiteSpace(field))
+                    continue;
+
+                if (!options.AllowedFields.Contains(field, StringComparer.OrdinalIgnoreCase))
+                    nonAllowedFields.Add(field);
             }
             if (nonAllowedFields.Count > 0)
                 context.AddValidationError($"Query uses field(s) ({String.Join(",", nonAllowedFields)}) that are not allowed to be used.");