From 09a41f9628460bd79475f915125946304e208373 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 24 Aug 2024 09:35:40 -0500 Subject: [PATCH] Entity Framework Dynamic LINQ support (#79) * Working on EF SQL adapter * Progress * Small progress * Some progress on SQL translation * Progress * Progress * More progress * Fix test * Fix tests * More ways to get entitytype * Params * More refactoring * More refactoring * More refactor * Make copy of cached fields so they don't get polluted * Add entity type filter * Change to navigation filter * Add date support * Add default field search support * Adding support for junction tables. Better recursion detection. * Don't exit on collections * Fix build warnings * Fix collection field support on default fields * Add support for money data type * PR feedback * Date math and fixed missing and exists * Fix double validation * Allow controlling what includes are called * Allow changing include name * Add checks for navigation collections (#80) * test(sql): add tests for using navigations * fix: correct tests to highlight navigation bug * fix: add check if skip navigation is collection * test: add checks to confirm skip navigations * fix: respect ISkipNavigation#IsCollection * Update deps * Dummy change --------- Co-authored-by: Luke Gordon --- .gitignore | 3 +- Foundatio.Parsers.sln | 12 + README.md | 8 +- build/common.props | 2 +- docker-compose.yml | 21 ++ .../Foundatio.Parsers.ElasticQueries.csproj | 4 +- .../Visitors/IncludeVisitor.cs | 12 +- .../Visitors/ValidationVisitor.cs | 15 +- .../Extensions/SqlNodeExtensions.cs | 329 +++++++++++++++++ .../Extensions/TypeExtensions.cs | 42 +++ .../Foundatio.Parsers.SqlQueries.csproj | 16 + .../SqlQueryParser.cs | 186 ++++++++++ .../SqlQueryParserConfiguration.cs | 249 +++++++++++++ .../Visitors/GenerateSqlVisitor.cs | 71 ++++ .../Visitors/ISqlQueryVisitorContext.cs | 8 + .../Visitors/SqlQueryVisitorContext.cs | 23 ++ tests/Directory.Build.props | 12 +- .../AggregationParserTests.cs | 95 +++-- .../ElasticQueryParserTests.cs | 302 ++++++++-------- .../DynamicFieldVisitor.cs | 65 ++++ .../Foundatio.Parsers.SqlQueries.Tests.csproj | 14 + .../SampleContext.cs | 171 +++++++++ .../SqlQueryParserTests.cs | 332 ++++++++++++++++++ 23 files changed, 1768 insertions(+), 224 deletions(-) create mode 100644 src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs create mode 100644 src/Foundatio.Parsers.SqlQueries/Extensions/TypeExtensions.cs create mode 100644 src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj create mode 100644 src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs create mode 100644 src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs create mode 100644 src/Foundatio.Parsers.SqlQueries/Visitors/GenerateSqlVisitor.cs create mode 100644 src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs create mode 100644 src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs create mode 100644 tests/Foundatio.Parsers.SqlQueries.Tests/DynamicFieldVisitor.cs create mode 100644 tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj create mode 100644 tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs create mode 100644 tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs diff --git a/.gitignore b/.gitignore index 897f2f01..dd0679d9 100644 --- a/.gitignore +++ b/.gitignore @@ -51,4 +51,5 @@ _NCrunch_* # Rider auto-generates .iml files, and contentModel.xml **/.idea/**/*.iml **/.idea/**/contentModel.xml -**/.idea/**/modules.xml \ No newline at end of file +**/.idea/**/modules.xml +**/.idea/copilot/chatSessions/ diff --git a/Foundatio.Parsers.sln b/Foundatio.Parsers.sln index 3f223df9..71a4681a 100644 --- a/Foundatio.Parsers.sln +++ b/Foundatio.Parsers.sln @@ -22,6 +22,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Foundatio.Parsers.ElasticQueries.Tests", "tests\Foundatio.Parsers.ElasticQueries.Tests\Foundatio.Parsers.ElasticQueries.Tests.csproj", "{C353E601-874C-4855-9B01-2980BC624BC4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foundatio.Parsers.SqlQueries", "src\Foundatio.Parsers.SqlQueries\Foundatio.Parsers.SqlQueries.csproj", "{53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foundatio.Parsers.SqlQueries.Tests", "tests\Foundatio.Parsers.SqlQueries.Tests\Foundatio.Parsers.SqlQueries.Tests.csproj", "{B30934E7-C69F-465B-BACF-61AAEC7DA775}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -44,6 +48,14 @@ Global {C353E601-874C-4855-9B01-2980BC624BC4}.Debug|Any CPU.Build.0 = Debug|Any CPU {C353E601-874C-4855-9B01-2980BC624BC4}.Release|Any CPU.ActiveCfg = Release|Any CPU {C353E601-874C-4855-9B01-2980BC624BC4}.Release|Any CPU.Build.0 = Release|Any CPU + {53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {53C0B9D8-6398-437C-AE15-43BD4CB7AD8D}.Release|Any CPU.Build.0 = Release|Any CPU + {B30934E7-C69F-465B-BACF-61AAEC7DA775}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B30934E7-C69F-465B-BACF-61AAEC7DA775}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B30934E7-C69F-465B-BACF-61AAEC7DA775}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B30934E7-C69F-465B-BACF-61AAEC7DA775}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/README.md b/README.md index 95a441ae..73e6fb9f 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,12 @@ Debug.WriteLine(DebugQueryVisitor.Run(result)); Here is the parse result as shown from the `DebugQueryVisitor` ``` Group: - Left - Term: + Left - Term: TermMax: 2 TermMin: 1 MinInclusive: True MaxInclusive: True - Field: + Field: Name: field ``` @@ -80,7 +80,7 @@ System.Diagnostics.Debug.Assert(query == generatedQuery); - Automatically resolves non-analyzed keyword sub-fields for sorting and aggregations - Aliases can be defined right on your NEST mappings - Supports both root and inner field name aliases - -## Thanks to all the people who have contributed +## Thanks to all the people who have contributed + [![contributors](https://contributors-img.web.app/image?repo=FoundatioFx/Foundatio.Parsers)](https://github.com/FoundatioFx/Foundatio.Parsers/graphs/contributors) diff --git a/build/common.props b/build/common.props index 9644f69b..34024f9b 100644 --- a/build/common.props +++ b/build/common.props @@ -1,7 +1,7 @@ - netstandard2.0 + netstandard2.0; Foundatio.Parsers A lucene style query parser that is extensible and allows additional syntax features. https://github.com/FoundatioFx/Foundatio.Parsers diff --git a/docker-compose.yml b/docker-compose.yml index 86067c01..eb818b54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,27 @@ services: networks: - foundatio + sqlserver: + image: mcr.microsoft.com/azure-sql-edge:1.0.7 + ports: + - "1433:1433" # login with sa:P@ssword1 + environment: + - "ACCEPT_EULA=Y" + - "SA_PASSWORD=P@ssword1" + - "MSSQL_PID=Developer" + healthcheck: + test: + [ + "CMD", + "/opt/mssql-tools/bin/sqlcmd", + "-Usa", + "-PP@ssword1", + "-Q", + "select 1", + ] + interval: 1s + retries: 20 + ready: image: andrewlock/wait-for-dependencies command: elasticsearch:9200 diff --git a/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj b/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj index 270ff397..61260e8d 100644 --- a/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj +++ b/src/Foundatio.Parsers.ElasticQueries/Foundatio.Parsers.ElasticQueries.csproj @@ -3,8 +3,8 @@ - - + + diff --git a/src/Foundatio.Parsers.LuceneQueries/Visitors/IncludeVisitor.cs b/src/Foundatio.Parsers.LuceneQueries/Visitors/IncludeVisitor.cs index 41b4d938..906e50dc 100644 --- a/src/Foundatio.Parsers.LuceneQueries/Visitors/IncludeVisitor.cs +++ b/src/Foundatio.Parsers.LuceneQueries/Visitors/IncludeVisitor.cs @@ -13,15 +13,17 @@ public class IncludeVisitor : ChainableMutatingQueryVisitor { private readonly LuceneQueryParser _parser = new(); private readonly ShouldSkipIncludeFunc _shouldSkipInclude; + private readonly string _includeName; - public IncludeVisitor(ShouldSkipIncludeFunc shouldSkipInclude = null) + public IncludeVisitor(ShouldSkipIncludeFunc shouldSkipInclude = null, string includeName = "include") { _shouldSkipInclude = shouldSkipInclude; + _includeName = includeName; } public override async Task VisitAsync(TermNode node, IQueryVisitorContext context) { - if (node.Field != "@include" || (_shouldSkipInclude != null && _shouldSkipInclude(node, context))) + if (node.Field != "@" + _includeName || (_shouldSkipInclude != null && _shouldSkipInclude(node, context))) return node; var includeResolver = context.GetIncludeResolver(); @@ -34,7 +36,7 @@ public override async Task VisitAsync(TermNode node, IQueryVisitorCo var includeStack = context.GetIncludeStack(); if (includeStack.Contains(node.Term)) { - context.AddValidationError($"Recursive include ({node.Term})"); + context.AddValidationError($"Recursive {_includeName} ({node.Term})"); return node; } @@ -43,7 +45,7 @@ public override async Task VisitAsync(TermNode node, IQueryVisitorCo string includedQuery = await includeResolver(node.Term).ConfigureAwait(false); if (includedQuery == null) { - context.AddValidationError($"Unresolved include ({node.Term})"); + context.AddValidationError($"Unresolved {_includeName} ({node.Term})"); context.GetValidationResult().UnresolvedIncludes.Add(node.Term); } @@ -62,7 +64,7 @@ public override async Task VisitAsync(TermNode node, IQueryVisitorCo } catch (Exception ex) { - context.AddValidationError($"Error in include resolver callback when resolving include ({node.Term}): {ex.Message}"); + context.AddValidationError($"Error in {_includeName} resolver callback when resolving {_includeName} ({node.Term}): {ex.Message}"); context.GetValidationResult().UnresolvedIncludes.Add(node.Term); return node; diff --git a/src/Foundatio.Parsers.LuceneQueries/Visitors/ValidationVisitor.cs b/src/Foundatio.Parsers.LuceneQueries/Visitors/ValidationVisitor.cs index b6b9f82c..721bd861 100644 --- a/src/Foundatio.Parsers.LuceneQueries/Visitors/ValidationVisitor.cs +++ b/src/Foundatio.Parsers.LuceneQueries/Visitors/ValidationVisitor.cs @@ -33,7 +33,7 @@ public override void Visit(TermNode node, IQueryVisitorContext context) AddOperation(validationResult, node.GetOperationType(), node.Field); var validationOptions = context.GetValidationOptions(); - if (validationOptions != null && !validationOptions.AllowLeadingWildcards && node.Term != null && (node.Term.StartsWith("*") || node.Term.StartsWith("?"))) + if (validationOptions is { AllowLeadingWildcards: false } && node.Term != null && (node.Term.StartsWith("*") || node.Term.StartsWith("?"))) context.AddValidationError("Terms must not start with a wildcard: " + node.Term); } @@ -72,7 +72,7 @@ private void AddField(QueryValidationResult validationResult, IFieldQueryNode no } else { - var fields = node.GetDefaultFields(context.DefaultFields); + string[] fields = node.GetDefaultFields(context.DefaultFields); if (fields == null || fields.Length == 0) validationResult.ReferencedFields.Add(""); else @@ -120,11 +120,14 @@ internal async Task ApplyQueryRestrictions(IQueryVisitorContext context) if (options.AllowedFields.Count > 0 && result.ReferencedFields.Count > 0) { - var nonAllowedFields = result.ReferencedFields.Where(f => !String.IsNullOrEmpty(f)).Distinct().ToList(); - foreach (var field in options.AllowedFields) + var nonAllowedFields = new List(); + foreach (var field in result.ReferencedFields) { - if (nonAllowedFields.Any(f => !String.IsNullOrEmpty(f) && field.Equals(f))) - nonAllowedFields.Remove(field); + if (String.IsNullOrWhiteSpace(field)) + continue; + + if (!options.AllowedFields.Contains(field, StringComparer.OrdinalIgnoreCase)) + nonAllowedFields.Add(field); } if (nonAllowedFields.Count > 0) context.AddValidationError($"Query uses field(s) ({String.Join(",", nonAllowedFields)}) that are not allowed to be used."); diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs new file mode 100644 index 00000000..d205adb9 --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -0,0 +1,329 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Foundatio.Parsers.LuceneQueries.Extensions; +using Foundatio.Parsers.LuceneQueries.Nodes; +using Foundatio.Parsers.SqlQueries.Visitors; + +namespace Foundatio.Parsers.SqlQueries.Extensions; + +public static class SqlNodeExtensions +{ + public static string ToDynamicLinqString(this GroupNode node, ISqlQueryVisitorContext context) + { + // support overriding the generated query + if (node.TryGetQuery(out string query)) + return query; + + if (node.Left == null && node.Right == null) + return String.Empty; + + var defaultOperator = context.DefaultOperator; + + var builder = new StringBuilder(); + var op = node.Operator != GroupOperator.Default ? node.Operator : defaultOperator; + + if (node.IsNegated.HasValue && node.IsNegated.Value) + builder.Append("NOT "); + + builder.Append(node.Prefix); + + if (!String.IsNullOrEmpty(node.Field)) + builder.Append(node.Field).Append(':'); + + if (node.HasParens) + builder.Append("("); + + if (node.Left != null) + builder.Append(node.Left is GroupNode groupNode ? groupNode.ToDynamicLinqString(context) : node.Left.ToDynamicLinqString(context)); + + if (node.Left != null && node.Right != null) + { + if (op == GroupOperator.Or || (op == GroupOperator.Default && defaultOperator == GroupOperator.Or)) + builder.Append(" OR "); + else if (node.Right != null) + builder.Append(" AND "); + } + + if (node.Right != null) + builder.Append(node.Right is GroupNode groupNode ? groupNode.ToDynamicLinqString(context) : node.Right.ToDynamicLinqString(context)); + + if (node.HasParens) + builder.Append(")"); + + if (node.Proximity != null) + builder.Append("~" + node.Proximity); + + if (node.Boost != null) + builder.Append("^" + node.Boost); + + return builder.ToString(); + } + + public static string ToDynamicLinqString(this ExistsNode node, ISqlQueryVisitorContext context) + { + if (String.IsNullOrEmpty(node.Field)) + context.AddValidationError("Field is required for exists node queries."); + + // support overriding the generated query + if (node.TryGetQuery(out string query)) + return query; + + var builder = new StringBuilder(); + + builder.Append(node.Field); + if (!node.IsNegated.HasValue || !node.IsNegated.Value) + builder.Append(" != null"); + else + builder.Append(" == null"); + + return builder.ToString(); + } + + public static string ToDynamicLinqString(this MissingNode node, ISqlQueryVisitorContext context) + { + if (String.IsNullOrEmpty(node.Field)) + context.AddValidationError("Field is required for missing node queries."); + + if (!String.IsNullOrEmpty(node.Prefix)) + context.AddValidationError("Prefix is not supported for term range queries."); + + // support overriding the generated query + if (node.TryGetQuery(out string query)) + return query; + + var builder = new StringBuilder(); + + builder.Append(node.Field); + if (!node.IsNegated.HasValue || !node.IsNegated.Value) + builder.Append(" == null"); + else + builder.Append(" != null"); + + return builder.ToString(); + } + + public static EntityFieldInfo GetFieldInfo(List fields, string field) + { + if (fields == null) + return new EntityFieldInfo { Field = field }; + + return fields.FirstOrDefault(f => f.Field.Equals(field, StringComparison.OrdinalIgnoreCase)) ?? + new EntityFieldInfo { Field = field }; + } + + public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorContext context) + { + if (!String.IsNullOrEmpty(node.Prefix)) + context.AddValidationError("Prefix is not supported for term range queries."); + + var builder = new StringBuilder(); + + if (String.IsNullOrEmpty(node.Field)) + { + if (context.DefaultFields == null || context.DefaultFields.Length == 0) + { + context.AddValidationError("Field or DefaultFields is required for term queries."); + return String.Empty; + } + + for (int index = 0; index < context.DefaultFields.Length; index++) + { + builder.Append(index == 0 ? "(" : " OR "); + + var defaultField = GetFieldInfo(context.Fields, context.DefaultFields[index]); + if (defaultField.IsCollection) + { + var dotIndex = defaultField.Field.LastIndexOf('.'); + var collectionField = defaultField.Field.Substring(0, dotIndex); + var fieldName = defaultField.Field.Substring(dotIndex + 1); + + builder.Append(collectionField); + builder.Append(".Any("); + builder.Append(fieldName); + builder.Append(".Contains(\"").Append(node.Term).Append("\")"); + builder.Append(")"); + } + else + { + builder.Append(defaultField.Field).Append(".Contains(\"").Append(node.Term).Append("\")"); + } + + if (index == context.DefaultFields.Length - 1) + builder.Append(")"); + } + + return builder.ToString(); + } + + // support overriding the generated query + if (node.TryGetQuery(out string query)) + return query; + + var field = GetFieldInfo(context.Fields, node.Field); + + if (node.IsNegated.HasValue && node.IsNegated.Value) + builder.Append("!"); + + if (field.IsCollection) + { + var index = node.Field.LastIndexOf('.'); + var collectionField = node.Field.Substring(0, index); + var fieldName = node.Field.Substring(index + 1); + + builder.Append(collectionField); + builder.Append(".Any("); + builder.Append(fieldName); + + if (node.IsNegated.HasValue && node.IsNegated.Value) + builder.Append(" != "); + else + builder.Append(" = "); + + AppendField(builder, field, node.Term); + + builder.Append(")"); + } + else + { + builder.Append(node.Field); + if (node.IsNegated.HasValue && node.IsNegated.Value) + builder.Append(" != "); + else + builder.Append(" = "); + + AppendField(builder, field, node.Term); + } + + return builder.ToString(); + } + + private static void AppendField(StringBuilder builder, EntityFieldInfo field, string term) + { + if (field == null) + return; + + if (field.IsNumber || field.IsBoolean || field.IsMoney) + { + builder.Append(term); + } + else if (field is { IsDate: true }) + { + term = term.Trim(); + if (term.StartsWith("now", StringComparison.OrdinalIgnoreCase)) + { + builder.Append("DateTime.UtcNow"); + + if (term.Length == 3) + return; + + builder.Append("."); + + var method = term[^1..] switch { + "y" => "AddYears", + "M" => "AddMonths", + "d" => "AddDays", + "h" => "AddHours", + "H" => "AddHours", + "m" => "AddMinutes", + "s" => "AddSeconds", + _ => throw new NotSupportedException("Invalid date operation.") + }; + + var subtract = term.Substring(3, 1) == "-"; + + builder.Append(method).Append("(").Append(subtract ? "-" : "").Append(term.Substring(4, term.Length - 5)).Append(")"); + } + else + { + builder.Append("DateTime.Parse(\"" + term + "\")"); + } + } + else + builder.Append("\"" + term + "\""); + } + + public static string ToDynamicLinqString(this TermRangeNode node, ISqlQueryVisitorContext context) + { + if (String.IsNullOrEmpty(node.Field)) + context.AddValidationError("Field is required for term range queries."); + if (!String.IsNullOrEmpty(node.Boost)) + context.AddValidationError("Boost is not supported for term range queries."); + if (!String.IsNullOrEmpty(node.Proximity)) + context.AddValidationError("Proximity is not supported for term range queries."); + + // support overriding the generated query + if (node.TryGetQuery(out string query)) + return query; + + var field = GetFieldInfo(context.Fields, node.Field); + if (!field.IsNumber && !field.IsDate && !field.IsMoney) + context.AddValidationError("Field must be a number, money or date for term range queries."); + + var builder = new StringBuilder(); + + if (node.IsNegated.HasValue && node.IsNegated.Value) + builder.Append("NOT "); + + if (node.Min != null && node.Max != null) + builder.Append("("); + + if (node.Min != null) + { + builder.Append(node.Field); + builder.Append(node.MinInclusive == true ? " >= " : " > "); + AppendField(builder, field, node.Min); + } + + if (node.Min != null && node.Max != null) + builder.Append(" AND "); + + if (node.Max != null) + { + builder.Append(node.Field); + builder.Append(node.MaxInclusive == true ? " <= " : " < "); + AppendField(builder, field, node.Max); + } + + if (node.Min != null && node.Max != null) + builder.Append(")"); + + return builder.ToString(); + } + + public static string ToDynamicLinqString(this IQueryNode node, ISqlQueryVisitorContext context) + { + return node switch + { + GroupNode groupNode => groupNode.ToDynamicLinqString(context), + ExistsNode existsNode => existsNode.ToDynamicLinqString(context), + MissingNode missingNode => missingNode.ToDynamicLinqString(context), + TermNode termNode => termNode.ToDynamicLinqString(context), + TermRangeNode termRangeNode => termRangeNode.ToDynamicLinqString(context), + _ => throw new NotSupportedException($"Node type {node.GetType().Name} is not supported.") + }; + } + + private const string QueryKey = "Query"; + public static void SetQuery(this IQueryNode node, string query) + { + node.Data[QueryKey] = query; + } + + public static string GetQuery(this IQueryNode node) + { + return node.Data.TryGetValue(QueryKey, out object query) ? query as string : null; + } + + public static bool TryGetQuery(this IQueryNode node, out string query) + { + query = null; + return node.Data.TryGetValue(QueryKey, out object value) && (query = value as string) != null; + } + + public static void RemoveQuery(this IQueryNode node) + { + node.Data.Remove(QueryKey); + } +} diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/TypeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/TypeExtensions.cs new file mode 100644 index 00000000..1592177a --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/TypeExtensions.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; + +namespace Foundatio.Parsers.SqlQueries.Extensions; + +public static class TypeExtensions +{ + private static readonly IList _integerTypes = new List() + { + typeof (byte), + typeof (short), + typeof (int), + typeof (long), + typeof (sbyte), + typeof (ushort), + typeof (uint), + typeof (ulong), + typeof (byte?), + typeof (short?), + typeof (int?), + typeof (long?), + typeof (sbyte?), + typeof (ushort?), + typeof (uint?), + typeof (ulong?) + }; + + public static Type UnwrapNullable(this Type type) + { + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>)) + return Nullable.GetUnderlyingType(type); + + return type; + } + + public static bool IsString(this Type type) => type == typeof(string); + public static bool IsDateTime(this Type typeToCheck) => typeToCheck == typeof(DateTime) || typeToCheck == typeof(DateTime?); + public static bool IsBoolean(this Type typeToCheck) => typeToCheck == typeof(bool) || typeToCheck == typeof(bool?); + public static bool IsNumeric(this Type type) => type.IsFloatingPoint() || type.IsIntegerBased(); + public static bool IsIntegerBased(this Type type) => _integerTypes.Contains(type); + public static bool IsFloatingPoint(this Type type) => type == typeof(decimal) || type == typeof(float) || type == typeof(double); +} diff --git a/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj b/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj new file mode 100644 index 00000000..feab314e --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/Foundatio.Parsers.SqlQueries.csproj @@ -0,0 +1,16 @@ + + + + net8.0; + + + + + + + + + + + + diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs new file mode 100644 index 00000000..c0e52721 --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Threading.Tasks; +using Foundatio.Parsers.LuceneQueries; +using Foundatio.Parsers.LuceneQueries.Extensions; +using Foundatio.Parsers.LuceneQueries.Nodes; +using Foundatio.Parsers.LuceneQueries.Visitors; +using Foundatio.Parsers.SqlQueries.Extensions; +using Foundatio.Parsers.SqlQueries.Visitors; +using Microsoft.EntityFrameworkCore.Metadata; +using Pegasus.Common; + +namespace Foundatio.Parsers.SqlQueries; + +public class SqlQueryParser : LuceneQueryParser { + public SqlQueryParser(Action configure = null) { + var config = new SqlQueryParserConfiguration(); + configure?.Invoke(config); + Configuration = config; + } + + public SqlQueryParserConfiguration Configuration { get; } + + public override async Task ParseAsync(string query, IQueryVisitorContext context = null) { + query ??= String.Empty; + context ??= new SqlQueryVisitorContext(); + + SetupQueryVisitorContextDefaults(context); + try { + var result = await base.ParseAsync(query, context).ConfigureAwait(false); + switch (context.QueryType) { + case QueryTypes.Aggregation: + result = await Configuration.AggregationVisitor.AcceptAsync(result, context).ConfigureAwait(false); + break; + case QueryTypes.Query: + result = await Configuration.QueryVisitor.AcceptAsync(result, context).ConfigureAwait(false); + break; + case QueryTypes.Sort: + result = await Configuration.SortVisitor.AcceptAsync(result, context).ConfigureAwait(false); + break; + } + + return result; + } catch (FormatException ex) { + var cursor = ex.Data["cursor"] as Cursor; + context.GetValidationResult().QueryType = context.QueryType; + context.AddValidationError(ex.Message, cursor.Column); + + return null; + } + } + + private static readonly ConcurrentDictionary> _entityFieldCache = new(); + public async Task ValidateAsync(string query, SqlQueryVisitorContext context) + { + var node = await ParseAsync(query, context); + return await ValidationVisitor.RunAsync(node, context); + } + + public async Task ToDynamicLinqAsync(string query, SqlQueryVisitorContext context) + { + var node = await ParseAsync(query, context); + var result = context.GetValidationResult(); + if (!result.IsValid) + throw new ValidationException("Invalid query: " + result.Message); + + return await GenerateSqlVisitor.RunAsync(node, context); + } + + public SqlQueryVisitorContext GetContext(IEntityType entityType) + { + if (!_entityFieldCache.TryGetValue(entityType, out var fields)) + { + fields = new List(); + AddEntityFields(fields, entityType); + _entityFieldCache.TryAdd(entityType, fields); + } + + // make copy of fields list to avoid modifying the cached list + fields = fields.ToList(); + + var validationOptions = new QueryValidationOptions(); + foreach (string field in fields.Select(f => f.Field)) + validationOptions.AllowedFields.Add(field); + + Configuration.SetValidationOptions(validationOptions); + return new SqlQueryVisitorContext + { + Fields = fields, + ValidationOptions = validationOptions + }; + } + + private void AddEntityFields(List fields, IEntityType entityType, Stack entityTypeStack = null, string prefix = null, bool isCollection = false, int depth = 0) + { + entityTypeStack ??= new Stack(); + + if (depth > 0 && entityTypeStack.Contains(entityType)) + return; + + entityTypeStack.Push(entityType); + + if (depth > Configuration.MaxFieldDepth) + return; + + prefix ??= ""; + + foreach (var property in entityType.GetProperties()) + { + if (!Configuration.EntityTypePropertyFilter(property)) + continue; + + string propertyPath = prefix + property.Name; + fields.Add(new EntityFieldInfo + { + Field = propertyPath, + IsNumber = property.ClrType.UnwrapNullable().IsNumeric(), + IsDate = property.ClrType.UnwrapNullable().IsDateTime(), + IsBoolean = property.ClrType.UnwrapNullable().IsBoolean(), + IsCollection = isCollection + }); + } + + foreach (var nav in entityType.GetNavigations()) + { + if (!Configuration.EntityTypeNavigationFilter(nav)) + continue; + + string propertyPath = prefix + nav.Name; + bool isNavCollection = nav is IReadOnlyNavigationBase { IsCollection: true }; + + AddEntityFields(fields, nav.TargetEntityType, entityTypeStack, propertyPath + ".", isNavCollection, depth + 1); + } + + foreach (var skipNav in entityType.GetSkipNavigations()) + { + if (!Configuration.EntityTypeSkipNavigationFilter(skipNav)) + continue; + + string propertyPath = prefix + skipNav.Name; + + AddEntityFields(fields, skipNav.TargetEntityType, entityTypeStack, propertyPath + ".", skipNav.IsCollection, depth + 1); + } + + entityTypeStack.Pop(); + } + + private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context) + { + if (!context.Data.ContainsKey("@OriginalContextResolver")) + context.SetValue("@OriginalContextResolver", context.GetFieldResolver()); + + context.SetFieldResolver(async (field, context) => + { + string resolvedField = null; + if (context.Data.TryGetValue("@OriginalContextResolver", out var data) && data is QueryFieldResolver resolver) + { + var contextResolvedField = await resolver(field, context).ConfigureAwait(false); + if (contextResolvedField != null) + resolvedField = contextResolvedField; + } + + if (Configuration.FieldResolver != null) + { + var configResolvedField = await Configuration.FieldResolver(resolvedField ?? field, context).ConfigureAwait(false); + if (configResolvedField != null) + resolvedField = configResolvedField; + } + + return resolvedField; + }); + + if (Configuration.ValidationOptions != null && !context.HasValidationOptions()) + context.SetValidationOptions(Configuration.ValidationOptions); + + if (context.QueryType == QueryTypes.Query) + { + context.SetDefaultFields(Configuration.DefaultFields); + if (Configuration.IncludeResolver != null && context.GetIncludeResolver() == null) + context.SetIncludeResolver(Configuration.IncludeResolver); + } + } +} diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs new file mode 100644 index 00000000..7c6e5652 --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Foundatio.Parsers.LuceneQueries; +using Foundatio.Parsers.LuceneQueries.Visitors; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Foundatio.Parsers.SqlQueries; + +public class SqlQueryParserConfiguration { + private ILogger _logger = NullLogger.Instance; + + public SqlQueryParserConfiguration() { + AddSortVisitor(new TermToFieldVisitor(), 0); + AddVisitor(new FieldResolverQueryVisitor((field, context) => FieldResolver != null ? FieldResolver(field, context) : Task.FromResult(null)), 10); + AddVisitor(new ValidationVisitor(), 30); + } + + public ILoggerFactory LoggerFactory { get; private set; } = NullLoggerFactory.Instance; + public string[] DefaultFields { get; private set; } + + public int MaxFieldDepth { get; private set; } = 10; + public QueryFieldResolver FieldResolver { get; private set; } + public EntityTypePropertyFilter EntityTypePropertyFilter { get; private set; } = static _ => true; + public EntityTypeNavigationFilter EntityTypeNavigationFilter { get; private set; } = static _ => true; + public EntityTypeSkipNavigationFilter EntityTypeSkipNavigationFilter { get; private set; } = static _ => true; + public IncludeResolver IncludeResolver { get; private set; } + public QueryValidationOptions ValidationOptions { get; private set; } + public ChainedQueryVisitor SortVisitor { get; } = new(); + public ChainedQueryVisitor QueryVisitor { get; } = new(); + public ChainedQueryVisitor AggregationVisitor { get; } = new(); + + public SqlQueryParserConfiguration SetLoggerFactory(ILoggerFactory loggerFactory) { + LoggerFactory = loggerFactory ?? NullLoggerFactory.Instance; + _logger = loggerFactory.CreateLogger(); + + return this; + } + + public SqlQueryParserConfiguration SetDefaultFields(string[] fields) { + DefaultFields = fields; + return this; + } + + public SqlQueryParserConfiguration SetFieldDepth(int maxFieldDepth) { + MaxFieldDepth = maxFieldDepth; + return this; + } + + public SqlQueryParserConfiguration UseEntityTypePropertyFilter(EntityTypePropertyFilter filter) { + EntityTypePropertyFilter = filter; + return this; + } + + public SqlQueryParserConfiguration UseEntityTypeNavigationFilter(EntityTypeNavigationFilter filter) { + EntityTypeNavigationFilter = filter; + return this; + } + + public SqlQueryParserConfiguration UseEntityTypeSkipNavigationFilter(EntityTypeSkipNavigationFilter filter) { + EntityTypeSkipNavigationFilter = filter; + return this; + } + + public SqlQueryParserConfiguration UseFieldResolver(QueryFieldResolver resolver, int priority = 10) { + FieldResolver = resolver; + ReplaceVisitor(new FieldResolverQueryVisitor(resolver), priority); + + return this; + } + + public SqlQueryParserConfiguration UseFieldMap(IDictionary fields, int priority = 10) { + if (fields != null) + return UseFieldResolver(fields.ToHierarchicalFieldResolver(), priority); + + return UseFieldResolver(null); + } + + public SqlQueryParserConfiguration UseIncludes(IncludeResolver includeResolver, ShouldSkipIncludeFunc shouldSkipInclude = null, string includeName = "include", int priority = 0) { + IncludeResolver = includeResolver; + + return AddVisitor(new IncludeVisitor(shouldSkipInclude, includeName), priority); + } + + public SqlQueryParserConfiguration UseIncludes(Func resolveInclude, ShouldSkipIncludeFunc shouldSkipInclude = null, string includeName = "include", int priority = 0) { + return UseIncludes(name => Task.FromResult(resolveInclude(name)), shouldSkipInclude, includeName, priority); + } + + public SqlQueryParserConfiguration UseIncludes(IDictionary includes, ShouldSkipIncludeFunc shouldSkipInclude = null, string includeName = "include", int priority = 0) { + return UseIncludes(name => includes.ContainsKey(name) ? includes[name] : null, shouldSkipInclude, includeName, priority); + } + + public SqlQueryParserConfiguration SetValidationOptions(QueryValidationOptions options) { + ValidationOptions = options; + return this; + } + + #region Combined Visitor Management + + public SqlQueryParserConfiguration AddVisitor(IChainableQueryVisitor visitor, int priority = 0) { + QueryVisitor.AddVisitor(visitor, priority); + AggregationVisitor.AddVisitor(visitor, priority); + SortVisitor.AddVisitor(visitor, priority); + + return this; + } + + public SqlQueryParserConfiguration RemoveVisitor() where T : IChainableQueryVisitor { + QueryVisitor.RemoveVisitor(); + AggregationVisitor.RemoveVisitor(); + SortVisitor.RemoveVisitor(); + + return this; + } + + public SqlQueryParserConfiguration ReplaceVisitor(IChainableQueryVisitor visitor, int? newPriority = null) where T : IChainableQueryVisitor { + QueryVisitor.ReplaceVisitor(visitor, newPriority); + AggregationVisitor.ReplaceVisitor(visitor, newPriority); + SortVisitor.ReplaceVisitor(visitor, newPriority); + + return this; + } + + public SqlQueryParserConfiguration AddVisitorBefore(IChainableQueryVisitor visitor) { + QueryVisitor.AddVisitorBefore(visitor); + AggregationVisitor.AddVisitorBefore(visitor); + SortVisitor.AddVisitorBefore(visitor); + + return this; + } + + public SqlQueryParserConfiguration AddVisitorAfter(IChainableQueryVisitor visitor) { + QueryVisitor.AddVisitorAfter(visitor); + AggregationVisitor.AddVisitorAfter(visitor); + SortVisitor.AddVisitorAfter(visitor); + + return this; + } + + #endregion + + #region Query Visitor Management + + public SqlQueryParserConfiguration AddQueryVisitor(IChainableQueryVisitor visitor, int priority = 0) { + QueryVisitor.AddVisitor(visitor, priority); + + return this; + } + + public SqlQueryParserConfiguration RemoveQueryVisitor() where T : IChainableQueryVisitor { + QueryVisitor.RemoveVisitor(); + + return this; + } + + public SqlQueryParserConfiguration ReplaceQueryVisitor(IChainableQueryVisitor visitor, int? newPriority = null) where T : IChainableQueryVisitor { + QueryVisitor.ReplaceVisitor(visitor, newPriority); + + return this; + } + + public SqlQueryParserConfiguration AddQueryVisitorBefore(IChainableQueryVisitor visitor) { + QueryVisitor.AddVisitorBefore(visitor); + + return this; + } + + public SqlQueryParserConfiguration AddQueryVisitorAfter(IChainableQueryVisitor visitor) { + QueryVisitor.AddVisitorAfter(visitor); + + return this; + } + + #endregion + + #region Sort Visitor Management + + public SqlQueryParserConfiguration AddSortVisitor(IChainableQueryVisitor visitor, int priority = 0) { + SortVisitor.AddVisitor(visitor, priority); + + return this; + } + + public SqlQueryParserConfiguration RemoveSortVisitor() where T : IChainableQueryVisitor { + SortVisitor.RemoveVisitor(); + + return this; + } + + public SqlQueryParserConfiguration ReplaceSortVisitor(IChainableQueryVisitor visitor, int? newPriority = null) where T : IChainableQueryVisitor { + SortVisitor.ReplaceVisitor(visitor, newPriority); + + return this; + } + + public SqlQueryParserConfiguration AddSortVisitorBefore(IChainableQueryVisitor visitor) { + SortVisitor.AddVisitorBefore(visitor); + + return this; + } + + public SqlQueryParserConfiguration AddSortVisitorAfter(IChainableQueryVisitor visitor) { + SortVisitor.AddVisitorAfter(visitor); + + return this; + } + + #endregion + + #region Aggregation Visitor Management + + public SqlQueryParserConfiguration AddAggregationVisitor(IChainableQueryVisitor visitor, int priority = 0) { + AggregationVisitor.AddVisitor(visitor, priority); + + return this; + } + + public SqlQueryParserConfiguration RemoveAggregationVisitor() where T : IChainableQueryVisitor { + AggregationVisitor.RemoveVisitor(); + + return this; + } + + public SqlQueryParserConfiguration ReplaceAggregationVisitor(IChainableQueryVisitor visitor, int? newPriority = null) where T : IChainableQueryVisitor { + AggregationVisitor.ReplaceVisitor(visitor, newPriority); + + return this; + } + + public SqlQueryParserConfiguration AddAggregationVisitorBefore(IChainableQueryVisitor visitor) { + AggregationVisitor.AddVisitorBefore(visitor); + + return this; + } + + public SqlQueryParserConfiguration AddAggregationVisitorAfter(IChainableQueryVisitor visitor) { + AggregationVisitor.AddVisitorAfter(visitor); + + return this; + } + + #endregion +} + +public delegate bool EntityTypeNavigationFilter(INavigation navigation); +public delegate bool EntityTypeSkipNavigationFilter(ISkipNavigation navigation); +public delegate bool EntityTypePropertyFilter(IProperty property); diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/GenerateSqlVisitor.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/GenerateSqlVisitor.cs new file mode 100644 index 00000000..c281e35d --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/GenerateSqlVisitor.cs @@ -0,0 +1,71 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using Foundatio.Parsers.LuceneQueries.Nodes; +using Foundatio.Parsers.LuceneQueries.Visitors; +using Foundatio.Parsers.SqlQueries.Extensions; + +namespace Foundatio.Parsers.SqlQueries.Visitors; + +public class GenerateSqlVisitor : QueryNodeVisitorWithResultBase +{ + private readonly StringBuilder _builder = new(); + + public override Task VisitAsync(GroupNode node, IQueryVisitorContext context) + { + if (context is not ISqlQueryVisitorContext sqlContext) + throw new InvalidOperationException("The context must be an ISqlQueryVisitorContext."); + + _builder.Append(node.ToDynamicLinqString(sqlContext)); + + return Task.CompletedTask; + } + + public override void Visit(TermNode node, IQueryVisitorContext context) + { + if (context is not ISqlQueryVisitorContext sqlContext) + throw new InvalidOperationException("The context must be an ISqlQueryVisitorContext."); + + _builder.Append(node.ToDynamicLinqString(sqlContext)); + } + + public override void Visit(TermRangeNode node, IQueryVisitorContext context) + { + if (context is not ISqlQueryVisitorContext sqlContext) + throw new InvalidOperationException("The context must be an ISqlQueryVisitorContext."); + + _builder.Append(node.ToDynamicLinqString(sqlContext)); + } + + public override void Visit(ExistsNode node, IQueryVisitorContext context) + { + if (context is not ISqlQueryVisitorContext sqlContext) + throw new InvalidOperationException("The context must be an ISqlQueryVisitorContext."); + + _builder.Append(node.ToDynamicLinqString(sqlContext)); + } + + public override void Visit(MissingNode node, IQueryVisitorContext context) + { + if (context is not ISqlQueryVisitorContext sqlContext) + throw new InvalidOperationException("The context must be an ISqlQueryVisitorContext."); + + _builder.Append(node.ToDynamicLinqString(sqlContext)); + } + + public override async Task AcceptAsync(IQueryNode node, IQueryVisitorContext context) + { + await node.AcceptAsync(this, context).ConfigureAwait(false); + return _builder.ToString(); + } + + public static Task RunAsync(IQueryNode node, IQueryVisitorContext context = null) + { + return new GenerateSqlVisitor().AcceptAsync(node, context); + } + + public static string Run(IQueryNode node, IQueryVisitorContext context = null) + { + return RunAsync(node, context).GetAwaiter().GetResult(); + } +} diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs new file mode 100644 index 00000000..1372797c --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs @@ -0,0 +1,8 @@ +using System.Collections.Generic; +using Foundatio.Parsers.LuceneQueries.Visitors; + +namespace Foundatio.Parsers.SqlQueries.Visitors; + +public interface ISqlQueryVisitorContext : IQueryVisitorContext { + List Fields { get; set; } +} diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs new file mode 100644 index 00000000..469af6d8 --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System.Diagnostics; +using Foundatio.Parsers.LuceneQueries.Visitors; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace Foundatio.Parsers.SqlQueries.Visitors; + +public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorContext { + public List Fields { get; set; } + public IEntityType EntityType { get; set; } +} + +[DebuggerDisplay("{Field} IsNumber: {IsNumber} IsMoney: {IsMoney} IsDate: {IsDate} IsBoolean: {IsBoolean} IsCollection: {IsCollection}")] +public class EntityFieldInfo +{ + public string Field { get; set; } + public bool IsNumber { get; set; } + public bool IsMoney { get; set; } + public bool IsDate { get; set; } + public bool IsBoolean { get; set; } + public bool IsCollection { get; set; } + public IDictionary Data { get; set; } = new Dictionary(); +} diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 1afd7eb9..b5ddbfca 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -1,17 +1,17 @@ - net8.0 + net8.0; False $(NoWarn);CS1591;NU1701 - - - - + + + + - + diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs index 5a8f7144..8501aad3 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/AggregationParserTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -6,7 +6,6 @@ using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Extensions; -using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; using Microsoft.Extensions.Logging; using Nest; @@ -23,13 +22,13 @@ public AggregationParserTests(ITestOutputHelper output, ElasticsearchFixture fix public async Task ProcessSingleAggregationAsync() { var index = CreateRandomIndex(d => d.Dynamic().Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)))); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, - new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, - new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, - new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, - new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } - }, index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, + new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, + new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, + new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, + new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } + }, index); await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseGeo(l => "51.5032520,-0.1278990")); @@ -54,13 +53,13 @@ public async Task ProcessSingleAggregationAsync() public async Task ProcessSingleAggregationWithAliasAsync() { var index = CreateRandomIndex(d => d.Dynamic().Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)))); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, - new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, - new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, - new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, - new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } - }, index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, + new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, + new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, + new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, + new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } + }, index); await Client.Indices.RefreshAsync(index); var fieldMap = new FieldMap { { "heynow", "field4" } }; @@ -90,13 +89,13 @@ public async Task ProcessAnalyzedAggregationWithAliasAsync() .Fields(k => k.Keyword(m => m.Name("keyword")))) .FieldAlias(f => f.Name("heynow").Path(k => k.Field1)) .GeoPoint(g => g.Name(f => f.Field3)))); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, - new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, - new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, - new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, - new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } - }, index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, + new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, + new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, + new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, + new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } + }, index); await Client.Indices.RefreshAsync(index); var fieldMap = new FieldMap { { "heynow2", "field1" } }; @@ -122,13 +121,13 @@ public async Task ProcessAnalyzedAggregationWithAliasAsync() public async Task ProcessAggregationsAsync() { var index = CreateRandomIndex(d => d.Dynamic().Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)))); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, - new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, - new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, - new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, - new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } - }, index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" }, + new MyType { Field1 = "value2", Field4 = 2, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(4)) }, + new MyType { Field1 = "value3", Field4 = 3, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(3)) }, + new MyType { Field1 = "value4", Field4 = 4, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(2)) }, + new MyType { Field1 = "value5", Field4 = 5, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)) } + }, index); await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index).UseGeo(l => "51.5032520,-0.1278990")); @@ -170,7 +169,7 @@ public async Task ProcessNestedAggregationsWithAliasesAsync() .Text(f3 => f3.Name("identity") .Fields(f => f.Keyword(k => k.Name("keyword").IgnoreAbove(256)))))))))); - Client.IndexMany(new[] { new MyType { Field1 = "value1" } }, index); + await Client.IndexManyAsync(new[] { new MyType { Field1 = "value1" } }, index); await Client.Indices.RefreshAsync(index); var aliasMap = new FieldMap { { "user", "data.@user.identity" }, { "alias1", "field1" } }; @@ -198,9 +197,9 @@ public async Task ProcessSingleAggregationWithAlias() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field2 = "field2" } - }, index); + await Client.IndexManyAsync(new[] { + new MyType { Field2 = "field2" } + }, index); await Client.Indices.RefreshAsync(index); var aliasMap = new FieldMap { { "alias2", "field2" } }; @@ -232,9 +231,9 @@ public async Task ProcessAggregationsWithAliasesAsync() .Object(o2 => o2.Name("@user").Properties(p2 => p2 .Text(f3 => f3.Name("identity").Fields(f => f.Keyword(k => k.Name("keyword").IgnoreAbove(256)))))))))); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" } - }, index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990", Field5 = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5)), Field2 = "field2" } + }, index); await Client.Indices.RefreshAsync(index); var aliasMap = new FieldMap { { "user", "data.@user.identity" }, { "alias1", "field1" }, { "alias2", "field2" }, { "alias3", "field3" }, { "alias4", "field4" }, { "alias5", "field5" } }; @@ -271,8 +270,8 @@ public async Task ProcessAggregationsWithAliasesAsync() public async Task ProcessTermAggregations() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { new MyType { Field1 = "value1" } }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { new MyType { Field1 = "value1" } }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("terms:(field1 @exclude:myexclude @include:myinclude @include:otherinclude @missing:mymissing @exclude:otherexclude @min:1)"); @@ -300,8 +299,8 @@ public async Task ProcessTermAggregations() public async Task ProcessHistogramIntervalAggregations() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { new MyType { Field1 = "value1" } }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { new MyType { Field1 = "value1" } }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("histogram:(field1~0.1)"); @@ -327,8 +326,8 @@ public async Task ProcessHistogramIntervalAggregations() public async Task ProcessTermTopHitsAggregations() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { new MyType { Field1 = "value1" } }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { new MyType { Field1 = "value1" } }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("terms:(field1~1000^2 tophits:(_~1000 @include:myinclude))"); @@ -355,11 +354,11 @@ public async Task ProcessTermTopHitsAggregations() public async Task ProcessSortedTermAggregations() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { new MyType { Field1 = "value1" } }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { new MyType { Field1 = "value1" } }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); - var aggregations =await processor.BuildAggregationsAsync("terms:(field1 -cardinality:field4)"); + var aggregations = await processor.BuildAggregationsAsync("terms:(field1 -cardinality:field4)"); var actualResponse = Client.Search(d => d.Index(index).Aggregations(aggregations)); string actualRequest = actualResponse.GetRequest(); @@ -419,8 +418,8 @@ public async Task ProcessDateHistogramAggregations() public async Task CanSpecifyDefaultValuesAggregations() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { new MyType { Field1 = "test" }, new MyType { Field4 = 1 } }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { new MyType { Field1 = "test" }, new MyType { Field4 = 1 } }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var aggregations = await processor.BuildAggregationsAsync("min:field4~0 max:field4~0 avg:field4~0 sum:field4~0 cardinality:field4~0"); diff --git a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs index cf0f0d5b..906883c5 100644 --- a/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs +++ b/tests/Foundatio.Parsers.ElasticQueries.Tests/ElasticQueryParserTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -77,12 +77,12 @@ public void CanUseElasticQueryParserWithVisitor() public async Task SimpleFilterProcessor() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { + await Client.IndexManyAsync(new[] { new MyType { Field1 = "value1", Field2 = "value2" }, new MyType { Field1 = "value2", Field2 = "value2" }, new MyType { Field1 = "value1", Field2 = "value4" } }, index); - Client.Indices.Refresh(index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1"); @@ -102,12 +102,12 @@ public async Task SimpleFilterProcessor() public async Task IncludeProcessor() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2" }, - new MyType { Field1 = "value2", Field2 = "value2" }, - new MyType { Field1 = "value1", Field2 = "value4" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2" }, + new MyType { Field1 = "value2", Field2 = "value2" }, + new MyType { Field1 = "value1", Field2 = "value4" } + }, index); + await Client.Indices.RefreshAsync(index); var includes = new Dictionary { {"stuff", "field2:value2"} @@ -134,8 +134,8 @@ public async Task IncludeProcessor() public async Task ShouldGenerateORedTermsQuery() { var index = CreateRandomIndex(); - Client.Index(new MyType { Field1 = "value1", Field2 = "value2", Field3 = "value3" }, i => i.Index(index)); - Client.Indices.Refresh(index); + await Client.IndexAsync(new MyType { Field1 = "value1", Field2 = "value2", Field3 = "value3" }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 field2:value2 field3:value3", @@ -162,8 +162,8 @@ public async Task ShouldHandleMultipleTermsForAnalyzedFields() .Text(e => e.Name(m => m.Field1).Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) .Keyword(e => e.Name(m => m.Field2)) )); - Client.Index(new MyType { Field1 = "value1", Field2 = "value2", Field3 = "value3" }, i => i.Index(index)); - Client.Indices.Refresh(index); + await Client.IndexAsync(new MyType { Field1 = "value1", Field2 = "value2", Field3 = "value3" }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetDefaultFields(new[] { "field1" }).UseMappings(Client, index)); @@ -258,12 +258,12 @@ TypeMappingDescriptor GetCodeMappings(TypeMappingDescriptor d) = public async Task EscapeFilterProcessor() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "\"now there\"", Field2 = "value2" }, - new MyType { Field1 = "value2", Field2 = "value2" }, - new MyType { Field1 = "value1", Field2 = "value4" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "\"now there\"", Field2 = "value2" }, + new MyType { Field1 = "value2", Field2 = "value2" }, + new MyType { Field1 = "value1", Field2 = "value4" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("\"\\\"now there\""); @@ -288,10 +288,10 @@ public async Task EscapeFilterProcessor() public async Task CanHandleEscapedQueryWithWildcards() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "one/two/three" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "one/two/three" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync(@"field1:one\\/two*"); @@ -316,10 +316,10 @@ public async Task CanHandleEscapedQueryWithWildcards() public async Task CanHandleEscapedQuery() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "one/two/three" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "one/two/three" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync(@"field1:one\\/two"); @@ -371,12 +371,12 @@ await Client.IndexManyAsync(new[] { public async Task MissingFilterProcessor() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2" }, - new MyType { Field1 = "value2", Field2 = "value2" }, - new MyType { Field2 = "value4" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2" }, + new MyType { Field1 = "value2", Field2 = "value2" }, + new MyType { Field2 = "value4" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync($"_missing_:{nameof(MyType.Field2)}", @@ -399,12 +399,12 @@ public async Task MissingFilterProcessor() public async Task MinMaxWithDateHistogramAggregation() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2", Field5 = DateTime.Now }, - new MyType { Field1 = "value2", Field2 = "value2", Field5 = DateTime.Now }, - new MyType { Field2 = "value4", Field5 = DateTime.Now } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2", Field5 = DateTime.Now }, + new MyType { Field1 = "value2", Field2 = "value2", Field5 = DateTime.Now }, + new MyType { Field2 = "value4", Field5 = DateTime.Now } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildAggregationsAsync("min:field2 max:field2 date:(field5~1d^\"America/Chicago\" min:field2 max:field2 min:field1 @offset:-6h)"); @@ -523,12 +523,12 @@ public void CanDoNestDateHistogram() public async Task DateAggregation() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2", Field5 = DateTime.Now }, - new MyType { Field1 = "value2", Field2 = "value2", Field5 = DateTime.Now }, - new MyType { Field2 = "value4", Field5 = DateTime.Now } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2", Field5 = DateTime.Now }, + new MyType { Field1 = "value2", Field2 = "value2", Field5 = DateTime.Now }, + new MyType { Field2 = "value4", Field5 = DateTime.Now } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildAggregationsAsync("date:field5"); @@ -553,12 +553,12 @@ public async Task SimpleQueryProcessor() .Properties(p => p .Text(e => e.Name(n => n.Field3).Fields(f => f.Keyword(k => k.Name("keyword").IgnoreAbove(256)))))); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2" }, - new MyType { Field1 = "value2", Field2 = "value2" }, - new MyType { Field1 = "value1", Field2 = "value4", Field3 = "hey now" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2" }, + new MyType { Field1 = "value2", Field2 = "value2" }, + new MyType { Field1 = "value1", Field2 = "value4", Field3 = "hey now" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:value1", new ElasticQueryVisitorContext().UseSearchMode()); @@ -622,12 +622,12 @@ public async Task SimpleQueryProcessor() public async Task NegativeQueryProcessor() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2" }, - new MyType { Field1 = "value2", Field2 = "value3" }, - new MyType { Field1 = "value1", Field2 = "value4" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2" }, + new MyType { Field1 = "value2", Field2 = "value3" }, + new MyType { Field1 = "value1", Field2 = "value4" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 AND -field2:value2", @@ -694,12 +694,12 @@ public async Task NegativeQueryProcessor() public async Task NestedQueryProcessor() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2" }, - new MyType { Field1 = "value2", Field2 = "value2" }, - new MyType { Field1 = "value1", Field2 = "value4" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2" }, + new MyType { Field1 = "value2", Field2 = "value2" }, + new MyType { Field1 = "value1", Field2 = "value4" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)", @@ -723,12 +723,12 @@ public async Task NestedQueryProcessor() public async Task NestedQuery() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2" }, - new MyType { Field1 = "value2", Field2 = "value2" }, - new MyType { Field1 = "value1", Field2 = "value4" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2" }, + new MyType { Field1 = "value2", Field2 = "value2" }, + new MyType { Field1 = "value1", Field2 = "value4" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)"); @@ -753,7 +753,7 @@ public async Task NestedQuery() public async Task MixedCaseTermFilterQueryProcessor() { var index = CreateRandomIndex(); - Client.Index(new MyType { Field1 = "Testing.Casing" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "Testing.Casing" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:Testing.Casing", new ElasticQueryVisitorContext { UseScoring = true }); @@ -773,7 +773,7 @@ public async Task MixedCaseTermFilterQueryProcessor() public async Task MultipleWordsTermFilterQueryProcessor() { var index = CreateRandomIndex(); - Client.Index(new MyType { Field1 = "Blake Niemyjski" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "Blake Niemyjski" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:\"Blake Niemyjski\"", new ElasticQueryVisitorContext { UseScoring = true }); @@ -793,7 +793,7 @@ public async Task MultipleWordsTermFilterQueryProcessor() public async Task CanTranslateTermQueryProcessor() { var index = CreateRandomIndex(); - Client.Index(new MyType { Field1 = "Testing.Casing" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "Testing.Casing" }, i => i.Index(index)); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).AddVisitor(new UpdateFixedTermFieldToDateFixedExistsQueryVisitor())); var result = await processor.BuildQueryAsync("fixed:true"); @@ -814,12 +814,12 @@ public async Task CanTranslateTermQueryProcessor() public async Task GroupedOrFilterProcessor() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyType { Field1 = "value1", Field2 = "value2" }, - new MyType { Field1 = "value2", Field2 = "value2" }, - new MyType { Field1 = "value1", Field2 = "value4" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyType { Field1 = "value1", Field2 = "value2" }, + new MyType { Field1 = "value2", Field2 = "value2" }, + new MyType { Field1 = "value1", Field2 = "value4" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = await processor.BuildQueryAsync("field1:value1 (field2:value2 OR field3:value3)", @@ -855,15 +855,15 @@ public async Task NestedFilterProcessor() )) )); await Client.IndexManyAsync(new[] { - new MyNestedType { - Field1 = "value1", - Field2 = "value2", - Nested = {new MyType {Field1 = "value1", Field4 = 4}} - }, - new MyNestedType {Field1 = "value2", Field2 = "value2"}, - new MyNestedType {Field1 = "value1", Field2 = "value4"} - }); - Client.Indices.Refresh(index); + new MyNestedType { + Field1 = "value1", + Field2 = "value2", + Nested = {new MyType {Field1 = "value1", Field4 = 4}} + }, + new MyNestedType {Field1 = "value2", Field2 = "value2"}, + new MyNestedType {Field1 = "value1", Field2 = "value4"} + }); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseFieldMap(new FieldMap { { "blah", "nested" } }).UseMappings(Client).UseNested()); var result = await processor.BuildQueryAsync("field1:value1 blah:(blah.field1:value1)", new ElasticQueryVisitorContext().UseScoring()); @@ -926,16 +926,16 @@ public async Task NestedFilterProcessor2() )) )); - Client.IndexMany(new[] { - new MyNestedType { - Field1 = "value1", - Field2 = "value2", - Nested = {new MyType {Field1 = "value1", Field4 = 4}} - }, - new MyNestedType {Field1 = "value2", Field2 = "value2"}, - new MyNestedType {Field1 = "value1", Field2 = "value4", Field3 = "value3"} - }); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyNestedType { + Field1 = "value1", + Field2 = "value2", + Nested = {new MyType {Field1 = "value1", Field4 = 4}} + }, + new MyNestedType {Field1 = "value2", Field2 = "value2"}, + new MyNestedType {Field1 = "value1", Field2 = "value4", Field3 = "value3"} + }); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client).UseNested()); var result = await processor.BuildQueryAsync("field1:value1 nested:(nested.field1:value1 nested.field4:4 nested.field3:value3)", @@ -1008,8 +1008,8 @@ public async Task CanBuildAliasQueryProcessor() public async Task NonAnalyzedPrefixQuery() { var index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Name(m => m.Field1)))); - Client.Index(new MyType { Field1 = "value123" }, i => i.Index(index)); - Client.Indices.Refresh(index); + await Client.IndexAsync(new MyType { Field1 = "value123" }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.UseMappings(Client, index)); var result = await processor.BuildQueryAsync("field1:value*", new ElasticQueryVisitorContext().UseSearchMode()); @@ -1033,10 +1033,10 @@ public async Task NonAnalyzedPrefixQuery() public async Task RangeQueryProcessor() { var index = CreateRandomIndex(); - var res = Client.Index(new MyType { Field1 = "value1", Field4 = 1 }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); - Client.Indices.Refresh(index); + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log)); var result = @@ -1066,10 +1066,10 @@ await processor.BuildQueryAsync("field4:[1 TO 2} OR field1:value1", public async Task DateRangeWithWildcardMinQueryProcessor() { var index = CreateRandomIndex(); - var res = Client.Index(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Indices.Refresh(index); + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var ctx = new ElasticQueryVisitorContext { UseScoring = true, DefaultTimeZone = () => Task.FromResult("America/Chicago") }; @@ -1098,10 +1098,10 @@ public async Task DateRangeWithWildcardMinQueryProcessor() public async Task DateRangeWithDateMathQueryProcessor() { var index = CreateRandomIndex(); - var res = Client.Index(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Indices.Refresh(index); + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var ctx = new ElasticQueryVisitorContext { UseScoring = true, DefaultTimeZone = () => Task.FromResult("America/Chicago") }; @@ -1128,10 +1128,10 @@ public async Task DateRangeWithDateMathQueryProcessor() public async Task DateRangeWithWildcardMaxQueryProcessor() { var index = CreateRandomIndex(); - var res = Client.Index(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Indices.Refresh(index); + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var ctx = new ElasticQueryVisitorContext { UseScoring = true, DefaultTimeZone = () => Task.FromResult("America/Chicago") }; @@ -1159,10 +1159,10 @@ public async Task DateRangeWithWildcardMaxQueryProcessor() public async Task DateRangeWithTimeZone() { var index = CreateRandomIndex(); - var res = Client.Index(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Indices.Refresh(index); + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var ctx = new ElasticQueryVisitorContext { UseScoring = true }; @@ -1190,10 +1190,10 @@ public async Task DateRangeWithTimeZone() public async Task DateRangeQueryProcessor() { var index = CreateRandomIndex(); - var res = Client.Index(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); - Client.Indices.Refresh(index); + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3, Field5 = DateTime.UtcNow }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var ctx = new ElasticQueryVisitorContext { UseScoring = true, DefaultTimeZone = () => Task.FromResult("America/Chicago") }; @@ -1222,11 +1222,11 @@ public async Task DateRangeQueryProcessor() public async Task SimpleGeoRangeQuery() { var index = CreateRandomIndex(m => m.Properties(p => p.GeoPoint(g => g.Name(f => f.Field3)))); - var res = Client.Index(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); - Client.Indices.Refresh(index); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c .UseMappings(Client, index) @@ -1253,8 +1253,8 @@ await processor.BuildQueryAsync("field3:[51.5032520,-0.1278990 TO 51.5032520,-0. public async Task CanUseValidationToGetUnresolvedFields() { var index = CreateRandomIndex(d => d.Properties(p => p.Keyword(e => e.Name(m => m.Field1)))); - Client.Index(new MyType { Field1 = "value123" }, i => i.Index(index)); - Client.Indices.Refresh(index); + await Client.IndexAsync(new MyType { Field1 = "value123" }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var context = new ElasticQueryVisitorContext(); var parser = new ElasticQueryParser(c => c.UseMappings(Client, index).SetValidationOptions(new QueryValidationOptions { AllowUnresolvedFields = false })); @@ -1322,11 +1322,11 @@ public async Task CanParseSort() .Text(e => e.Name(m => m.Field1).Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) .Text(e => e.Name(m => m.Field2).Fields(f2 => f2.Keyword(e1 => e1.Name("keyword")).Keyword(e2 => e2.Name("sort")))) )); - var res = Client.Index(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, + var res = await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); - Client.Indices.Refresh(index); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var aliasMap = new FieldMap { { "geo", "field3" } }; var processor = new ElasticQueryParser(c => c @@ -1359,18 +1359,18 @@ public async Task CanHandleSpacedFields() { var index = CreateRandomIndex(); - Client.IndexMany(new[] { - new MyNestedType { Field1 = "value1", Field2 = "value2", Nested = new MyType[] { - new MyType { Field1 = "banana", Data = { - { "number-0001", 23 }, - { "text-0001", "Hey" }, - { "spaced field", "hey" } - }} - }}, - new MyNestedType { Field1 = "value2", Field2 = "value2" }, - new MyNestedType { Field1 = "value1", Field2 = "value4" } - }, index); - Client.Indices.Refresh(index); + await Client.IndexManyAsync(new[] { + new MyNestedType { Field1 = "value1", Field2 = "value2", Nested = new MyType[] { + new MyType { Field1 = "banana", Data = { + { "number-0001", 23 }, + { "text-0001", "Hey" }, + { "spaced field", "hey" } + }} + }}, + new MyNestedType { Field1 = "value2", Field2 = "value2" }, + new MyNestedType { Field1 = "value1", Field2 = "value4" } + }, index); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var sort = await processor.BuildSortAsync("nested.data.spaced\\ field"); @@ -1399,8 +1399,8 @@ public async Task CanParseMixedCaseSort() .Text(e => e.Name(m => m.MultiWord).Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))) )); - var res = Client.Index(new MyType { MultiWord = "value1" }, i => i.Index(index)); - Client.Indices.Refresh(index); + var res = await Client.IndexAsync(new MyType { MultiWord = "value1" }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var processor = new ElasticQueryParser(c => c.SetLoggerFactory(Log).UseMappings(Client, index)); var sort = await processor.BuildSortAsync("multiWord -multiword"); var actualResponse = Client.Search(d => d.Index(index).Sort(sort)); @@ -1424,10 +1424,10 @@ public async Task GeoRangeQueryProcessor() .GeoPoint(g => g.Name(f => f.Field3)) .Text(e => e.Name(m => m.Field1).Fields(f1 => f1.Keyword(e1 => e1.Name("keyword")))))); - Client.Index(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); - Client.Index(new MyType { Field4 = 2 }, i => i.Index(index)); - Client.Index(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); - Client.Indices.Refresh(index); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 1, Field3 = "51.5032520,-0.1278990" }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field4 = 2 }, i => i.Index(index)); + await Client.IndexAsync(new MyType { Field1 = "value1", Field4 = 3 }, i => i.Index(index)); + await Client.Indices.RefreshAsync(index); var aliasMap = new FieldMap { { "geo", "field3" } }; var processor = new ElasticQueryParser(c => c diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/DynamicFieldVisitor.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/DynamicFieldVisitor.cs new file mode 100644 index 00000000..7e62071d --- /dev/null +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/DynamicFieldVisitor.cs @@ -0,0 +1,65 @@ +using System.Text; +using Foundatio.Parsers.LuceneQueries.Nodes; +using Foundatio.Parsers.LuceneQueries.Visitors; +using Foundatio.Parsers.SqlQueries.Extensions; +using Foundatio.Parsers.SqlQueries.Visitors; + +namespace Foundatio.Parsers.SqlQueries.Tests; + +public class DynamicFieldVisitor : ChainableMutatingQueryVisitor +{ + public override IQueryNode Visit(TermNode node, IQueryVisitorContext context) + { + if (context is not SqlQueryVisitorContext sqlContext) + return node; + + var field = SqlNodeExtensions.GetFieldInfo(sqlContext.Fields, node.Field); + + if (field == null || !field.Data.TryGetValue("DataDefinitionId", out object value) || + value is not int dataDefinitionId) + { + return node; + } + + var customFieldBuilder = new StringBuilder(); + + customFieldBuilder.Append("DataValues.Any(DataDefinitionId = "); + customFieldBuilder.Append(dataDefinitionId); + customFieldBuilder.Append(" AND "); + switch (field) + { + case { IsMoney: true }: + customFieldBuilder.Append("MoneyValue"); + break; + case { IsNumber: true }: + customFieldBuilder.Append("NumberValue"); + break; + case { IsBoolean: true }: + customFieldBuilder.Append("BooleanValue"); + break; + case { IsDate: true }: + customFieldBuilder.Append("DateValue"); + break; + default: + customFieldBuilder.Append("StringValue"); + break; + } + + customFieldBuilder.Append(" = "); + if (field is { IsNumber: true } or { IsBoolean: true }) + { + customFieldBuilder.Append(node.Term); + } + else + { + customFieldBuilder.Append("\""); + customFieldBuilder.Append(node.Term); + customFieldBuilder.Append("\""); + } + customFieldBuilder.Append(")"); + + node.SetQuery(customFieldBuilder.ToString()); + + return node; + } +} diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj b/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj new file mode 100644 index 00000000..56be4a1e --- /dev/null +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs new file mode 100644 index 00000000..47f0c57d --- /dev/null +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; + +namespace Foundatio.Parsers.SqlQueries.Tests; + +public class SampleContext : DbContext { + public SampleContext(DbContextOptions options) : base(options) { } + public DbSet Employees => Set(); + public DbSet Companies => Set(); + public DbSet DataDefinitions => Set(); + public DbSet DataValues => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + // Employee + modelBuilder.Entity().HasIndex(e => new { e.FullName, e.Title }); + + // Company + modelBuilder.Entity().HasIndex(e => new { e.Name }); + + // DataDefinition + modelBuilder.Entity().Property(c => c.DataType).IsRequired(); + modelBuilder.Entity().HasIndex(c => new { c.CompanyId, c.Key }).IsUnique(); + + // DataValue + modelBuilder.Entity().HasIndex(c => new { c.DataDefinitionId, c.CompanyId, c.EmployeeId }).HasFilter(null).IsUnique(); + modelBuilder.Entity().Property(e => e.StringValue).HasMaxLength(4000).IsSparse(); + modelBuilder.Entity().Property(e => e.DateValue).IsSparse(); + modelBuilder.Entity().Property(e => e.MoneyValue).IsSparse().HasColumnType("money").HasPrecision(2); + modelBuilder.Entity().Property(e => e.BooleanValue).IsSparse(); + modelBuilder.Entity().Property(e => e.NumberValue).HasColumnType("decimal").HasPrecision(15,3).IsSparse(); + modelBuilder.Entity().HasIndex(e => new { e.StringValue, e.DateValue, e.MoneyValue, e.BooleanValue, e.NumberValue }); + } +} + +public class Employee { + public int Id { get; set; } + public string FullName { get; set; } + public string Title { get; set; } + public int Salary { get; set; } + public List Companies { get; set; } + public List DataValues { get; set; } + public DateTime Created { get; set; } = DateTime.Now; +} + +public class Company { + public int Id { get; set; } + public string Name { get; set; } + public string Description { get; set; } + public List Employees { get; set; } + public List DataDefinitions { get; set; } +} + +public class DataValue +{ + public int Id { get; set; } + public int DataDefinitionId { get; set; } + public int CompanyId { get; set; } + public int EmployeeId { get; set; } + + // store the values separately as sparse columns for querying purposes + public string StringValue { get; set; } + public DateTime? DateValue { get; set; } + public decimal? MoneyValue { get; set; } + public bool? BooleanValue { get; set; } + public decimal? NumberValue { get; set; } + + public DataDefinition Definition { get; set; } = null; + + public object GetValue(DataType? dataType = null) + { + if (!dataType.HasValue && Definition != null) + dataType = Definition.DataType; + + if (dataType.HasValue) + { + return dataType switch + { + DataType.String => StringValue, + DataType.Date => DateValue, + DataType.Number => NumberValue, + DataType.Boolean => BooleanValue, + DataType.Money => MoneyValue, + DataType.Percent => NumberValue, + _ => null + }; + } + + if (MoneyValue.HasValue) + return MoneyValue.Value; + if (BooleanValue.HasValue) + return BooleanValue.Value; + if (NumberValue.HasValue) + return NumberValue.Value; + if (DateValue.HasValue) + return DateValue.Value; + + return StringValue ?? null; + } + + public void ClearValue() + { + StringValue = null; + DateValue = null; + NumberValue = null; + BooleanValue = null; + MoneyValue = null; + } + + public void SetValue(object value, DataType? dataType = null) + { + ClearValue(); + + if (value == null) + return; + + switch (dataType ?? Definition!.DataType) + { + case DataType.String: + StringValue = value.ToString(); + break; + case DataType.Date: + if (DateTime.TryParse(value.ToString(), out DateTime dateResult)) + DateValue = dateResult; + break; + case DataType.Number: + case DataType.Percent: + if (Decimal.TryParse(value.ToString(), out decimal numberResult)) + NumberValue = numberResult; + break; + case DataType.Boolean: + if (Boolean.TryParse(value.ToString(), out bool boolResult)) + BooleanValue = boolResult; + break; + case DataType.Money: + if (Decimal.TryParse(value.ToString(), out decimal decimalResult)) + MoneyValue = decimalResult; + break; + } + } + + // relationships + [DeleteBehavior(DeleteBehavior.NoAction)] + public Employee Employee { get; set; } = null; +} + +public class DataDefinition +{ + public int Id { get; set; } + public int CompanyId { get; set; } + + public DataType DataType { get; set; } + public string Key { get; set; } = String.Empty; + + // relationships + [DeleteBehavior(DeleteBehavior.Cascade)] + public Company Company { get; set; } = null; +} + +public enum DataType +{ + String, + Number, + Boolean, + Date, + Money, + Percent +} diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs new file mode 100644 index 00000000..f188bd7e --- /dev/null +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -0,0 +1,332 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Linq.Dynamic.Core; +using System.Threading.Tasks; +using Foundatio.Parsers.LuceneQueries.Nodes; +using Foundatio.Parsers.LuceneQueries.Visitors; +using Foundatio.Parsers.SqlQueries.Visitors; +using Foundatio.Xunit; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Pegasus.Common.Tracing; +using Xunit; +using Xunit.Abstractions; + +namespace Foundatio.Parsers.SqlQueries.Tests; + +public class SqlQueryParserTests : TestWithLoggingBase { + public SqlQueryParserTests(ITestOutputHelper output) : base(output) { + Log.DefaultMinimumLevel = LogLevel.Trace; + } + + [Theory] + [InlineData("field:[1 TO 5]", "(field >= 1 AND field <= 5)")] + [InlineData("field:{1 TO 5}", "(field > 1 AND field < 5)")] + [InlineData("field:[1 TO 5}", "(field >= 1 AND field < 5)")] + [InlineData("field:>5", "field > 5")] + [InlineData("field:>=5", "field >= 5")] + [InlineData("field:<5", "field < 5")] + [InlineData("field:<=5", "field <= 5")] + // [InlineData("date:>now")] + // [InlineData("date:now")] + // [InlineData("data.date:[now/d-4d TO now/d+1d}")] + // [InlineData("data.date:[2012-01-01 TO 2012-12-31]")] + // [InlineData("data.date:[* TO 2012-12-31]")] + // [InlineData("data.date:[2012-01-01 TO *]")] + // [InlineData("(data.date:[now/d-4d TO now/d+1d})")] + // [InlineData("count:>1")] + // [InlineData("count:>=1")] + // [InlineData("count:[1..5}")] + // [InlineData(@"count:a\:a")] + // [InlineData(@"count:a\:a more:stuff")] + // [InlineData("data.count:[1..5}")] + // [InlineData("age:[1 TO 2]")] + // [InlineData("data.Windows-identity:ejsmith")] + // [InlineData("data.age:[* TO 10]")] + // [InlineData("hidden:true")] + public Task ValidQueries(string query, string expected) + { + return ParseAndValidateQuery(query, expected, true); + } + + [Fact] + public async Task CanSearchDefaultFields() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + parser.Configuration.SetDefaultFields(["FullName", "Title"]); + + var context = parser.GetContext(db.Employees.EntityType); + + string sqlExpected = db.Employees.Where(e => e.FullName.Contains("John") || e.Title.Contains("John")).ToQueryString(); + string sqlActual = db.Employees.Where("""FullName.Contains("John") || Title.Contains("John")""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("John", context); + sqlActual = db.Employees.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + } + + [Fact] + public async Task CanUseDateFilter() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + + var context = parser.GetContext(db.Employees.EntityType); + + string sqlExpected = db.Employees.Where(e => e.Created > new DateTime(2024, 1, 1)).ToQueryString(); + string sqlActual = db.Employees.Where("""created > DateTime.Parse("2024-01-01")""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("created:>2024-01-01", context); + sqlActual = db.Employees.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + } + + [Fact] + public async Task CanUseExistsFilter() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + + var context = parser.GetContext(db.Employees.EntityType); + + string sqlExpected = db.Employees.Where(e => e.Title != null).ToQueryString(); + string sqlActual = db.Employees.Where("""Title != null""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("_exists_:title", context); + sqlActual = db.Employees.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + } + + [Fact] + public async Task CanUseMissingFilter() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + + var context = parser.GetContext(db.Employees.EntityType); + + string sqlExpected = db.Employees.Where(e => e.Title == null).ToQueryString(); + string sqlActual = db.Employees.Where("""Title == null""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("_missing_:title", context); + sqlActual = db.Employees.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + } + + [Fact] + public async Task CanUseDateMathFilter() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + + var context = parser.GetContext(db.Employees.EntityType); + + string sqlExpected = db.Employees.Where(e => e.Created > DateTime.UtcNow.AddDays(-90)).ToQueryString(); + string sqlActual = db.Employees.Where("""created > DateTime.UtcNow.AddDays(-90)""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("created:>now-90d", context); + sqlActual = db.Employees.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + } + + [Fact] + public async Task CanUseCollectionDefaultFields() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + parser.Configuration.SetDefaultFields(["Companies.Name"]); + + var context = parser.GetContext(db.Employees.EntityType); + + string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name.Contains("acme"))).ToQueryString(); + string sqlActual = db.Employees.Where("""Companies.Any(Name.Contains("acme"))""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("acme", context); + sqlActual = db.Employees.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + } + + [Fact] + public async Task CanUseNavigationFields() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + + var context = parser.GetContext(db.Companies.EntityType); + + Assert.Contains(db.Companies.EntityType.GetNavigations(), e => e.TargetEntityType == db.DataDefinitions.EntityType); + + string sqlExpected = db.Companies.Where(e => e.DataDefinitions.Any(c => c.Key == "age")).ToQueryString(); + string sqlActual = db.Companies.Where("""DataDefinitions.Any(Key.Equals("age"))""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("datadefinitions.key:age", context); + sqlActual = db.Companies.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + + var query = db.Companies.AsQueryable(); + var companies = await query.Where(sql).ToListAsync(); + + Assert.Single(companies); + } + + [Fact] + public async Task CanUseSkipNavigationFields() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + + var context = parser.GetContext(db.Companies.EntityType); + + Assert.Contains(db.Companies.EntityType.GetSkipNavigations(), e => e.TargetEntityType == db.Employees.EntityType); + + string sqlExpected = db.Companies.Where(e => e.Employees.Any(c => c.Salary.Equals(80_000))).ToQueryString(); + string sqlActual = db.Companies.Where("""Employees.Any(Salary.Equals(80000))""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("employees.salary:80000", context); + sqlActual = db.Companies.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + + var query = db.Companies.AsQueryable(); + var companies = await query.Where(sql).ToListAsync(); + + Assert.Single(companies); + } + + [Fact] + public async Task CanGenerateSql() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + + var context = parser.GetContext(db.Employees.EntityType); + context.Fields.Add(new EntityFieldInfo { Field = "age", IsNumber = true, Data = {{ "DataDefinitionId", 1 }}}); + context.ValidationOptions.AllowedFields.Add("age"); + + string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name == "acme") && e.DataValues.Any(dv => dv.DataDefinitionId == 1 && dv.NumberValue == 30)).ToQueryString(); + string sqlActual = db.Employees.Where("""Companies.Any(Name = "acme") AND DataValues.Any(DataDefinitionId = 1 AND NumberValue = 30) """).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("companies.name:acme age:30", context); + sqlActual = db.Employees.Where(sql).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + + var q = db.Employees.AsNoTracking(); + sql = await parser.ToDynamicLinqAsync("companies.name:acme age:30", context); + sqlActual = q.Where(sql, db.Employees).ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + + await Assert.ThrowsAsync(() => parser.ToDynamicLinqAsync("companies.description:acme", context)); + + var employees = await db.Employees.Where(e => e.Title == "software developer" && e.DataValues.Any(dv => dv.DataDefinitionId == 1 && dv.NumberValue == 30)) + .ToListAsync(); + + Assert.Single(employees); + var employee = employees.Single(); + Assert.Equal("John Doe", employee.FullName); + } + + public IServiceProvider GetServiceProvider() + { + var services = new ServiceCollection(); + services.AddDbContext((_, x) => + { + x.UseSqlServer("Server=localhost;User Id=sa;Password=P@ssword1;Timeout=5;Initial Catalog=foundatio;Encrypt=False"); + }, ServiceLifetime.Scoped, ServiceLifetime.Singleton); + var parser = new SqlQueryParser(); + parser.Configuration.UseEntityTypePropertyFilter(p => p.Name != nameof(Company.Description)); + parser.Configuration.AddQueryVisitor(new DynamicFieldVisitor()); + services.AddSingleton(parser); + return services.BuildServiceProvider(); + } + + public async Task GetSampleContextWithDataAsync(IServiceProvider sp) + { + var db = sp.GetRequiredService(); + var parser = sp.GetRequiredService(); + + var dbParser = db.GetService(); + Assert.Same(parser, dbParser); + var dbSetParser = db.Employees.GetService(); + Assert.Same(parser, dbSetParser); + + await db.Database.EnsureDeletedAsync(); + await db.Database.EnsureCreatedAsync(); + + var company = new Company { + Name = "Acme", + DataDefinitions = [ new() { Key = "age", DataType = DataType.Number } ] + }; + db.Companies.Add(company); + db.Employees.Add(new Employee + { + FullName = "John Doe", + Title = "Software Developer", + Salary = 80_000, + DataValues = [ new() { Definition = company.DataDefinitions[0], NumberValue = 30 } ], + Companies = [company] + }); + db.Employees.Add(new Employee + { + FullName = "Jane Doe", + Title = "Software Developer", + Salary = 90_000, + DataValues = [ new() { Definition = company.DataDefinitions[0], NumberValue = 23 } ], + Companies = [company] + }); + await db.SaveChangesAsync(); + + return db; + } + + private async Task ParseAndValidateQuery(string query, string expected, bool isValid) + { +#if ENABLE_TRACING + var tracer = new LoggingTracer(_logger, reportPerformance: true); +#else + var tracer = NullTracer.Instance; +#endif + var parser = new SqlQueryParser + { + Tracer = tracer + }; + + IQueryNode result; + try + { + result = await parser.ParseAsync(query); + } + catch (FormatException ex) + { + Assert.False(isValid, ex.Message); + return; + } + + string nodes = await DebugQueryVisitor.RunAsync(result); + _logger.LogInformation(nodes); + var context = new SqlQueryVisitorContext { Fields = + [ + new EntityFieldInfo { Field = "field", IsNumber = true } + ] + }; + string generatedQuery = await GenerateSqlVisitor.RunAsync(result, context); + Assert.Equal(expected, generatedQuery); + } +}