[release/9.0-staging] Cosmos Full Text Search support (query part)

src/EFCore.Analyzers/EFDiagnostics.cs

@@ -19,4 +19,5 @@ public static class EFDiagnostics
     public const string MetricsExperimental = "EF9101";
     public const string PagingExperimental = "EF9102";
     public const string CosmosVectorSearchExperimental = "EF9103";
+    public const string CosmosFullTextSearchExperimental = "EF9104";
 }

src/EFCore.Cosmos/EFCore.Cosmos.csproj

@@ -12,6 +12,7 @@
     <NoWarn>$(NoWarn);EF9101</NoWarn> <!-- Metrics is experimental -->
     <NoWarn>$(NoWarn);EF9102</NoWarn> <!-- Paging is experimental -->
     <NoWarn>$(NoWarn);EF9103</NoWarn> <!-- Vector search is experimental -->
+    <NoWarn>$(NoWarn);EF9104</NoWarn> <!-- Full-text search is experimental -->
   </PropertyGroup>
 
   <ItemGroup>

src/EFCore.Cosmos/Extensions/CosmosDbFunctionsExtensions.cs

@@ -52,6 +52,60 @@ public static T CoalesceUndefined<T>(
         T expression2)
         => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(CoalesceUndefined)));
 
+    /// <summary>
+    ///     Checks if the specified property contains the given keyword using full-text search.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="property">The property to search.</param>
+    /// <param name="keyword">The keyword to search for.</param>
+    /// <returns><see langword="true" /> if the property contains the keyword; otherwise, <see langword="false" />.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static bool FullTextContains(this DbFunctions _, string property, string keyword)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContains)));
+
+    /// <summary>
+    ///     Checks if the specified property contains all the given keywords using full-text search.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="property">The property to search.</param>
+    /// <param name="keywords">The keywords to search for.</param>
+    /// <returns><see langword="true" /> if the property contains all the keywords; otherwise, <see langword="false" />.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static bool FullTextContainsAll(this DbFunctions _, string property, params string[] keywords)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContainsAll)));
+
+    /// <summary>
+    ///     Checks if the specified property contains any of the given keywords using full-text search.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="property">The property to search.</param>
+    /// <param name="keywords">The keywords to search for.</param>
+    /// <returns><see langword="true" /> if the property contains any of the keywords; otherwise, <see langword="false" />.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static bool FullTextContainsAny(this DbFunctions _, string property, params string[] keywords)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContainsAny)));
+
+    /// <summary>
+    ///     Returns the full-text search score for the specified property and keywords.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="property">The property to search.</param>
+    /// <param name="keywords">The keywords to search for.</param>
+    /// <returns>The full-text search score.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static double FullTextScore(this DbFunctions _, string property, string[] keywords)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextScore)));
+
+    /// <summary>
+    ///     Combines scores provided by two or more specified functions.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="functions">The functions to compute the score for.</param>
+    /// <returns>The combined score.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static double Rrf(this DbFunctions _, params double[] functions)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(Rrf)));
+
     /// <summary>
     ///     Returns the distance between two vectors, using the distance function and data type defined using
     ///     <see

src/EFCore.Cosmos/Properties/CosmosStrings.resx

@@ -283,6 +283,15 @@
   <data name="OneOfTwoValuesMustBeSet" xml:space="preserve">
     <value>Exactly one of '{param1}' or '{param2}' must be set.</value>
   </data>
+  <data name="OrderByDescendingScoringFunction" xml:space="preserve">
+    <value>Ordering based on scoring function is not supported inside '{orderByDescending}'. Use '{orderBy}' instead.</value>
+  </data>
+  <data name="OrderByMultipleScoringFunctionWithoutRrf" xml:space="preserve">
+    <value>Only one ordering using scoring function is allowed. Use 'EF.Functions.{rrf}' method to combine multiple scoring functions.</value>
+  </data>
+  <data name="OrderByScoringFunctionMixedWithRegularOrderby" xml:space="preserve">
+    <value>Ordering using a scoring function is mutually exclusive with other forms of ordering.</value>
+  </data>
   <data name="OrphanedNestedDocument" xml:space="preserve">
     <value>The entity of type '{entityType}' is mapped as a part of the document mapped to '{missingEntityType}', but there is no tracked entity of this type with the corresponding key value. Consider using 'DbContextOptionsBuilder.EnableSensitiveDataLogging' to see the key values.</value>
   </data>

src/EFCore.Cosmos/Query/Internal/CosmosMethodCallTranslatorProvider.cs

@@ -36,7 +36,8 @@ public CosmosMethodCallTranslatorProvider(
             new CosmosRegexTranslator(sqlExpressionFactory),
             new CosmosStringMethodTranslator(sqlExpressionFactory),
             new CosmosTypeCheckingTranslator(sqlExpressionFactory),
-            new CosmosVectorSearchTranslator(sqlExpressionFactory, typeMappingSource)
+            new CosmosVectorSearchTranslator(sqlExpressionFactory, typeMappingSource),
+            new CosmosFullTextSearchTranslator(sqlExpressionFactory, typeMappingSource)
             //new LikeTranslator(sqlExpressionFactory),
             //new EnumHasFlagTranslator(sqlExpressionFactory),
             //new GetValueOrDefaultTranslator(sqlExpressionFactory),

src/EFCore.Cosmos/Query/Internal/CosmosQuerySqlGenerator.cs

@@ -14,6 +14,9 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal;
 /// </summary>
 public class CosmosQuerySqlGenerator(ITypeMappingSource typeMappingSource) : SqlExpressionVisitor
 {
+    private static readonly bool UseOldBehavior35476 =
+          AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue35476", out var enabled35476) && enabled35476;
+
     private readonly IndentedStringBuilder _sqlBuilder = new();
     private IReadOnlyDictionary<string, object> _parameterValues = null!;
     private List<SqlParameter> _sqlParameters = null!;
@@ -341,6 +344,15 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
         {
             _sqlBuilder.AppendLine().Append("ORDER BY ");
 
+            var orderByScoringFunction = selectExpression.Orderings is [{ Expression: SqlFunctionExpression { IsScoringFunction: true } }];
+            if (!UseOldBehavior35476 && orderByScoringFunction)
+            {
+                _sqlBuilder.Append("RANK ");
+            }
+
+            Check.DebugAssert(UseOldBehavior35476 || orderByScoringFunction || selectExpression.Orderings.All(x => x.Expression is not SqlFunctionExpression { IsScoringFunction: true }),
+                "Scoring function can only appear as first (and only) ordering, or not at all.");
+
             GenerateList(selectExpression.Orderings, e => Visit(e));
         }
 

src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.InExpressionValuesExpandingExpressionVisitor.cs

@@ -9,48 +9,106 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal;
 
 public partial class CosmosShapedQueryCompilingExpressionVisitor
 {
-    private sealed class InExpressionValuesExpandingExpressionVisitor(
+    private static readonly bool UseOldBehavior35476 =
+          AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue35476", out var enabled35476) && enabled35476;
+
+    private sealed class ParameterInliner(
         ISqlExpressionFactory sqlExpressionFactory,
         IReadOnlyDictionary<string, object> parametersValues)
         : ExpressionVisitor
     {
         protected override Expression VisitExtension(Expression expression)
         {
-            if (expression is InExpression inExpression)
+            if (!UseOldBehavior35476)
             {
-                IReadOnlyList<SqlExpression> values;
+                expression = base.VisitExtension(expression);
+            }
 
-                switch (inExpression)
+            switch (expression)
+            {
+                // Inlines array parameter of InExpression, transforming: 'item IN (@valuesArray)' to: 'item IN (value1, value2)'
+                case InExpression inExpression:
                 {
-                    case { Values: IReadOnlyList<SqlExpression> values2 }:
-                        values = values2;
-                        break;
+                    IReadOnlyList<SqlExpression> values;
 
-                    // TODO: IN with subquery (return immediately, nothing to do here)
-
-                    case { ValuesParameter: SqlParameterExpression valuesParameter }:
+                    switch (inExpression)
                     {
-                        var typeMapping = valuesParameter.TypeMapping;
-                        var mutableValues = new List<SqlExpression>();
-                        foreach (var value in (IEnumerable)parametersValues[valuesParameter.Name])
+                        case { Values: IReadOnlyList<SqlExpression> values2 }:
+                            values = values2;
+                            break;
+
+                        // TODO: IN with subquery (return immediately, nothing to do here)
+
+                        case { ValuesParameter: SqlParameterExpression valuesParameter }:
                         {
-                            mutableValues.Add(sqlExpressionFactory.Constant(value, value?.GetType() ?? typeof(object), typeMapping));
+                            var typeMapping = valuesParameter.TypeMapping;
+                            var mutableValues = new List<SqlExpression>();
+                            foreach (var value in (IEnumerable)parametersValues[valuesParameter.Name])
+                            {
+                                mutableValues.Add(sqlExpressionFactory.Constant(value, value?.GetType() ?? typeof(object), typeMapping));
+                            }
+
+                            values = mutableValues;
+                            break;
                         }
 
-                        values = mutableValues;
-                        break;
+                        default:
+                            throw new UnreachableException();
                     }
 
-                    default:
-                        throw new UnreachableException();
+                    return values.Count == 0
+                        ? sqlExpressionFactory.ApplyDefaultTypeMapping(sqlExpressionFactory.Constant(false))
+                        : sqlExpressionFactory.In((SqlExpression)Visit(inExpression.Item), values);
                 }
 
-                return values.Count == 0
-                    ? sqlExpressionFactory.ApplyDefaultTypeMapping(sqlExpressionFactory.Constant(false))
-                    : sqlExpressionFactory.In((SqlExpression)Visit(inExpression.Item), values);
-            }
+                // Converts Offset and Limit parameters to constants when ORDER BY RANK is detected in the SelectExpression (i.e. we order by scoring function)
+                // Cosmos only supports constants in Offset and Limit for this scenario currently (ORDER BY RANK limitation)
+                case SelectExpression { Orderings: [{ Expression: SqlFunctionExpression { IsScoringFunction: true } }], Limit: var limit, Offset: var offset } hybridSearch
+                    when !UseOldBehavior35476 && (limit is SqlParameterExpression || offset is SqlParameterExpression):
+                {
+                    if (hybridSearch.Limit is SqlParameterExpression limitPrm)
+                    {
+                        hybridSearch.ApplyLimit(
+                            sqlExpressionFactory.Constant(
+                                parametersValues[limitPrm.Name],
+                                limitPrm.TypeMapping));
+                    }
+
+                    if (hybridSearch.Offset is SqlParameterExpression offsetPrm)
+                    {
+                        hybridSearch.ApplyOffset(
+                            sqlExpressionFactory.Constant(
+                                parametersValues[offsetPrm.Name],
+                                offsetPrm.TypeMapping));
+                    }
+
+                    return base.VisitExtension(expression);
+                }
+
+                // Inlines array parameter of full-text functions, transforming FullTextContains(x, @keywordsArray) to FullTextContains(x, keyword1, keyword2)) 
+                case SqlFunctionExpression
+                {
+                    Name: "FullTextContainsAny" or "FullTextContainsAll",
+                    Arguments: [var property, SqlParameterExpression { TypeMapping: { ElementTypeMapping: var elementTypeMapping } } keywords]
+                } fullTextContainsAllAnyFunction
+                when !UseOldBehavior35476:
+                {
+                    var keywordValues = new List<SqlExpression>();
+                    foreach (var value in (IEnumerable)parametersValues[keywords.Name])
+                    {
+                        keywordValues.Add(sqlExpressionFactory.Constant(value, typeof(string), elementTypeMapping));
+                    }
 
-            return base.VisitExtension(expression);
+                    return sqlExpressionFactory.Function(
+                        fullTextContainsAllAnyFunction.Name,
+                        [property, .. keywordValues],
+                        fullTextContainsAllAnyFunction.Type,
+                        fullTextContainsAllAnyFunction.TypeMapping);
+                }
+
+                default:
+                    return expression;
+            }
         }
     }
 }

src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.PagingQueryingEnumerable.cs

@@ -75,7 +75,7 @@ public IAsyncEnumerator<CosmosPage<T>> GetAsyncEnumerator(CancellationToken canc
 
         private CosmosSqlQuery GenerateQuery()
             => _querySqlGeneratorFactory.Create().GetSqlQuery(
-                (SelectExpression)new InExpressionValuesExpandingExpressionVisitor(
+                (SelectExpression)new ParameterInliner(
                         _sqlExpressionFactory,
                         _cosmosQueryContext.ParameterValues)
                     .Visit(_selectExpression),

src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.QueryingEnumerable.cs

@@ -71,7 +71,7 @@ IEnumerator IEnumerable.GetEnumerator()
 
         private CosmosSqlQuery GenerateQuery()
             => _querySqlGeneratorFactory.Create().GetSqlQuery(
-                (SelectExpression)new InExpressionValuesExpandingExpressionVisitor(
+                (SelectExpression)new ParameterInliner(
                         _sqlExpressionFactory,
                         _cosmosQueryContext.ParameterValues)
                     .Visit(_selectExpression),

src/EFCore.Cosmos/Query/Internal/Expressions/SelectExpression.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using Microsoft.EntityFrameworkCore.Cosmos.Extensions;
 using Microsoft.EntityFrameworkCore.Cosmos.Internal;
 using Microsoft.EntityFrameworkCore.Internal;
 
@@ -16,6 +17,9 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal;
 [DebuggerDisplay("{PrintShortSql(), nq}")]
 public sealed class SelectExpression : Expression, IPrintableExpression
 {
+    private static readonly bool UseOldBehavior35476 =
+          AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue35476", out var enabled35476) && enabled35476;
+
     private IDictionary<ProjectionMember, Expression> _projectionMapping = new Dictionary<ProjectionMember, Expression>();
     private readonly List<SourceExpression> _sources = [];
     private readonly List<ProjectionExpression> _projection = [];
@@ -381,6 +385,12 @@ public void ApplyOffset(SqlExpression sqlExpression)
     /// </summary>
     public void ApplyOrdering(OrderingExpression orderingExpression)
     {
+        if (!UseOldBehavior35476 && orderingExpression is { Expression: SqlFunctionExpression { IsScoringFunction: true }, IsAscending: false })
+        {
+            throw new InvalidOperationException(
+                CosmosStrings.OrderByDescendingScoringFunction(nameof(Queryable.OrderByDescending), nameof(Queryable.OrderBy)));
+        }
+
         _orderings.Clear();
         _orderings.Add(orderingExpression);
     }
@@ -393,6 +403,19 @@ public void ApplyOrdering(OrderingExpression orderingExpression)
     /// </summary>
     public void AppendOrdering(OrderingExpression orderingExpression)
     {
+        if (!UseOldBehavior35476 && _orderings.Count > 0)
+        {
+            var existingScoringFunctionOrdering = _orderings is [{ Expression: SqlFunctionExpression { IsScoringFunction: true } }];
+            var appendingScoringFunctionOrdering = orderingExpression.Expression is SqlFunctionExpression { IsScoringFunction: true };
+            if (appendingScoringFunctionOrdering || existingScoringFunctionOrdering)
+            {
+                throw new InvalidOperationException(
+                    appendingScoringFunctionOrdering && existingScoringFunctionOrdering
+                    ? CosmosStrings.OrderByMultipleScoringFunctionWithoutRrf(nameof(CosmosDbFunctionsExtensions.Rrf))
+                    : CosmosStrings.OrderByScoringFunctionMixedWithRegularOrderby);
+            }
+        }
+
         if (_orderings.FirstOrDefault(o => o.Expression.Equals(orderingExpression.Expression)) == null)
         {
             _orderings.Add(orderingExpression);
@@ -752,6 +775,11 @@ private void PrintSql(ExpressionPrinter expressionPrinter, bool withTags = true)
         if (Orderings.Any())
         {
             expressionPrinter.AppendLine().Append("ORDER BY ");
+            if (!UseOldBehavior35476 && Orderings is [{ Expression: SqlFunctionExpression { IsScoringFunction: true } }])
+            {
+                expressionPrinter.Append("RANK ");
+            }
+
             expressionPrinter.VisitCollection(Orderings);
         }