Skip to content

Query: Adds LINQ support for Multi-key Group By translation #4857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Nov 25, 2024
Merged

Query: Adds LINQ support for Multi-key Group By translation #4857

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c243422
skeletal changes
Jul 16, 2024
57700d3
add t ranslation for basic multikey multivalue scenarios
Nov 3, 2024
8ca5daa
Fix translation with multikey multivalue translating into JOINS
Nov 11, 2024
cfaf599
Merge branch 'master' into users/leminh/groupbyLinqMultiKey
leminh98 Nov 11, 2024
7c8ff08
clean up
Nov 11, 2024
91d005d
Merge branch 'master' into users/leminh/groupbyLinqMultiKey
Nov 22, 2024
18fc4f1
address code review
Nov 22, 2024
d9e1820
Merge branch 'master' into users/leminh/groupbyLinqMultiKey
leminh98 Nov 22, 2024
8deeb3d
Merge branch 'master' into users/leminh/groupbyLinqMultiKey
leminh98 Nov 23, 2024
99b9104
address code review
Nov 24, 2024
afeb0fd
remove unnecssary comment
Nov 24, 2024
19053a2
Merge branch 'master' into users/leminh/groupbyLinqMultiKey
leminh98 Nov 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 152 additions & 52 deletions Microsoft.Azure.Cosmos/src/Linq/ExpressionToSQL.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ namespace Microsoft.Azure.Cosmos.Linq
using System.Linq.Expressions;
using System.Reflection;
using System.Text.RegularExpressions;
using Microsoft.Azure.Cosmos.CosmosElements;
using Microsoft.Azure.Cosmos.CosmosElements;
using Microsoft.Azure.Cosmos.Query.Core.ClientDistributionPlan.Cql;
using Microsoft.Azure.Cosmos.Serialization.HybridRow;
using Microsoft.Azure.Cosmos.Serializer;
using Microsoft.Azure.Cosmos.Spatial;
Expand Down Expand Up @@ -1361,14 +1362,6 @@ private static SqlSelectClause VisitGroupByAggregateMethodCall(MethodCallExpress
throw new DocumentQueryException(ClientResources.ExpectedMethodCallsMethods);
}

Expression inputCollection = inputExpression.Arguments[0]; // all these methods are static extension methods, so argument[0] is the collection

Collection collection = ExpressionToSql.Translate(inputCollection, context);
context.PushCollection(collection);

bool shouldBeOnNewQuery = context.CurrentQuery.ShouldBeOnNewQuery(inputExpression.Method.Name, inputExpression.Arguments.Count);
context.PushSubqueryBinding(shouldBeOnNewQuery);

if (context.LastExpressionIsGroupBy)
{
throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, "Group By cannot be followed by other methods"));
Expand Down Expand Up @@ -1406,8 +1399,6 @@ private static SqlSelectClause VisitGroupByAggregateMethodCall(MethodCallExpress
throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.MethodNotSupported, inputExpression.Method.Name));
}

context.PopSubqueryBinding();
context.PopCollection();
context.PopMethod();
return select;
}
Expand Down Expand Up @@ -1793,44 +1784,111 @@ private static Collection VisitGroupBy(Type returnElementType, ReadOnlyCollectio
{
throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.InvalidArgumentsCount, LinqMethods.GroupBy, 3, arguments.Count));
}

// bind the parameters in the value selector to the current input
foreach (ParameterExpression par in Utilities.GetLambda(arguments[2]).Parameters)
{
context.PushParameter(par, context.CurrentSubqueryBinding.ShouldBeOnNewQuery);
}


// Key Selector handling
// First argument is input, second is key selector and third is value selector
LambdaExpression keySelectorLambda = Utilities.GetLambda(arguments[1]);

// Current GroupBy doesn't allow subquery, so we need to visit non subquery scalar lambda
SqlScalarExpression keySelectorFunc = ExpressionToSql.VisitNonSubqueryScalarLambda(keySelectorLambda, context);

SqlGroupByClause groupby = SqlGroupByClause.Create(keySelectorFunc);

context.CurrentQuery = context.CurrentQuery.AddGroupByClause(groupby, context);

// Create a GroupBy collection and bind the new GroupBy collection to the new parameters created from the key
Collection collection = ExpressionToSql.ConvertToCollection(keySelectorFunc);
collection.isOuter = true;
collection.Name = "GroupBy";

ParameterExpression parameterExpression = context.GenerateFreshParameter(returnElementType, keySelectorFunc.ToString(), includeSuffix: false);
Binding binding = new Binding(parameterExpression, collection.inner, isInCollection: false, isInputParameter: true);

context.CurrentQuery.GroupByParameter = new FromParameterBindings();
context.CurrentQuery.GroupByParameter.Add(binding);

// The alias for the key in the value selector lambda is the first arguemt lambda - we bound it to the parameter expression, which already has substitution
ParameterExpression valueSelectorKeyExpressionAlias = Utilities.GetLambda(arguments[2]).Parameters[0];
context.GroupByKeySubstitution.AddSubstitution(valueSelectorKeyExpressionAlias, parameterExpression/*Utilities.GetLambda(arguments[1]).Body*/);

// Translate the body of the value selector lambda
LambdaExpression keySelectorLambda = Utilities.GetLambda(arguments[1]);

Collection collection = new Collection("Group By");
context.CurrentQuery.GroupByParameter = new FromParameterBindings();

SqlGroupByClause groupby;
ParameterExpression parameterExpression;
switch (keySelectorLambda.Body.NodeType)
{
case ExpressionType.Parameter:
case ExpressionType.Call:
case ExpressionType.MemberAccess:
{
// bind the parameters in the value selector to the current input
foreach (ParameterExpression par in Utilities.GetLambda(arguments[2]).Parameters)
{
context.PushParameter(par, context.CurrentSubqueryBinding.ShouldBeOnNewQuery);
}

//Current GroupBy doesn't allow subquery, so we need to visit non subquery scalar lambda
SqlScalarExpression keySelectorFunc = ExpressionToSql.VisitNonSubqueryScalarLambda(keySelectorLambda, context);

// The group by clause don't need to handle the value selector, so adding the clause to the uery now.
groupby = SqlGroupByClause.Create(keySelectorFunc);
parameterExpression = context.GenerateFreshParameter(returnElementType, keySelectorFunc.ToString(), includeSuffix: false);

break;
}
case ExpressionType.New:
{
// bind the parameters in the key selector to the current input - in this case, the value selector key is being substituted by the key selector
foreach (ParameterExpression par in Utilities.GetLambda(arguments[1]).Parameters)
{
context.PushParameter(par, context.CurrentSubqueryBinding.ShouldBeOnNewQuery);
}

NewExpression newExpression = (NewExpression)keySelectorLambda.Body;

if (newExpression.Members == null)
{
throw new DocumentQueryException(ClientResources.ConstructorInvocationNotSupported);
}

ReadOnlyCollection<Expression> newExpressionArguments = newExpression.Arguments;

List<SqlScalarExpression> keySelectorFunctions = new List<SqlScalarExpression>();
for (int i = 0; i < newExpressionArguments.Count; i++)
{
//Current GroupBy doesn't allow subquery, so we need to visit non subquery scalara
SqlScalarExpression keySelectorFunc = ExpressionToSql.VisitNonSubqueryScalarExpression(newExpressionArguments[i], context);
keySelectorFunctions.Add(keySelectorFunc);
}

groupby = SqlGroupByClause.Create(keySelectorFunctions.ToImmutableArray());
parameterExpression = context.GenerateFreshParameter(returnElementType, keySelectorFunctions.ToString(), includeSuffix: false);

break;
}
default:
throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.ExpressionTypeIsNotSupported, keySelectorLambda.Body.NodeType));
}

// The group by clause don't need to handle the value selector, so adding the clause to the qery now.
context.CurrentQuery = context.CurrentQuery.AddGroupByClause(groupby, context);

// Bind the alias
Binding binding = new Binding(parameterExpression, collection.inner, isInCollection: false, isInputParameter: true);
context.CurrentQuery.GroupByParameter.Add(binding);

// The alias for the key in the value selector lambda is the first arguemt lambda - we bound it to the parameter expression, which already has substitution
ParameterExpression valueSelectorKeyExpressionAlias = Utilities.GetLambda(arguments[2]).Parameters[0];
context.GroupByKeySubstitution.AddSubstitution(valueSelectorKeyExpressionAlias, parameterExpression);

// Value Selector Handingling
// Translate the body of the value selector lambda
Expression valueSelectorExpression = Utilities.GetLambda(arguments[2]).Body;

// The value selector function needs to be either a MethodCall or an AnonymousType
switch (valueSelectorExpression.NodeType)
{
{
case ExpressionType.MemberAccess:
{
MemberExpression memberAccessExpression = (MemberExpression)valueSelectorExpression;

if (memberAccessExpression.Expression.NodeType == ExpressionType.Parameter)
{
// Look up the object of the expression to see if it is the key
ParameterExpression memberAccessObject = (ParameterExpression)memberAccessExpression.Expression;
Expression subst = context.GroupByKeySubstitution.Lookup(memberAccessObject);
if (subst != null)
{
// If there is a match, we construct a new Member Access expression with the substituted expression and visit it to create a select clause
MemberExpression newMemberAccessExpression = memberAccessExpression.Update(keySelectorLambda.Body);
SqlScalarExpression selectExpression = ExpressionToSql.VisitMemberAccess(newMemberAccessExpression, context);

SqlSelectSpec sqlSpec = SqlSelectValueSpec.Create(selectExpression);
SqlSelectClause select = SqlSelectClause.Create(sqlSpec, null);
context.CurrentQuery = context.CurrentQuery.AddSelectClause(select, context);
}
}
break;
}
case ExpressionType.Constant:
{
ConstantExpression constantExpression = (ConstantExpression)valueSelectorExpression;
Expand Down Expand Up @@ -1908,6 +1966,27 @@ private static Collection VisitGroupBy(Type returnElementType, ReadOnlyCollectio
selectItems[i] = prop;
break;
}
case ExpressionType.MemberAccess:
{
MemberExpression memberAccessExpression = (MemberExpression)arg;

if (memberAccessExpression.Expression.NodeType == ExpressionType.Parameter)
{
// Look up the object of the expression to see if it is the key
ParameterExpression memberAccessObject = (ParameterExpression)memberAccessExpression.Expression;
Expression subst = context.GroupByKeySubstitution.Lookup(memberAccessObject);
if (subst != null)
{
// If there is a match, we construct a new Member Access expression with the substituted expression and visit it to create a select clause
MemberExpression newMemberAccessExpression = memberAccessExpression.Update(keySelectorLambda.Body); /*System.Linq.Expressions.Expression.Field(subst, memberAccessExpression.Member.Name);*/
SqlScalarExpression selectExpression = ExpressionToSql.VisitMemberAccess(newMemberAccessExpression, context);

SqlSelectItem prop = SqlSelectItem.Create(selectExpression, alias);
selectItems[i] = prop;
}
}
break;
}
default:
throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.ExpressionTypeIsNotSupported, arg.NodeType));
}
Expand All @@ -1921,13 +2000,34 @@ private static Collection VisitGroupBy(Type returnElementType, ReadOnlyCollectio
}
default:
throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.ExpressionTypeIsNotSupported, valueSelectorExpression.NodeType));
}

foreach (ParameterExpression par in Utilities.GetLambda(arguments[2]).Parameters)
{
context.PopParameter();
}

}

// Pop the correct number of items off the parameter stack
switch (keySelectorLambda.Body.NodeType)
{
case ExpressionType.Parameter:
case ExpressionType.Call:
case ExpressionType.MemberAccess:
{
foreach (ParameterExpression param in Utilities.GetLambda(arguments[2]).Parameters)
{
context.PopParameter();
}
break;
}
case ExpressionType.New:
{
//bind the parameters in the value selector to the current input
foreach (ParameterExpression param in Utilities.GetLambda(arguments[1]).Parameters)
{
context.PopParameter();
}
break;
}
default:
break;
}

return collection;
}

Expand Down Expand Up @@ -2234,7 +2334,7 @@ private static SqlInputPathCollection ConvertMemberIndexerToPath(SqlMemberIndexe
if (parent == null)
{
break;
}
}

if (parent is SqlPropertyRefScalarExpression sqlPropertyRefScalarExpression)
{
Expand Down
9 changes: 5 additions & 4 deletions Microsoft.Azure.Cosmos/src/Linq/QueryUnderConstruction.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,8 +250,8 @@ public QueryUnderConstruction FlattenAsPossible()
break;
}

// In case of Select -> Group by cases, the Select query should not be flattened and kept as a subquery
if ((query.inputQuery?.selectClause != null) && (query.groupByClause != null))
// In case of Select/Order By -> Group by cases, the Select/Order By query should not be flattened and kept as a subquery
if (((query.inputQuery?.selectClause != null) || (query.inputQuery?.orderByClause != null)) && (query.groupByClause != null))
{
flattenQuery = this;
break;
Expand Down Expand Up @@ -564,11 +564,12 @@ public bool ShouldBeOnNewQuery(string methodName, int argumentCount)
break;

case LinqMethods.GroupBy:
// New query is needed when there is already a Take or a Select or a Group by clause
// New query is needed when there is already a Take or a Select or a Group by clause or an Order By clause
shouldPackage = (this.topSpec != null) ||
(this.offsetSpec != null) ||
(this.selectClause != null) ||
(this.groupByClause != null);
(this.groupByClause != null) ||
(this.orderByClause != null);
break;

case LinqMethods.Skip:
Expand Down
4 changes: 2 additions & 2 deletions Microsoft.Azure.Cosmos/src/Linq/TranslationContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,12 @@ public void PushCollection(Collection collection)
throw new ArgumentNullException("collection");
}

if (this.CurrentQuery.GroupByParameter == null) this.collectionStack.Add(collection);
this.collectionStack.Add(collection);
}

public void PopCollection()
{
if (this.CurrentQuery.GroupByParameter == null) this.collectionStack.RemoveAt(this.collectionStack.Count - 1);
this.collectionStack.RemoveAt(this.collectionStack.Count - 1);
}

/// <summary>
Expand Down
Loading