Skip to content

Commit 8733ed6

Browse files
gs-jp1gs-jp1
authored
Pure2SQL - optimise tds rename to avoid unnecessary nested subqueries (#4473)
1 parent 7cac070 commit 8733ed6

File tree

5 files changed

+73
-19
lines changed
  • legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/tests
  • legend-engine-xts-python/legend-engine-xt-python-reversePCT-legendQL/src/main/resources/core_external_python_reverse_pct_legend_ql/relation
  • legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational
  • legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-reversePCT/src/main/resources/core_external_query_sql_reverse_pct/relation

5 files changed

+73
-19
lines changed

‎legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/tests/composition.pure‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -923,13 +923,15 @@ function <<PCT.test>> meta::pure::functions::relation::tests::composition::testM
923923
#->rename(~colOneNum, ~_col_one_num)
924924
->rename(~Col_two_letter, ~colTwoLetter)
925925
->rename(~_col_3Mix, ~Col_3mix)
926-
->filter(x|$x._col_one_num > 2 && $x.colTwoLetter == 'C' && $x.Col_3mix == 3)
926+
->rename(~Col_3mix, ~Col_3mix_)
927+
->rename(~Col_3mix_, ~_Col_3mix_)
928+
->filter(x|$x._col_one_num > 2 && $x.colTwoLetter == 'C' && $x._Col_3mix_ == 3)
927929
};
928930

929931
let res = $f->eval($expr);
930932

931933
assertEquals( '#TDS\n'+
932-
' _col_one_num,colTwoLetter,Col_3mix\n'+
934+
' _col_one_num,colTwoLetter,_Col_3mix_\n'+
933935
' 9,C,3\n'+
934936
'#', $res->toString());
935937
}

‎legend-engine-xts-python/legend-engine-xt-python-reversePCT-legendQL/src/main/resources/core_external_python_reverse_pct_legend_ql/relation/relation.pure‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1992,9 +1992,9 @@ function meta::external::python::reversePCT::legendQL::pythonLegendQLReversesEss
19921992
'7,C,4\n'+
19931993
'8,A,2\n'+
19941994
'9,C,3\n'+
1995-
'10,D,1}#->meta::pure::functions::relation::rename(~colOneNum, ~_col_one_num)->meta::pure::functions::relation::rename(~Col_two_letter, ~colTwoLetter)->meta::pure::functions::relation::rename(~_col_3Mix, ~Col_3mix)->meta::pure::functions::relation::filter(x: (_col_one_num:Integer[0..1], colTwoLetter:String[0..1], Col_3mix:Integer[0..1])[1]|(($x._col_one_num > 2) && ($x.colTwoLetter == \'C\')) && ($x.Col_3mix == 3))',
1995+
'10,D,1}#->meta::pure::functions::relation::rename(~colOneNum, ~_col_one_num)->meta::pure::functions::relation::rename(~Col_two_letter, ~colTwoLetter)->meta::pure::functions::relation::rename(~_col_3Mix, ~Col_3mix)->meta::pure::functions::relation::rename(~Col_3mix, ~Col_3mix_)->meta::pure::functions::relation::rename(~Col_3mix_, ~_Col_3mix_)->meta::pure::functions::relation::filter(x: (_col_one_num:Integer[0..1], colTwoLetter:String[0..1], _Col_3mix_:Integer[0..1])[1]|(($x._col_one_num > 2) && ($x.colTwoLetter == \'C\')) && ($x._Col_3mix_ == 3))',
19961996
'(lambda frame:' +
1997-
' frame.rename(lambda r : (r.colOneNum , \'_col_one_num\')).rename(lambda r : (r.Col_two_letter , \'colTwoLetter\')).rename(lambda r : (r._col_3Mix , \'Col_3mix\')).filter((lambda x: ((x._col_one_num > 2) & (x.colTwoLetter == "C") & (x.Col_3mix == 3))))' +
1997+
' frame.rename(lambda r : (r.colOneNum , \'_col_one_num\')).rename(lambda r : (r.Col_two_letter , \'colTwoLetter\')).rename(lambda r : (r._col_3Mix , \'Col_3mix\')).rename(lambda r : (r.Col_3mix , \'Col_3mix_\')).rename(lambda r : (r.Col_3mix_ , \'_Col_3mix_\')).filter((lambda x: ((x._col_one_num > 2) & (x.colTwoLetter == "C") & (x._Col_3mix_ == 3))))' +
19981998
')(csv_frame("colOneNum,Col_two_letter,_col_3Mix\\n1,A,1\\n2,A,1\\n3,B,2\\n4,D,3\\n5,B,2\\n6,A,1\\n7,C,4\\n8,A,2\\n9,C,3\\n10,D,1"))')
19991999
]
20002000
),

‎legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6526,19 +6526,41 @@ function <<access.private>> meta::relational::functions::pureToSqlQuery::process
65266526
);
65276527
}
65286528

6529+
//this function collapses chained renames to avoid unnecessary nested subqueries
6530+
function meta::relational::functions::pureToSqlQuery::collapseTDSRename(v:ValueSpecification[1], vars:Map<VariableExpression, ValueSpecification>[1], state:State[1], columns:Map<String,String>[1]):Pair<ValueSpecification, Map<String, String>>[1]
6531+
{
6532+
$v->match([
6533+
f:FunctionExpression[1] |
6534+
if ($f.func->in([renameColumns_TabularDataSet_1__Pair_MANY__TabularDataSet_1_, rename_Relation_1__ColSpec_1__ColSpec_1__Relation_1_]),
6535+
| let pairs = $f->instanceValuesAtParameter(1, $vars, $state.inScopeVars)->match([
6536+
c:meta::pure::metamodel::relation::ColSpec<Any>[1] |
6537+
let newName = $f->instanceValuesAtParameter(2, $vars, $state.inScopeVars)->cast(@meta::pure::metamodel::relation::ColSpec<Any>).name->toOne();
6538+
pair($c.name->toOne(), $newName);,
6539+
p:Pair<String,String>[*] | $p,
6540+
vss:ValueSpecification[*] | $vss->map(vs|$vs->reprocessVS()->reactivate($state.inScopeVars)->evaluateAndDeactivate()->cast(@Pair<String,String>))
6541+
]);
6542+
collapseTDSRename($f.parametersValues->at(0), $vars, $state, resolveAndRemoveChainedRenames($pairs, $columns));,
6543+
| pair($v, $columns)),
6544+
v:ValueSpecification[1] | pair($v, $columns)
6545+
]);
6546+
}
6547+
6548+
function meta::relational::functions::pureToSqlQuery::resolveAndRemoveChainedRenames(pairs:Pair<String,String>[*], columns:Map<String,String>[1]):Map<String,String>[1]
6549+
{
6550+
$columns->keyValues()
6551+
->filter(kv | !$kv.first->in($pairs.second))
6552+
->newMap()
6553+
->putAll($pairs->map(pr | pair($pr.first, $columns->get($pr.second)->orElse($pr.second))));
6554+
}
6555+
65296556
function meta::relational::functions::pureToSqlQuery::processTdsRenameColumns(expression:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map<VariableExpression, ValueSpecification>[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List<ColumnGroup>[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1]
65306557
{
6531-
let mainQuery = processValueSpecification($expression.parametersValues->at(0), [], $operation, $vars, $state, JoinType.LEFT_OUTER, $nodeId, $aggFromMap, $context, $extensions)->toOne()->cast(@SelectWithCursor);
6558+
let collapsed = collapseTDSRename($expression, $vars, $state, ^Map<String, String>());
6559+
6560+
let mainQuery = processValueSpecification($collapsed.first, [], $operation, $vars, $state, JoinType.LEFT_OUTER, $nodeId, $aggFromMap, $context, $extensions)->toOne()->cast(@SelectWithCursor);
65326561
let select = $mainQuery.select->cast(@TdsSelectSqlQuery);
65336562

6534-
let projectColPairNames = $expression->instanceValuesAtParameter(1, $vars, $state.inScopeVars)->match([
6535-
c:meta::pure::metamodel::relation::ColSpec<Any>[1] |pair (
6536-
$expression->instanceValuesAtParameter(1, $vars, $state.inScopeVars)->cast(@meta::pure::metamodel::relation::ColSpec<Any>).name->toOne(),
6537-
$expression->instanceValuesAtParameter(2, $vars, $state.inScopeVars)->cast(@meta::pure::metamodel::relation::ColSpec<Any>).name->toOne()
6538-
),
6539-
p:Pair<String,String>[*] | $p,
6540-
vss:ValueSpecification[*] | $vss->map(vs|$vs->reprocessVS()->reactivate($state.inScopeVars)->evaluateAndDeactivate()->cast(@Pair<String,String>));
6541-
]);
6563+
let projectColPairNames = $collapsed.second->keyValues();
65426564

65436565
let projectColNamesMap = $projectColPairNames->map(p|^$p(first = $p.first->addQuotesIfNecessary(), second = $p.second->addQuotesIfNecessary()))->newMap();
65446566
let noWrappedProjectColNamesMap = $projectColPairNames->map(p|^$p(first = $p.first->stripMatchingQuotes(), second = $p.second))->newMap();

‎legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tds/tests/testTdsRenameColumns.pure‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import meta::relational::runtime::*;
2020
import meta::relational::tests::*;
2121
import meta::external::store::relational::tests::*;
2222

23-
function <<test.Test>> meta::relational::tests::tds::tdsProject::testRenameColumns():Boolean[1]
23+
function <<test.Test>> meta::relational::tests::tds::tdsRename::testRenameColumns():Boolean[1]
2424
{
2525
let result = execute(
2626
|Person.all()
@@ -40,8 +40,37 @@ function <<test.Test>> meta::relational::tests::tds::tdsProject::testRenameColum
4040
$result->sqlRemoveFormatting());
4141
}
4242

43+
function <<test.Test>> meta::relational::tests::tds::tdsRename::testChainedRenameColumns():Boolean[1]
44+
{
45+
let result = execute(
46+
|Person.all()
47+
->project(p|$p.firstName,'firstName')
48+
->project([col(r:TDSRow[1]|$r.getString('firstName'), 'firstName'), col({r:TDSRow[1]|'Hello'}, 'hello')])
49+
->renameColumns([
50+
^Pair<String,String>(first = 'firstName', second = 'First Name'),
51+
^Pair<String,String>(first = 'hello', second = 'Other')
52+
])
53+
->renameColumns([
54+
^Pair<String,String>(first = 'First Name', second = 'first')
55+
])
56+
->limit(10)
57+
->renameColumns([
58+
^Pair<String,String>(first = 'Other', second = 'other')
59+
]),
60+
simpleRelationalMapping,
61+
meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions());
62+
63+
assertSize($result.values.columns, 2);
64+
65+
assertEquals('Peter|Hello,John|Hello,John|Hello,Anthony|Hello,Fabrice|Hello,Oliver|Hello,David|Hello',
66+
$result.values.rows->map(r|$r.values->makeString('|'))->makeString(','));
67+
68+
assertEquals('select "persontable_0"."first" as "first", "persontable_0"."Other" as "other" from (select "subselect"."first" as "first", "subselect"."Other" as "Other" from (select top 10 "persontable_2"."firstName" as "first", "persontable_2"."hello" as "Other" from (select "root".FIRSTNAME as "firstName", \'Hello\' as "hello" from personTable as "root") as "persontable_2") as "subselect") as "persontable_0"',
69+
$result->sqlRemoveFormatting());
70+
}
71+
4372

44-
function <<test.Test>> meta::relational::tests::tds::tdsProject::testRenameColumnsAfterGroupBy():Boolean[1]
73+
function <<test.Test>> meta::relational::tests::tds::tdsRename::testRenameColumnsAfterGroupBy():Boolean[1]
4574
{
4675
let result = execute(
4776
|Person.all()
@@ -66,7 +95,7 @@ function <<test.Test>> meta::relational::tests::tds::tdsProject::testRenameColum
6695
}
6796

6897

69-
function <<test.Test>> meta::relational::tests::tds::tdsProject::testRenameColumnsAfterConcatenate():Boolean[1]
98+
function <<test.Test>> meta::relational::tests::tds::tdsRename::testRenameColumnsAfterConcatenate():Boolean[1]
7099
{
71100
let result = execute(
72101
|Person.all()->project(col(p|$p.firstName,'firstName'))
@@ -87,7 +116,7 @@ function <<test.Test>> meta::relational::tests::tds::tdsProject::testRenameColum
87116
$result->sqlRemoveFormatting());
88117
}
89118

90-
function <<test.Test>> meta::relational::tests::tds::tdsProject::testRenameColumnsWithQuotes():Boolean[1]
119+
function <<test.Test>> meta::relational::tests::tds::tdsRename::testRenameColumnsWithQuotes():Boolean[1]
91120
{
92121
let queryWithoutQuotes = {|tableToTDS(meta::relational::functions::database::tableReference(meta::relational::tests::db,'default','tableWithQuotedColumns'))
93122
->project([

‎legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-reversePCT/src/main/resources/core_external_query_sql_reverse_pct/relation/composition.pure‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,8 +380,9 @@ function meta::external::query::sql::reversePCT::tests::relation::tests::composi
380380
'7,C,4\n'+
381381
'8,A,2\n'+
382382
'9,C,3\n'+
383-
'10,D,1\n}#->meta::pure::functions::relation::rename(~colOneNum, ~_col_one_num)->meta::pure::functions::relation::rename(~Col_two_letter, ~colTwoLetter)->meta::pure::functions::relation::rename(~_col_3Mix, ~Col_3mix)->meta::pure::functions::relation::filter(x: (_col_one_num:Integer, colTwoLetter:String, Col_3mix:Integer)[1]|(($x._col_one_num > 2) && ($x.colTwoLetter == \'C\')) && ($x.Col_3mix == 3))',
384-
'select "table0_0"."_col_one_num" as "_col_one_num", "table0_0"."colTwoLetter" as "colTwoLetter", "table0_0"."_col_3Mix" as "Col_3mix" from (select "table0_1"."_col_one_num" as "_col_one_num", "table0_1"."Col_two_letter" as "colTwoLetter", "table0_1"."_col_3Mix" as "_col_3Mix" from (select "table0_2"."colOneNum" as "_col_one_num", "table0_2"."Col_two_letter" as "Col_two_letter", "table0_2"."_col_3Mix" as "_col_3Mix" from (select "table0_3".colOneNum as "colOneNum", "table0_3".Col_two_letter as "Col_two_letter", "table0_3"._col_3Mix as "_col_3Mix" from csv(\'colOneNum,Col_two_letter,_col_3Mix\n1,A,1\n2,A,1\n3,B,2\n4,D,3\n5,B,2\n6,A,1\n7,C,4\n8,A,2\n9,C,3\n10,D,1\') as "table0_3" where "table0_3".colOneNum is not null and "table0_3".colOneNum > 2 and "table0_3".Col_two_letter = Text\'C\' and "table0_3"._col_3Mix = 3) as "table0_2" where "table0_2"."colOneNum" is not null and "table0_2"."colOneNum" > 2 and "table0_2"."Col_two_letter" = Text\'C\' and "table0_2"."_col_3Mix" = 3) as "table0_1" where "table0_1"."_col_one_num" is not null and "table0_1"."_col_one_num" > 2 and "table0_1"."Col_two_letter" = Text\'C\' and "table0_1"."_col_3Mix" = 3) as "table0_0" where ((("table0_0"."_col_one_num" is not null and "table0_0"."_col_one_num" > 2) and "table0_0"."colTwoLetter" = Text\'C\') and "table0_0"."_col_3Mix" = 3)'
383+
'10,D,1\n'+
384+
'}#->meta::pure::functions::relation::rename(~colOneNum, ~_col_one_num)->meta::pure::functions::relation::rename(~Col_two_letter, ~colTwoLetter)->meta::pure::functions::relation::rename(~_col_3Mix, ~Col_3mix)->meta::pure::functions::relation::rename(~Col_3mix, ~Col_3mix_)->meta::pure::functions::relation::rename(~Col_3mix_, ~_Col_3mix_)->meta::pure::functions::relation::filter(x: (_col_one_num:Integer[0..1], colTwoLetter:String[0..1], _Col_3mix_:Integer[0..1])[1]|(($x._col_one_num > 2) && ($x.colTwoLetter == \'C\')) && ($x._Col_3mix_ == 3))',
385+
'select "table0_0"."colOneNum" as "_col_one_num", "table0_0"."Col_two_letter" as "colTwoLetter", "table0_0"."_col_3Mix" as "_Col_3mix_" from (select "table0_1".colOneNum as "colOneNum", "table0_1".Col_two_letter as "Col_two_letter", "table0_1"._col_3Mix as "_col_3Mix" from csv(\'colOneNum,Col_two_letter,_col_3Mix\n1,A,1\n2,A,1\n3,B,2\n4,D,3\n5,B,2\n6,A,1\n7,C,4\n8,A,2\n9,C,3\n10,D,1\') as "table0_1" where "table0_1".colOneNum is not null and "table0_1".colOneNum > 2 and "table0_1".Col_two_letter = Text\'C\' and "table0_1"._col_3Mix = 3) as "table0_0" where ((("table0_0"."colOneNum" is not null and "table0_0"."colOneNum" > 2) and "table0_0"."Col_two_letter" = Text\'C\') and "table0_0"."_col_3Mix" = 3)'
385386
)
386387
]
387388
),

0 commit comments

Comments
 (0)