Skip to content

Commit e1f6bf6

Browse files
sahil37sahil37
authored
Small fixes to dq to make runtime optional (#4474)
1 parent 34ceac8 commit e1f6bf6

8 files changed

Lines changed: 97 additions & 42 deletions

File tree

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ public Response generatePlan(DataQualityExecuteTrialInput dataQualityExecuteInpu
133133
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_START).toString());
134134
try (Scope scope = GlobalTracer.get().buildSpan("DataQuality: planGeneration").startActive(true))
135135
{
136-
SingleExecutionPlan singleExecutionPlan = generateExecutionPlan(dataQualityExecuteInput, identity, false);
136+
SingleExecutionPlan singleExecutionPlan = generateExecutionPlan(dataQualityExecuteInput, identity, false, "generate plan");
137137
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_END, System.currentTimeMillis() - start).toString());
138138
return ManageConstantResult.manageResult(identity.getName(), singleExecutionPlan, objectMapper);
139139
}
@@ -155,7 +155,7 @@ public Response generatePlanDebug(DataQualityExecuteTrialInput dataQualityExecut
155155
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_PLAN_GENERATION_DEBUG_START).toString());
156156
try (Scope scope = GlobalTracer.get().buildSpan("DataQuality: planGenerationDebug").startActive(true))
157157
{
158-
PlanWithDebug planWithDebug = generateExecutionPlanDebug(dataQualityExecuteInput, identity, false);
158+
PlanWithDebug planWithDebug = generateExecutionPlanDebug(dataQualityExecuteInput, identity, false, "generate plan debug");
159159
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_PLAN_GENERATION_DEBUG_STOP, System.currentTimeMillis() - start).toString());
160160
return ManageConstantResult.manageResult(identity.getName(), planWithDebug, objectMapper);
161161
}
@@ -177,7 +177,7 @@ public Response generatePlanRowCount(DataQualityExecuteTrialInput dataQualityExe
177177
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_QUERY_COUNT_START).toString());
178178
try (Scope scope = GlobalTracer.get().buildSpan("DataQuality: planGenerationRowCount").startActive(true))
179179
{
180-
SingleExecutionPlan singleExecutionPlan = generateExecutionPlan(dataQualityExecuteInput, identity, true);
180+
SingleExecutionPlan singleExecutionPlan = generateExecutionPlan(dataQualityExecuteInput, identity, true, "generate plan for row count");
181181
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_QUERY_COUNT_END, System.currentTimeMillis() - start).toString());
182182
return ManageConstantResult.manageResult(identity.getName(), singleExecutionPlan, objectMapper);
183183
}
@@ -200,7 +200,7 @@ public Response execute(@Context HttpServletRequest request, DataQualityExecuteT
200200
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_PLAN_EXECUTION_START).toString());
201201
try (Scope scope = GlobalTracer.get().buildSpan("DataQuality: executeTrial").startActive(true))
202202
{
203-
SingleExecutionPlan singleExecutionPlan = generateExecutionPlan(dataQualityExecuteInput, identity, false);
203+
SingleExecutionPlan singleExecutionPlan = generateExecutionPlan(dataQualityExecuteInput, identity, false, "execute");
204204
Map<String, Object> lambdaParameterMap = dataQualityExecuteInput.lambdaParameterValues != null ? dataQualityExecuteInput.lambdaParameterValues.stream().collect(Collectors.<ParameterValue, String, Object>toMap(p -> p.name, p -> p.value.accept(new PrimitiveValueSpecificationToObjectVisitor()))) : Maps.mutable.empty();
205205
Response response = executePlan(request, identity, format, start, singleExecutionPlan, lambdaParameterMap);
206206
if (response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL))
@@ -229,7 +229,7 @@ public Response executeRowCount(@Context HttpServletRequest request, DataQuality
229229
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_RELATION_QUERY_COUNT_PLAN_EXECUTION_START).toString());
230230
try (Scope scope = GlobalTracer.get().buildSpan("DataQuality: executeRelationValidationRowCount").startActive(true))
231231
{
232-
SingleExecutionPlan singleExecutionPlan = generateExecutionPlan(dataQualityExecuteInput, identity, true);
232+
SingleExecutionPlan singleExecutionPlan = generateExecutionPlan(dataQualityExecuteInput, identity, true, "execute row count");
233233
Map<String, Object> lambdaParameterMap = dataQualityExecuteInput.lambdaParameterValues != null ? dataQualityExecuteInput.lambdaParameterValues.stream().collect(Collectors.<ParameterValue, String, Object>toMap(p -> p.name, p -> p.value.accept(new PrimitiveValueSpecificationToObjectVisitor()))) : Maps.mutable.empty();
234234
Response response = executePlan(request, identity, format, start, singleExecutionPlan, lambdaParameterMap);
235235
if (response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL))
@@ -358,33 +358,33 @@ private Result getDataReconciliation(HttpServletRequest request, PureModel pureM
358358
return executePlanToResult(request, identity, singleExecutionPlan, Maps.mutable.empty());
359359
}
360360

361-
private SingleExecutionPlan generateExecutionPlan(DataQualityExecuteTrialInput dataQualityExecuteInput, Identity identity, boolean rowCount)
361+
private SingleExecutionPlan generateExecutionPlan(DataQualityExecuteTrialInput dataQualityExecuteInput, Identity identity, boolean rowCount, String queryType)
362362
{
363363
// 1. load pure model from PureModelContext
364364
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
365365
// 2. call DQ PURE func to generate lambda
366-
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = generateLambdaFunction(dataQualityExecuteInput, pureModel, rowCount);
366+
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = generateLambdaFunction(dataQualityExecuteInput, pureModel, rowCount, queryType);
367367
// 3. Generate Plan from the lambda generated in the previous step
368368
return PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);
369369
}
370370

371-
private PlanWithDebug generateExecutionPlanDebug(DataQualityExecuteTrialInput dataQualityExecuteInput, Identity identity, boolean rowCount)
371+
private PlanWithDebug generateExecutionPlanDebug(DataQualityExecuteTrialInput dataQualityExecuteInput, Identity identity, boolean rowCount, String queryType)
372372
{
373373
// 1. load pure model from PureModelContext
374374
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
375375
// 2. call DQ PURE func to generate lambda
376-
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = generateLambdaFunction(dataQualityExecuteInput, pureModel, rowCount);
376+
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = generateLambdaFunction(dataQualityExecuteInput, pureModel, rowCount, queryType);
377377
// 3. Generate Plan debug from the lambda generated in the previous step
378378
return PlanGenerator.generateExecutionPlanDebug(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);
379379
}
380380

381-
private org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction generateLambdaFunction(DataQualityExecuteTrialInput dataQualityExecuteInput, PureModel pureModel, boolean rowCount)
381+
private org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction generateLambdaFunction(DataQualityExecuteTrialInput dataQualityExecuteInput, PureModel pureModel, boolean rowCount, String queryType)
382382
{
383383
if (rowCount)
384384
{
385385
return DataQualityLambdaGenerator.generateRelationValidationMainQueryRowCount(pureModel, dataQualityExecuteInput.packagePath);
386386
}
387-
return DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns);
387+
return DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, queryType);
388388
}
389389

390390
@POST
@@ -398,7 +398,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr
398398
// 1. load pure model from PureModelContext
399399
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
400400
// 2. call DQ PURE func to generate lambda
401-
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns);
401+
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, null);
402402
LambdaFunction lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions);
403403
return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper);
404404
}

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ private LambdaFunction<?> buildDataqualityRelationValidationQuery(DataqualityRel
219219
}
220220
LambdaFunction<?> relationQuery = HelperValueSpecificationBuilder.buildLambda(dataqualityRelationValidation.query, compileContext);
221221
String relationQueryReturnType = getLambdaFunctionRawReturnTypeName(relationQuery);
222-
if (!(relationQuery._expressionSequence().getLast() instanceof Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl) && !"Relation".equals(relationQueryReturnType))
222+
if (!(relationQuery._expressionSequence().getLast() instanceof Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl) && !compileContext.pureModel.taxonomyTypes("cov_relation_Relation").contains(relationQueryReturnType))
223223
{
224224
throw new EngineException("Relation expected from lambda", dataqualityRelationValidation.query.sourceInformation, EngineErrorType.COMPILATION);
225225
}

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,20 @@ public void testRelationValidation()
225225
" }\n" +
226226
" ];\n" +
227227
"}");
228+
229+
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(COMPILATION_PREREQUISITE_CODE +
230+
"###DataQualityValidation\n" +
231+
"DataQualityRelationValidation meta::external::dataquality::testvalidation\n" +
232+
"{\n" +
233+
" query: #>{meta::dataquality::db.personTable}#;\n" +
234+
" validations: [\n" +
235+
" {\n" +
236+
" name: 'nonEmptyDataset';\n" +
237+
" description: 'dataset cannot be empty';\n" +
238+
" assertion: rel|$rel->assertRelationNotEmpty();\n" +
239+
" }\n" +
240+
" ];\n" +
241+
"}");
228242
}
229243

230244
@Test

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,13 @@
118118
</dependency>
119119
<!-- LOG -->
120120

121+
<!-- COMMONS-LANG -->
122+
<dependency>
123+
<groupId>org.apache.commons</groupId>
124+
<artifactId>commons-lang3</artifactId>
125+
</dependency>
126+
<!-- COMMONS-LANG -->
127+
121128
<!-- TEST -->
122129
<dependency>
123130
<groupId>junit</groupId>

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,24 +37,27 @@
3737
import java.util.Objects;
3838
import java.util.Set;
3939

40+
import static java.lang.String.format;
41+
import static org.apache.commons.lang3.StringUtils.isNotBlank;
42+
4043
public class DataQualityLambdaGenerator
4144
{
4245

43-
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, String qualifiedPath, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns)
46+
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, String qualifiedPath, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
4447
{
4548
PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath);
46-
return generateLambda(pureModel, packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns);
49+
return generateLambda(pureModel, packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, queryType);
4750
}
4851

49-
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, PackageableElement packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns)
52+
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, PackageableElement packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
5053
{
5154
if (packageableElement instanceof Root_meta_external_dataquality_DataQuality)
5255
{
5356
return generateModelConstraintLambda(pureModel, null, (Root_meta_external_dataquality_DataQuality) packageableElement);
5457
}
5558
else if (packageableElement instanceof Root_meta_external_dataquality_DataQualityRelationValidation)
5659
{
57-
return generateRelationValidationLambda(pureModel, (Root_meta_external_dataquality_DataQualityRelationValidation) packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns);
60+
return generateRelationValidationLambda(pureModel, (Root_meta_external_dataquality_DataQualityRelationValidation) packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, queryType);
5861
}
5962
throw new EngineException("Unsupported Dataquality element! " + packageableElement.getClass().getSimpleName(), ExceptionCategory.USER_EXECUTION_ERROR);
6063
}
@@ -64,8 +67,12 @@ private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.functio
6467
return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_$0_1$__LambdaFunction_1_(packageableElement, Objects.isNull(queryLimit) ? null : queryLimit.longValue(), pureModel.getExecutionSupport());
6568
}
6669

67-
private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateRelationValidationLambda(PureModel pureModel, Root_meta_external_dataquality_DataQualityRelationValidation packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns)
70+
private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateRelationValidationLambda(PureModel pureModel, Root_meta_external_dataquality_DataQualityRelationValidation packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
6871
{
72+
if (isNotBlank(queryType) && !core_dataquality_generation_dataquality.Root_meta_external_dataquality_isEndingWithFromFunction_FunctionDefinition_1__Boolean_1_(packageableElement._query(), pureModel.getExecutionSupport()))
73+
{
74+
throw new EngineException(format("The query for the DataQualityRelationValidation '%s' does not end with a 'from' so unable to %s.", packageableElement._name(), queryType), ExceptionCategory.USER_EXECUTION_ERROR);
75+
}
6976
if (Boolean.TRUE.equals(runQuery))
7077
{
7178
return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object>) packageableElement._query();

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public List<Artifact> generate(PackageableElement packageableElement, PureModel
8080

8181
Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport()));
8282
// 1. call DQ PURE func to generate lambda
83-
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null, false, null, true);
83+
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null, false, null, true, "generate artifacts for dq");
8484
// 2. Generate Plan from the lambda generated in the previous step
8585
SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, clientVersion, PlanPlatform.JAVA, null,
8686
routerExtensions.apply(pureModel), LegendPlanTransformers.transformers);// since lambda has from function we dont need mapping, runtime and context

0 commit comments

Comments
 (0)