From fc4bfc9479cf603ed933b7a25d7daade18c3bb7a Mon Sep 17 00:00:00 2001 From: Will Girten Date: Wed, 22 Sep 2021 22:32:21 -0400 Subject: [PATCH] Fix permitted value when Rule is an expression. --- .../databricks/labs/validation/Validator.scala | 2 +- .../labs/validation/utils/Structures.scala | 16 ---------------- .../labs/validation/ValidatorTestSuite.scala | 12 ++++++------ 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/main/scala/com/databricks/labs/validation/Validator.scala b/src/main/scala/com/databricks/labs/validation/Validator.scala index 09c2259..2b48ee1 100644 --- a/src/main/scala/com/databricks/labs/validation/Validator.scala +++ b/src/main/scala/com/databricks/labs/validation/Validator.scala @@ -48,7 +48,7 @@ class Validator(ruleSet: RuleSet, detailLvl: Int) extends SparkSessionWrapper { struct( lit(rule.ruleName).alias("ruleName"), (rule.inputColumn === rule.validExpr).alias("passed"), - lit(rule.inputColumnName).alias("permitted"), + rule.inputColumn.cast("string").alias("permitted"), rule.inputColumn.cast("string").alias("actual") ).alias(rule.ruleName) } diff --git a/src/main/scala/com/databricks/labs/validation/utils/Structures.scala b/src/main/scala/com/databricks/labs/validation/utils/Structures.scala index 113c55d..d36cc78 100644 --- a/src/main/scala/com/databricks/labs/validation/utils/Structures.scala +++ b/src/main/scala/com/databricks/labs/validation/utils/Structures.scala @@ -3,22 +3,6 @@ package com.databricks.labs.validation.utils import com.databricks.labs.validation.Rule import org.apache.spark.sql.{Column, DataFrame} -/** - * Lookups is a handy way to identify categorical values - * This is meant as an example but does should be rewritten outside of this - * As of 0.1 release it should either be Array of Int/Long/Double/String - */ -object Lookups { - final val validStoreIDs = Array(1001, 1002) - - final val validRegions = Array("Northeast", "Southeast", "Midwest", "Northwest", "Southcentral", "Southwest") - - final val validSkus = Array(123456, 122987, 123256, 173544, 163212, 365423, 168212) - - final val invalidSkus = Array(9123456, 9122987, 9123256, 9173544, 9163212, 9365423, 9168212) - -} - object Structures { case class Bounds( diff --git a/src/test/scala/com/databricks/labs/validation/ValidatorTestSuite.scala b/src/test/scala/com/databricks/labs/validation/ValidatorTestSuite.scala index 67007c7..f97e649 100644 --- a/src/test/scala/com/databricks/labs/validation/ValidatorTestSuite.scala +++ b/src/test/scala/com/databricks/labs/validation/ValidatorTestSuite.scala @@ -532,9 +532,9 @@ class ValidatorTestSuite extends AnyFunSuite with SparkSessionFixture { val expectedColumns = testDF.columns ++ Seq("TemperatureDiffExpressionRule") val expectedDF = Seq( - (1, "iot_thermostat_1", 84.00, 74.00, ValidationValue("TemperatureDiffExpressionRule", passed = true, "(abs((current_temp - target_temp)) < 50.0)", "true")), - (2, "iot_thermostat_2", 67.05, 72.00, ValidationValue("TemperatureDiffExpressionRule", passed = true, "(abs((current_temp - target_temp)) < 50.0)", "true")), - (3, "iot_thermostat_3", 91.14, 76.00, ValidationValue("TemperatureDiffExpressionRule", passed = true, "(abs((current_temp - target_temp)) < 50.0)", "true")) + (1, "iot_thermostat_1", 84.00, 74.00, ValidationValue("TemperatureDiffExpressionRule", passed = true, "true", "true")), + (2, "iot_thermostat_2", 67.05, 72.00, ValidationValue("TemperatureDiffExpressionRule", passed = true, "true", "true")), + (3, "iot_thermostat_3", 91.14, 76.00, ValidationValue("TemperatureDiffExpressionRule", passed = true, "true", "true")) ).toDF(expectedColumns: _*) val exprRuleSet = RuleSet(testDF) @@ -565,13 +565,13 @@ class ValidatorTestSuite extends AnyFunSuite with SparkSessionFixture { val expectedColumns = testDF.columns ++ Seq("ImplicitCoolingExpressionRule") val expectedDF = Seq( (1, "iot_thermostat_1", 84, 74, -10, -10, - ValidationValue("CoolingExpressionRule", passed = true, "abs(cooling_rate)", "10.0") + ValidationValue("CoolingExpressionRule", passed = true, "10.0", "10.0") ), (2, "iot_thermostat_2", 76, 66, -10, -10, - ValidationValue("CoolingExpressionRule", passed = true, "abs(cooling_rate)", "10.0") + ValidationValue("CoolingExpressionRule", passed = true, "10.0", "10.0") ), (3, "iot_thermostat_3", 91, 69, -20, -10, - ValidationValue("CoolingExpressionRule", passed = false, "abs(cooling_rate)", "10.0") + ValidationValue("CoolingExpressionRule", passed = false, "10.0", "10.0") ) ).toDF(expectedColumns: _*)