Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -430,18 +430,30 @@ object ViewHelper extends SQLConfHelper with Logging {
"spark.sql.hive.convertMetastoreCtas",
SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.key)

private val configAllowList = Seq(
private val configAllowList = Set(
SQLConf.DISABLE_HINTS.key
)

/**
* Set of single-pass resolver confs that shouldn't be stored during view/UDF/proc creation.
* This is needed to avoid accidental failures in tentative and dual-run modes when querying the
* view.
*/
private val singlePassResolverDenyList = Set(
SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED_TENTATIVELY.key,
SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER.key
)

/**
* Capture view config either of:
* 1. exists in allowList
* 2. do not exists in denyList
*/
private def shouldCaptureConfig(key: String): Boolean = {
configAllowList.exists(prefix => key.equals(prefix)) ||
!configPrefixDenyList.exists(prefix => key.startsWith(prefix))
configAllowList.contains(key) || (
!configPrefixDenyList.exists(prefix => key.startsWith(prefix)) &&
!singlePassResolverDenyList.contains(key)
)
}

import CatalogTable._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ import org.scalactic.source.Position
import org.scalatest.Tag

import org.apache.spark.sql.{AnalysisException, QueryTest}
import org.apache.spark.sql.catalyst.{ExtendedAnalysisException, QueryPlanningTracker}
import org.apache.spark.sql.catalyst.{
ExtendedAnalysisException,
QueryPlanningTracker,
TableIdentifier
}
import org.apache.spark.sql.catalyst.analysis.{
AnalysisContext,
Analyzer,
Expand Down Expand Up @@ -440,7 +444,43 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
}
}

test("Tentative mode conf is not stored during view creation when explicitly set") {
withSQLConf(SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED_TENTATIVELY.key -> "true") {
validateConfStoredInView(
conf = SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED_TENTATIVELY.key,
shouldStore = false
)
}
}

test("Dual-run mode conf is not stored during view creation when explicitly set") {
withSQLConf(SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER.key -> "true") {
validateConfStoredInView(
conf = SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER.key,
shouldStore = false
)
}
}

test("Single-pass result conf is stored during view creation when explicitly set") {
withSQLConf(SQLConf.ANALYZER_DUAL_RUN_RETURN_SINGLE_PASS_RESULT.key -> "true") {
validateConfStoredInView(
conf = SQLConf.ANALYZER_DUAL_RUN_RETURN_SINGLE_PASS_RESULT.key,
shouldStore = true
)
}
}

private def assertPlansEqual(actualPlan: LogicalPlan, expectedPlan: LogicalPlan) = {
assert(NormalizePlan(actualPlan) == NormalizePlan(expectedPlan))
}

private def validateConfStoredInView(conf: String, shouldStore: Boolean): Unit = {
withView("v1") {
sql("CREATE VIEW v1 AS SELECT 1")

val viewMetadata = spark.sessionState.catalog.getTableMetadata(TableIdentifier("v1"))
assert(viewMetadata.properties.contains(s"view.sqlConfig.$conf") == shouldStore)
}
}
}