Skip to content

[SPARK-33721][SQL] Support to use Hive build-in functions by configuration #30686

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
southernriver wants to merge 1 commit into from
Closed

[SPARK-33721][SQL] Support to use Hive build-in functions by configuration #30686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2973,6 +2973,24 @@ object SQLConf {
.booleanConf
.createWithDefault(false)

val USE_HIVE_BUILD_IN_FUNCTIONS_ENABLED =
buildConf("spark.sql.hive.buildin.functions.enabled")
.internal()
.doc("When true, Spark will register hive build-in functions like unix_timestamp,to_date," +
" datediff,collect_set instead of self build-in functions.")
.booleanConf
.createWithDefault(false)

val HIVE_BUILD_IN_FUNCTIONS_LIST =
buildConf("spark.sql.hive.buildin.functions.list")
.internal()
.doc("Configures a list of hive build-in functions, here is an example for the format: " +
" org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp:unix_timestamp;" +
" org.apache.hadoop.hive.ql.udf.generic.GenericUDFDate:to_date. This configuration only " +
s" has an effect when '${USE_HIVE_BUILD_IN_FUNCTIONS_ENABLED.key}' is set to true.")
.stringConf
.createWithDefault("")

/**
* Holds information about keys that have been deprecated.
*
Expand Down Expand Up @@ -3627,6 +3645,10 @@ class SQLConf extends Serializable with Logging {

def charVarcharAsString: Boolean = getConf(SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING)

def useHiveBuildInFunctionsEnabled: Boolean = getConf(SQLConf.USE_HIVE_BUILD_IN_FUNCTIONS_ENABLED)

def hiveBuildINFunctionsList: String = getConf(SQLConf.HIVE_BUILD_IN_FUNCTIONS_LIST)

/** ********************** SQLConf functionality methods ************ */

/** Set Spark SQL configuration properties. */
Expand Down
29 changes: 28 additions & 1 deletion sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ package org.apache.spark.sql.hive

import org.apache.spark.annotation.Unstable
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.analysis.{Analyzer, ResolveSessionCatalog}
import org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener
import org.apache.spark.sql.catalyst.catalog.{CatalogFunction, ExternalCatalogWithListener}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution.SparkPlanner
Expand Down Expand Up @@ -66,6 +67,32 @@ class HiveSessionStateBuilder(
sqlParser,
resourceLoader)
parentState.foreach(_.catalog.copyStateTo(catalog))

// Use hive build-in functions first.
if (conf.useHiveBuildInFunctionsEnabled) {
catalog.registerFunction(CatalogFunction(FunctionIdentifier("unix_timestamp", None),
"org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp", Seq.empty), true)
catalog.registerFunction(CatalogFunction(FunctionIdentifier("to_date", None),
"org.apache.hadoop.hive.ql.udf.generic.GenericUDFDate", Seq.empty), true)
catalog.registerFunction(CatalogFunction(FunctionIdentifier("collect_set", None),
"org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCollectSet", Seq.empty), true)
catalog.registerFunction(CatalogFunction(FunctionIdentifier("datediff", None),
"org.apache.hadoop.hive.ql.udf.generic.GenericUDFDateDiff", Seq.empty), true)

// Support to add additional hive build-in functions through configuration
val hiveBuildINFunctionsList = conf.hiveBuildINFunctionsList
if (!hiveBuildINFunctionsList.isEmpty) {
hiveBuildINFunctionsList.split(";").foreach(oneUdf => {
val parts = oneUdf.split(":")
if (parts.length == 2) {
catalog.registerFunction(CatalogFunction(FunctionIdentifier(parts(0), None),
parts(1), Seq.empty), true)
}
})
}
}


catalog
}

Expand Down