diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 548941f7a0..342bca788a 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -95,12 +95,12 @@ incompatible expressions. ## Date/Time Functions | Expression | SQL | Spark-Compatible? | Compatibility Notes | -| -------------- | ---------------------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------- | +| -------------- | ---------------------------- | ----------------- |----------------------------------------------------------------------------------------------------------------------| | DateAdd | `date_add` | Yes | | | DateSub | `date_sub` | Yes | | | DatePart | `date_part(field, source)` | Yes | Supported values of `field`: `year`/`month`/`week`/`day`/`dayofweek`/`dayofweek_iso`/`doy`/`quarter`/`hour`/`minute` | | Extract | `extract(field FROM source)` | Yes | Supported values of `field`: `year`/`month`/`week`/`day`/`dayofweek`/`dayofweek_iso`/`doy`/`quarter`/`hour`/`minute` | -| FromUnixTime | `from_unixtime` | No | Does not support format, supports only -8334601211038 <= sec <= 8210266876799 | +| FromUnixTime | `from_unixtime` | No | Supports only -8334601211038 <= sec <= 8210266876799 | | Hour | `hour` | Yes | | | Minute | `minute` | Yes | | | Second | `second` | Yes | | diff --git a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala index 198c7d3101..ed3204e954 100644 --- a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala @@ -20,7 +20,6 @@ package org.apache.comet.serde import org.apache.spark.sql.catalyst.expressions.{Attribute, FromUnixTime, Literal} -import org.apache.spark.sql.catalyst.util.TimestampFormatter import org.apache.comet.CometSparkSessionExtensions.withInfo import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto} @@ -36,17 +35,10 @@ object CometFromUnixTime extends CometExpressionSerde[FromUnixTime] { inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { val secExpr = exprToProtoInternal(expr.sec, inputs, binding) - // TODO: DataFusion toChar does not support Spark datetime pattern format - // https://github.com/apache/datafusion/issues/16577 - // https://github.com/apache/datafusion/issues/14536 - // After fixing these issues, use provided `format` instead of the manual replacement below - val formatExpr = exprToProtoInternal(Literal("%Y-%m-%d %H:%M:%S"), inputs, binding) + val formatExpr = exprToProtoInternal(expr.format, inputs, binding) val timeZone = exprToProtoInternal(Literal(expr.timeZoneId.orNull), inputs, binding) - if (expr.format != Literal(TimestampFormatter.defaultPattern)) { - withInfo(expr, "Datetime pattern format is unsupported") - None - } else if (secExpr.isDefined && formatExpr.isDefined) { + if (secExpr.isDefined && formatExpr.isDefined) { val timestampExpr = scalarFunctionExprToProto("from_unixtime", Seq(secExpr, timeZone): _*) val optExpr = scalarFunctionExprToProto("to_char", Seq(timestampExpr, formatExpr): _*) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index fc45d2cb3a..1c6ec692ec 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1769,17 +1769,13 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { val where = "where _5 BETWEEN -8334601211038 AND 8210266876799" checkSparkAnswerAndOperator(s"SELECT from_unixtime(_5) FROM $table $where") checkSparkAnswerAndOperator(s"SELECT from_unixtime(_8) FROM $table $where") - // TODO: DataFusion toChar does not support Spark datetime pattern format - // https://github.com/apache/datafusion/issues/16577 - // https://github.com/apache/datafusion/issues/14536 - // After fixing these issues, change checkSparkAnswer to checkSparkAnswerAndOperator - checkSparkAnswer(s"SELECT from_unixtime(_5, 'yyyy') FROM $table $where") - checkSparkAnswer(s"SELECT from_unixtime(_8, 'yyyy') FROM $table $where") + checkSparkAnswerAndOperator(s"SELECT from_unixtime(_5, 'yyyy') FROM $table $where") + checkSparkAnswerAndOperator(s"SELECT from_unixtime(_8, 'yyyy') FROM $table $where") withSQLConf(SESSION_LOCAL_TIMEZONE.key -> "Asia/Kathmandu") { checkSparkAnswerAndOperator(s"SELECT from_unixtime(_5) FROM $table $where") checkSparkAnswerAndOperator(s"SELECT from_unixtime(_8) FROM $table $where") - checkSparkAnswer(s"SELECT from_unixtime(_5, 'yyyy') FROM $table $where") - checkSparkAnswer(s"SELECT from_unixtime(_8, 'yyyy') FROM $table $where") + checkSparkAnswerAndOperator(s"SELECT from_unixtime(_5, 'yyyy') FROM $table $where") + checkSparkAnswerAndOperator(s"SELECT from_unixtime(_8, 'yyyy') FROM $table $where") } } }