Skip to content

[SPARK-33627][SQL] Add new function UNIX_SECONDS, UNIX_MILLIS and UNIX_MICROS #30566

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gengliangwang wants to merge 9 commits into from
Closed

[SPARK-33627][SQL] Add new function UNIX_SECONDS, UNIX_MILLIS and UNIX_MICROS #30566

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
80d06c3
add new functions
gengliangwang Dec 2, 2020
0b31348
revise test case name
gengliangwang Dec 2, 2020
35ef5d2
update datetime-legacy.sql.out
gengliangwang Dec 2, 2020
45495c7
address comments
gengliangwang Dec 2, 2020
82ba97b
update
gengliangwang Dec 2, 2020
f5e4b53
fix test failure
gengliangwang Dec 2, 2020
270aa95
fix
gengliangwang Dec 2, 2020
46c8651
fix
gengliangwang Dec 2, 2020
025fdd2
address comment
gengliangwang Dec 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,9 @@ object FunctionRegistry {
expression[SecondsToTimestamp]("timestamp_seconds"),
expression[MillisToTimestamp]("timestamp_millis"),
expression[MicrosToTimestamp]("timestamp_micros"),
expression[UnixSeconds]("unix_seconds"),
expression[UnixMillis]("unix_millis"),
expression[UnixMicros]("unix_micros"),

// collection functions
expression[CreateArray]("array"),
Expand Down
73 changes: 73 additions & 0 deletions ...talyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,79 @@ case class MicrosToTimestamp(child: Expression)
override def prettyName: String = "timestamp_micros"
}

abstract class TimestampToLongBase extends UnaryExpression
with ExpectsInputTypes with NullIntolerant {

protected def scaleFactor: Long

override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)

override def dataType: DataType = LongType

override def nullSafeEval(input: Any): Any = {
Math.floorDiv(input.asInstanceOf[Number].longValue(), scaleFactor)
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
if (scaleFactor == 1) {
defineCodeGen(ctx, ev, c => c)
} else {
defineCodeGen(ctx, ev, c => s"java.lang.Math.floorDiv($c, ${scaleFactor}L)")
}
}
}

// scalastyle:off line.size.limit
@ExpressionDescription(
usage = "_FUNC_(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.",
examples = """
Examples:
> SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01'));
1
""",
group = "datetime_funcs",
since = "3.1.0")
// scalastyle:on line.size.limit
case class UnixSeconds(child: Expression) extends TimestampToLongBase {
override def scaleFactor: Long = MICROS_PER_SECOND

override def prettyName: String = "unix_seconds"
}

// scalastyle:off line.size.limit
@ExpressionDescription(
usage = "_FUNC_(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.",
examples = """
Examples:
> SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01'));
1000
""",
group = "datetime_funcs",
since = "3.1.0")
// scalastyle:on line.size.limit
case class UnixMillis(child: Expression) extends TimestampToLongBase {
override def scaleFactor: Long = MICROS_PER_MILLIS

override def prettyName: String = "unix_millis"
}

// scalastyle:off line.size.limit
@ExpressionDescription(
usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC.",
examples = """
Examples:
> SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01'));
1000000
""",
group = "datetime_funcs",
since = "3.1.0")
// scalastyle:on line.size.limit
case class UnixMicros(child: Expression) extends TimestampToLongBase {
override def scaleFactor: Long = 1L

override def prettyName: String = "unix_micros"
}

@ExpressionDescription(
usage = "_FUNC_(date) - Returns the year component of the date/timestamp.",
examples = """
Expand Down
45 changes: 45 additions & 0 deletions ...alyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1197,6 +1197,51 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkResult(Int.MinValue.toLong - 100)
}

test("UNIX_SECONDS") {
Copy link
Contributor

@cloud-fan cloud-fan Dec 2, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's also test null and negative input (that truncates)

checkEvaluation(UnixSeconds(Literal(null, TimestampType)), null)
var timestamp = Literal(new Timestamp(0L))
checkEvaluation(UnixSeconds(timestamp), 0L)
timestamp = Literal(new Timestamp(1000L))
checkEvaluation(UnixSeconds(timestamp), 1L)
timestamp = Literal(new Timestamp(-1000L))
checkEvaluation(UnixSeconds(timestamp), -1L)
// -1ms is considered to be in -1st second, as 0-999ms is in 0th second.
timestamp = Literal(new Timestamp(-1L))
checkEvaluation(UnixSeconds(timestamp), -1L)
timestamp = Literal(new Timestamp(-1000L))
checkEvaluation(UnixSeconds(timestamp), -1L)
// Truncates higher levels of precision
timestamp = Literal(new Timestamp(1999L))
checkEvaluation(UnixSeconds(timestamp), 1L)
}

test("UNIX_MILLIS") {
checkEvaluation(UnixMillis(Literal(null, TimestampType)), null)
var timestamp = Literal(new Timestamp(0L))
checkEvaluation(UnixMillis(timestamp), 0L)
timestamp = Literal(new Timestamp(1000L))
checkEvaluation(UnixMillis(timestamp), 1000L)
timestamp = Literal(new Timestamp(-1000L))
checkEvaluation(UnixMillis(timestamp), -1000L)
// Truncates higher levels of precision
val timestampWithNanos = new Timestamp(1000L)
timestampWithNanos.setNanos(999999)
checkEvaluation(UnixMillis(Literal(timestampWithNanos)), 1000L)
}

test("UNIX_MICROS") {
checkEvaluation(UnixMicros(Literal(null, TimestampType)), null)
var timestamp = Literal(new Timestamp(0L))
checkEvaluation(UnixMicros(timestamp), 0L)
timestamp = Literal(new Timestamp(1000L))
checkEvaluation(UnixMicros(timestamp), 1000000L)
timestamp = Literal(new Timestamp(-1000L))
checkEvaluation(UnixMicros(timestamp), -1000000L)
val timestampWithNanos = new Timestamp(1000L)
timestampWithNanos.setNanos(1000) // 1 microsecond
checkEvaluation(UnixMicros(Literal(timestampWithNanos)), 1000001L)
}

test("TIMESTAMP_SECONDS") {
def testIntegralFunc(value: Number): Unit = {
checkEvaluation(
Expand Down
5 changes: 4 additions & 1 deletion sql/core/src/test/resources/sql-functions/sql-expression-schema.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<!-- Automatically generated by ExpressionsSchemaSuite -->
## Summary
- Number of queries: 342
- Number of queries: 345
- Number of expressions that missing example: 13
- Expressions missing examples: bigint,binary,boolean,date,decimal,double,float,int,smallint,string,timestamp,tinyint,window
## Schema of Built-in Functions
Expand Down Expand Up @@ -289,6 +289,9 @@
| org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct<negative(1):int> |
| org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | SELECT positive(1) | struct<(+ 1):int> |
| org.apache.spark.sql.catalyst.expressions.Unhex | unhex | SELECT decode(unhex('537061726B2053514C'), 'UTF-8') | struct<decode(unhex(537061726B2053514C), UTF-8):string> |
| org.apache.spark.sql.catalyst.expressions.UnixMicros | unix_micros | SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01')) | struct<unix_micros(CAST(1970-01-01 00:00:01 AS TIMESTAMP)):bigint> |
| org.apache.spark.sql.catalyst.expressions.UnixMillis | unix_millis | SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01')) | struct<unix_millis(CAST(1970-01-01 00:00:01 AS TIMESTAMP)):bigint> |
| org.apache.spark.sql.catalyst.expressions.UnixSeconds | unix_seconds | SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01')) | struct<unix_seconds(CAST(1970-01-01 00:00:01 AS TIMESTAMP)):bigint> |
Copy link
Member

@dongjoon-hyun dongjoon-hyun Dec 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please regenerate this, @gengliangwang .

-| org.apache.spark.sql.catalyst.expressions.UnixMicros | unix_micros | SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01')) | struct<unix_micros(CAST(1970-01-01 00:00:01 AS TIMESTAMP)):bigint> |
-| org.apache.spark.sql.catalyst.expressions.UnixMillis | unix_millis | SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01')) | struct<unix_millis(CAST(1970-01-01 00:00:01 AS TIMESTAMP)):bigint> |
-| org.apache.spark.sql.catalyst.expressions.UnixSeconds | unix_seconds | SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01')) | struct<unix_seconds(CAST(1970-01-01 00:00:01 AS TIMESTAMP)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.UnixMicros | unix_micros | SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')) | struct<unix_micros(CAST(1970-01-01 00:00:01Z AS TIMESTAMP)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.UnixMillis | unix_millis | SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')) | struct<unix_millis(CAST(1970-01-01 00:00:01Z AS TIMESTAMP)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.UnixSeconds | unix_seconds | SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')) | struct<unix_seconds(CAST(1970-01-01 00:00:01Z AS TIMESTAMP)):bigint> |

Copy link
Member Author

@gengliangwang gengliangwang Dec 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, updated.

| org.apache.spark.sql.catalyst.expressions.UnixTimestamp | unix_timestamp | SELECT unix_timestamp() | struct<unix_timestamp(current_timestamp(), yyyy-MM-dd HH:mm:ss):bigint> |
| org.apache.spark.sql.catalyst.expressions.Upper | ucase | SELECT ucase('SparkSql') | struct<ucase(SparkSql):string> |
| org.apache.spark.sql.catalyst.expressions.Upper | upper | SELECT upper('SparkSql') | struct<upper(SparkSql):string> |
Expand Down
4 changes: 4 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/datetime.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ select TIMESTAMP_MILLIS(-92233720368547758);
select TIMESTAMP_SECONDS(0.1234567);
-- truncation is OK for float/double
select TIMESTAMP_SECONDS(0.1234567d), TIMESTAMP_SECONDS(FLOAT(0.1234567));
-- UNIX_SECONDS, UNIX_MILLISECONDS and UNIX_MICROSECONDS
select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_SECONDS(null);
select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null);
select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null);
Copy link
Member

@dongjoon-hyun dongjoon-hyun Dec 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add timezone into timestamp literals?

Copy link
Member Author

@gengliangwang gengliangwang Dec 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, updated.


-- [SPARK-16836] current_date and current_timestamp literals
select current_date = current_date(), current_timestamp = current_timestamp();
Expand Down
26 changes: 25 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 117
-- Number of queries: 120


-- !query
Expand Down Expand Up @@ -87,6 +87,30 @@ struct<timestamp_seconds(0.1234567):timestamp,timestamp_seconds(CAST(0.1234567 A
1969-12-31 16:00:00.123456 1969-12-31 16:00:00.123456


-- !query
select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_SECONDS(null)
-- !query schema
struct<unix_seconds(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_seconds(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_seconds(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808 1606861808 NULL


-- !query
select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null)
-- !query schema
struct<unix_millis(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_millis(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_millis(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808000 1606861808999 NULL


-- !query
select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null)
-- !query schema
struct<unix_micros(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_micros(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_micros(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808000000 1606861808999999 NULL


-- !query
select current_date = current_date(), current_timestamp = current_timestamp()
-- !query schema
Expand Down
26 changes: 25 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 117
-- Number of queries: 120


-- !query
Expand Down Expand Up @@ -87,6 +87,30 @@ struct<timestamp_seconds(0.1234567):timestamp,timestamp_seconds(CAST(0.1234567 A
1969-12-31 16:00:00.123456 1969-12-31 16:00:00.123456


-- !query
select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_SECONDS(null)
-- !query schema
struct<unix_seconds(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_seconds(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_seconds(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808 1606861808 NULL


-- !query
select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null)
-- !query schema
struct<unix_millis(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_millis(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_millis(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808000 1606861808999 NULL


-- !query
select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null)
-- !query schema
struct<unix_micros(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_micros(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_micros(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808000000 1606861808999999 NULL


-- !query
select current_date = current_date(), current_timestamp = current_timestamp()
-- !query schema
Expand Down
26 changes: 25 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/datetime.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 117
-- Number of queries: 120


-- !query
Expand Down Expand Up @@ -87,6 +87,30 @@ struct<timestamp_seconds(0.1234567):timestamp,timestamp_seconds(CAST(0.1234567 A
1969-12-31 16:00:00.123456 1969-12-31 16:00:00.123456


-- !query
select UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_SECONDS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_SECONDS(null)
-- !query schema
struct<unix_seconds(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_seconds(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_seconds(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808 1606861808 NULL


-- !query
select UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MILLIS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MILLIS(null)
-- !query schema
struct<unix_millis(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_millis(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_millis(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808000 1606861808999 NULL


-- !query
select UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08')), UNIX_MICROS(TIMESTAMP('2020-12-01 14:30:08.999999')), UNIX_MICROS(null)
-- !query schema
struct<unix_micros(CAST(2020-12-01 14:30:08 AS TIMESTAMP)):bigint,unix_micros(CAST(2020-12-01 14:30:08.999999 AS TIMESTAMP)):bigint,unix_micros(CAST(NULL AS TIMESTAMP)):bigint>
-- !query output
1606861808000000 1606861808999999 NULL


-- !query
select current_date = current_date(), current_timestamp = current_timestamp()
-- !query schema
Expand Down
3 changes: 3 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,9 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
"org.apache.spark.sql.catalyst.expressions.CurrentTimestamp",
"org.apache.spark.sql.catalyst.expressions.CurrentTimeZone",
"org.apache.spark.sql.catalyst.expressions.Now",
"org.apache.spark.sql.catalyst.expressions.UnixSeconds",
"org.apache.spark.sql.catalyst.expressions.UnixMillis",
"org.apache.spark.sql.catalyst.expressions.UnixMicros",
Copy link
Contributor

@cloud-fan cloud-fan Dec 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why can't we run the example SQL for them?

Copy link
Member Author

@gengliangwang gengliangwang Dec 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Originally, the timestamp creation in the examples will be affected by the local timezone

_FUNC_(TIMESTAMP('1970-01-01 00:00:01'))

Now I update the example and add the timezone in the timestamp literal. We can test examples.

Copy link
Member

@dongjoon-hyun dongjoon-hyun Dec 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds better.

// Random output without a seed
"org.apache.spark.sql.catalyst.expressions.Rand",
"org.apache.spark.sql.catalyst.expressions.Randn",
Expand Down