From 88418e05f335db2ead7377824689774f32a449a7 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 7 Nov 2019 11:09:02 +0800 Subject: [PATCH 01/16] [SPARK-29783][SQL] Support SQL Standard output style for interval type --- .../spark/sql/catalyst/expressions/Cast.scala | 4 +- .../sql/catalyst/expressions/literals.scala | 2 + .../sql/catalyst/util/IntervalUtils.scala | 38 ++++++++ .../catalyst/util/IntervalUtilsSuite.scala | 26 ++++- .../spark/sql/execution/HiveResult.scala | 8 +- .../sql-tests/results/ansi/interval.sql.out | 70 +++++++------- .../results/postgreSQL/interval.sql.out | 96 +++++++++---------- 7 files changed, 158 insertions(+), 86 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 862b2bb515a19..b1960fd6ccaa8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.unsafe.UTF8StringBuilder -import org.apache.spark.unsafe.types.UTF8String +import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} import org.apache.spark.unsafe.types.UTF8String.{IntWrapper, LongWrapper} object Cast { @@ -280,6 +280,8 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit // UDFToString private[this] def castToString(from: DataType): Any => Any = from match { + case CalendarIntervalType if ansiEnabled => buildCast[CalendarInterval](_, + i => UTF8String.fromString(IntervalUtils.toSqlStandardString(i))) case BinaryType => buildCast[Array[Byte]](_, UTF8String.fromBytes) case DateType => buildCast[Int](_, d => UTF8String.fromString(dateFormatter.format(d))) case TimestampType => buildCast[Long](_, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 5a5d7a17acd99..960035af640c9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -409,6 +409,8 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression { DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)) s"TIMESTAMP('${formatter.format(v)}')" case (v: Array[Byte], BinaryType) => s"X'${DatatypeConverter.printHexBinary(v)}'" + case (v: CalendarInterval, CalendarIntervalType) if SQLConf.get.ansiEnabled => + IntervalUtils.toSqlStandardString(v) case _ => value.toString } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 1f63f1e14b9f5..18d3b5d5e43d5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.catalyst.util +import java.math.BigDecimal import java.util.concurrent.TimeUnit import scala.util.control.NonFatal @@ -388,4 +389,41 @@ object IntervalUtils { def divide(interval: CalendarInterval, num: Double): CalendarInterval = { fromDoubles(interval.months / num, interval.days / num, interval.microseconds / num) } + + def toSqlStandardString(interval: CalendarInterval): String = { + val yearMonthPart = if (interval.months != 0) { + interval.months / 12 + "-" + math.abs(interval.months) % 12 + } else { + "" + } + + val dayPart = if (interval.days!= 0) interval.days.toString else "" + + val timePart = if (interval.microseconds != 0) { + val sb = new StringBuilder() + var rest = interval.microseconds + sb.append(rest / MICROS_PER_HOUR) + sb.append(':') + rest = math.abs(rest % MICROS_PER_HOUR) + val minutes = rest / MICROS_PER_MINUTE; + if (minutes < 10) { + sb.append(0) + } + sb.append(minutes) + sb.append(':') + rest %= MICROS_PER_MINUTE + val db = BigDecimal.valueOf(rest, 6) + if (db.compareTo(new BigDecimal(10)) < 0) { + sb.append(0) + } + val s = db.stripTrailingZeros().toPlainString + sb.append(s) + sb.toString() + } else { + "" + } + + val intervalList = Seq(yearMonthPart, dayPart, timePart).filter(_.nonEmpty) + if (intervalList.nonEmpty) intervalList.mkString(" ") else "0" + } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala index 7015475c4b208..fc8003bc0089e 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala @@ -20,9 +20,10 @@ package org.apache.spark.sql.catalyst.util import java.util.concurrent.TimeUnit import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.catalyst.util.DateTimeUtils.{MICROS_PER_MILLIS, MICROS_PER_SECOND} +import org.apache.spark.sql.catalyst.util.DateTimeUtils._ import org.apache.spark.sql.catalyst.util.IntervalUtils._ import org.apache.spark.unsafe.types.CalendarInterval +import org.apache.spark.unsafe.types.CalendarInterval.{MICROS_PER_HOUR, MICROS_PER_MINUTE} class IntervalUtilsSuite extends SparkFunSuite { @@ -225,4 +226,27 @@ class IntervalUtilsSuite extends SparkFunSuite { assert(e.getMessage.contains("overflow")) } } + + test("to ansi sql standard string") { + val i1 = new CalendarInterval(0, 0, 0) + assert(IntervalUtils.toSqlStandardString(i1) === "0") + val i2 = new CalendarInterval(34, 0, 0) + assert(IntervalUtils.toSqlStandardString(i2) === "2-10") + val i3 = new CalendarInterval(-34, 0, 0) + assert(IntervalUtils.toSqlStandardString(i3) === "-2-10") + val i4 = new CalendarInterval(0, 31, 0) + assert(IntervalUtils.toSqlStandardString(i4) === "31") + val i5 = new CalendarInterval(0, -31, 0) + assert(IntervalUtils.toSqlStandardString(i5) === "-31") + val i6 = new CalendarInterval(0, 0, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123) + assert(IntervalUtils.toSqlStandardString(i6) === "3:13:00.000123") + val i7 = new CalendarInterval(0, 0, -3 * MICROS_PER_HOUR - 13 * MICROS_PER_MINUTE - 123) + assert(IntervalUtils.toSqlStandardString(i7) === "-3:13:00.000123") + val i8 = new CalendarInterval(-34, 31, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123) + assert(IntervalUtils.toSqlStandardString(i8) === "-2-10 31 3:13:00.000123") + val i9 = new CalendarInterval(0, 0, -3000 * MICROS_PER_HOUR) + assert(IntervalUtils.toSqlStandardString(i9) === "-3000:00:00") + + + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala index 75abac4cfd1da..d0bdb2fa552e1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala @@ -21,10 +21,11 @@ import java.nio.charset.StandardCharsets import java.sql.{Date, Timestamp} import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter} +import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, IntervalUtils, TimestampFormatter} import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.CalendarInterval /** * Runs a query returning the result in Hive compatible form. @@ -80,6 +81,7 @@ object HiveResult { private lazy val zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone) private lazy val dateFormatter = DateFormatter(zoneId) private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId) + private lazy val ansiEnabled = SQLConf.get.ansiEnabled /** Hive outputs fields of structs slightly differently than top level attributes. */ private def toHiveStructString(a: (Any, DataType)): String = a match { @@ -97,6 +99,8 @@ object HiveResult { case (null, _) => "null" case (s: String, StringType) => "\"" + s + "\"" case (decimal, DecimalType()) => decimal.toString + case (interval: CalendarInterval, CalendarIntervalType) if ansiEnabled => + IntervalUtils.toSqlStandardString(interval) case (interval, CalendarIntervalType) => interval.toString case (other, tpe) if primitiveTypes contains tpe => other.toString } @@ -120,6 +124,8 @@ object HiveResult { DateTimeUtils.timestampToString(timestampFormatter, DateTimeUtils.fromJavaTimestamp(t)) case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8) case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal) + case (interval: CalendarInterval, CalendarIntervalType) if ansiEnabled => + IntervalUtils.toSqlStandardString(interval) case (interval, CalendarIntervalType) => interval.toString case (other, _ : UserDefinedType[_]) => other.toString case (other, tpe) if primitiveTypes.contains(tpe) => other.toString diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index d0087bbcce551..ba31482503908 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -25,9 +25,9 @@ select '1' year, 2 years -- !query 1 schema -struct +struct<0:00:01:interval,0:00:02:interval,0:01:00:interval,0:02:00:interval,1:00:00:interval,2:00:00:interval,1:interval,2:interval,0-1:interval,0-2:interval,1-0:interval,2-0:interval> -- !query 1 output -interval 1 seconds interval 2 seconds interval 1 minutes interval 2 minutes interval 1 hours interval 2 hours interval 1 days interval 2 days interval 1 months interval 2 months interval 1 years interval 2 years +0:00:01 0:00:02 0:01:00 0:02:00 1:00:00 2:00:00 1 2 0-1 0-2 1-0 2-0 -- !query 2 @@ -36,9 +36,9 @@ select interval '10' year, interval '11' month -- !query 2 schema -struct +struct<10-11:interval,10-0:interval,0-11:interval> -- !query 2 output -interval 10 years 11 months interval 10 years interval 11 months +10-11 10-0 0-11 -- !query 3 @@ -47,9 +47,9 @@ select '10' year, '11' month -- !query 3 schema -struct +struct<10-11:interval,10-0:interval,0-11:interval> -- !query 3 output -interval 10 years 11 months interval 10 years interval 11 months +10-11 10-0 0-11 -- !query 4 @@ -61,9 +61,9 @@ select interval '13' second, interval '13.123456789' second -- !query 4 schema -struct +struct<10 9:08:07.987654:interval,10:interval,11:00:00:interval,0:12:00:interval,0:00:13:interval,0:00:13.123456:interval> -- !query 4 output -interval 10 days 9 hours 8 minutes 7.987654 seconds interval 10 days interval 11 hours interval 12 minutes interval 13 seconds interval 13.123456 seconds +10 9:08:07.987654 10 11:00:00 0:12:00 0:00:13 0:00:13.123456 -- !query 5 @@ -75,25 +75,25 @@ select '13' second, '13.123456789' second -- !query 5 schema -struct +struct<10 9:08:07.987654:interval,10:interval,11:00:00:interval,0:12:00:interval,0:00:13:interval,0:00:13.123456:interval> -- !query 5 output -interval 10 days 9 hours 8 minutes 7.987654 seconds interval 10 days interval 11 hours interval 12 minutes interval 13 seconds interval 13.123456 seconds +10 9:08:07.987654 10 11:00:00 0:12:00 0:00:13 0:00:13.123456 -- !query 6 select map(1, interval 1 day, 2, interval 3 week) -- !query 6 schema -struct> +struct> -- !query 6 output -{1:interval 1 days,2:interval 21 days} +{1:1,2:21} -- !query 7 select map(1, 1 day, 2, 3 week) -- !query 7 schema -struct> +struct> -- !query 7 output -{1:interval 1 days,2:interval 21 days} +{1:1,2:21} -- !query 8 @@ -118,7 +118,7 @@ select interval '2-2' year to month + dateval from interval_arithmetic -- !query 9 schema -struct +struct -- !query 9 output 2012-01-01 2009-11-01 2014-03-01 2014-03-01 2009-11-01 2009-11-01 2014-03-01 @@ -134,7 +134,7 @@ select '2-2' year to month + dateval from interval_arithmetic -- !query 10 schema -struct +struct -- !query 10 output 2012-01-01 2009-11-01 2014-03-01 2014-03-01 2009-11-01 2009-11-01 2014-03-01 @@ -150,7 +150,7 @@ select interval '2-2' year to month + tsval from interval_arithmetic -- !query 11 schema -struct +struct -- !query 11 output 2012-01-01 00:00:00 2009-11-01 00:00:00 2014-03-01 00:00:00 2014-03-01 00:00:00 2009-11-01 00:00:00 2009-11-01 00:00:00 2014-03-01 00:00:00 @@ -166,7 +166,7 @@ select '2-2' year to month + tsval from interval_arithmetic -- !query 12 schema -struct +struct -- !query 12 output 2012-01-01 00:00:00 2009-11-01 00:00:00 2014-03-01 00:00:00 2014-03-01 00:00:00 2009-11-01 00:00:00 2009-11-01 00:00:00 2014-03-01 00:00:00 @@ -177,9 +177,9 @@ select interval '2-2' year to month - interval '3-3' year to month from interval_arithmetic -- !query 13 schema -struct<(interval 2 years 2 months + interval 3 years 3 months):interval,(interval 2 years 2 months - interval 3 years 3 months):interval> +struct<(2-2 + 3-3):interval,(2-2 - 3-3):interval> -- !query 13 output -interval 5 years 5 months interval -1 years -1 months +5-5 -1-1 -- !query 14 @@ -188,9 +188,9 @@ select '2-2' year to month - '3-3' year to month from interval_arithmetic -- !query 14 schema -struct<(interval 2 years 2 months + interval 3 years 3 months):interval,(interval 2 years 2 months - interval 3 years 3 months):interval> +struct<(2-2 + 3-3):interval,(2-2 - 3-3):interval> -- !query 14 output -interval 5 years 5 months interval -1 years -1 months +5-5 -1-1 -- !query 15 @@ -204,7 +204,7 @@ select interval '99 11:22:33.123456789' day to second + dateval from interval_arithmetic -- !query 15 schema -struct +struct -- !query 15 output 2012-01-01 2011-09-23 2012-04-09 2012-04-09 2011-09-23 2011-09-23 2012-04-09 @@ -220,7 +220,7 @@ select '99 11:22:33.123456789' day to second + dateval from interval_arithmetic -- !query 16 schema -struct +struct -- !query 16 output 2012-01-01 2011-09-23 2012-04-09 2012-04-09 2011-09-23 2011-09-23 2012-04-09 @@ -236,7 +236,7 @@ select interval '99 11:22:33.123456789' day to second + tsval from interval_arithmetic -- !query 17 schema -struct +struct -- !query 17 output 2012-01-01 00:00:00 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 2012-04-09 11:22:33.123456 2011-09-23 12:37:26.876544 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 @@ -252,7 +252,7 @@ select '99 11:22:33.123456789' day to second + tsval from interval_arithmetic -- !query 18 schema -struct +struct -- !query 18 output 2012-01-01 00:00:00 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 2012-04-09 11:22:33.123456 2011-09-23 12:37:26.876544 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 @@ -263,9 +263,9 @@ select interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second from interval_arithmetic -- !query 19 schema -struct<(interval 99 days 11 hours 22 minutes 33.123456 seconds + interval 10 days 9 hours 8 minutes 7.123456 seconds):interval,(interval 99 days 11 hours 22 minutes 33.123456 seconds - interval 10 days 9 hours 8 minutes 7.123456 seconds):interval> +struct<(99 11:22:33.123456 + 10 9:08:07.123456):interval,(99 11:22:33.123456 - 10 9:08:07.123456):interval> -- !query 19 output -interval 109 days 20 hours 30 minutes 40.246912 seconds interval 89 days 2 hours 14 minutes 26 seconds +109 20:30:40.246912 89 2:14:26 -- !query 20 @@ -274,17 +274,17 @@ select '99 11:22:33.123456789' day to second - '10 9:8:7.123456789' day to second from interval_arithmetic -- !query 20 schema -struct<(interval 99 days 11 hours 22 minutes 33.123456 seconds + interval 10 days 9 hours 8 minutes 7.123456 seconds):interval,(interval 99 days 11 hours 22 minutes 33.123456 seconds - interval 10 days 9 hours 8 minutes 7.123456 seconds):interval> +struct<(99 11:22:33.123456 + 10 9:08:07.123456):interval,(99 11:22:33.123456 - 10 9:08:07.123456):interval> -- !query 20 output -interval 109 days 20 hours 30 minutes 40.246912 seconds interval 89 days 2 hours 14 minutes 26 seconds +109 20:30:40.246912 89 2:14:26 -- !query 21 select 30 day -- !query 21 schema -struct +struct<30:interval> -- !query 21 output -interval 30 days +30 -- !query 22 @@ -318,7 +318,7 @@ select 30 day day day -- !query 24 select date '2012-01-01' - 30 day -- !query 24 schema -struct +struct -- !query 24 output 2011-12-02 @@ -354,7 +354,7 @@ select date '2012-01-01' - 30 day day day -- !query 27 select date '2012-01-01' + '-30' day -- !query 27 schema -struct +struct -- !query 27 output 2011-12-02 @@ -362,7 +362,7 @@ struct +struct -- !query 28 output 2011-12-02 diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out index bd4825003aa56..836273f6d2c66 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out @@ -5,190 +5,190 @@ -- !query 0 SELECT interval '999' second -- !query 0 schema -struct +struct<0:16:39:interval> -- !query 0 output -interval 16 minutes 39 seconds +0:16:39 -- !query 1 SELECT interval '999' minute -- !query 1 schema -struct +struct<16:39:00:interval> -- !query 1 output -interval 16 hours 39 minutes +16:39:00 -- !query 2 SELECT interval '999' hour -- !query 2 schema -struct +struct<999:00:00:interval> -- !query 2 output -interval 999 hours +999:00:00 -- !query 3 SELECT interval '999' day -- !query 3 schema -struct +struct<999:interval> -- !query 3 output -interval 999 days +999 -- !query 4 SELECT interval '999' month -- !query 4 schema -struct +struct<83-3:interval> -- !query 4 output -interval 83 years 3 months +83-3 -- !query 5 SELECT interval '1' year -- !query 5 schema -struct +struct<1-0:interval> -- !query 5 output -interval 1 years +1-0 -- !query 6 SELECT interval '2' month -- !query 6 schema -struct +struct<0-2:interval> -- !query 6 output -interval 2 months +0-2 -- !query 7 SELECT interval '3' day -- !query 7 schema -struct +struct<3:interval> -- !query 7 output -interval 3 days +3 -- !query 8 SELECT interval '4' hour -- !query 8 schema -struct +struct<4:00:00:interval> -- !query 8 output -interval 4 hours +4:00:00 -- !query 9 SELECT interval '5' minute -- !query 9 schema -struct +struct<0:05:00:interval> -- !query 9 output -interval 5 minutes +0:05:00 -- !query 10 SELECT interval '6' second -- !query 10 schema -struct +struct<0:00:06:interval> -- !query 10 output -interval 6 seconds +0:00:06 -- !query 11 SELECT interval '1-2' year to month -- !query 11 schema -struct +struct<1-2:interval> -- !query 11 output -interval 1 years 2 months +1-2 -- !query 12 SELECT interval '1 2:03' day to hour -- !query 12 schema -struct +struct<1 2:00:00:interval> -- !query 12 output -interval 1 days 2 hours +1 2:00:00 -- !query 13 SELECT interval '1 2:03:04' day to hour -- !query 13 schema -struct +struct<1 2:00:00:interval> -- !query 13 output -interval 1 days 2 hours +1 2:00:00 -- !query 14 SELECT interval '1 2:03' day to minute -- !query 14 schema -struct +struct<1 2:03:00:interval> -- !query 14 output -interval 1 days 2 hours 3 minutes +1 2:03:00 -- !query 15 SELECT interval '1 2:03:04' day to minute -- !query 15 schema -struct +struct<1 2:03:00:interval> -- !query 15 output -interval 1 days 2 hours 3 minutes +1 2:03:00 -- !query 16 SELECT interval '1 2:03' day to second -- !query 16 schema -struct +struct<1 2:03:00:interval> -- !query 16 output -interval 1 days 2 hours 3 minutes +1 2:03:00 -- !query 17 SELECT interval '1 2:03:04' day to second -- !query 17 schema -struct +struct<1 2:03:04:interval> -- !query 17 output -interval 1 days 2 hours 3 minutes 4 seconds +1 2:03:04 -- !query 18 SELECT interval '1 2:03' hour to minute -- !query 18 schema -struct +struct<1 2:03:00:interval> -- !query 18 output -interval 1 days 2 hours 3 minutes +1 2:03:00 -- !query 19 SELECT interval '1 2:03:04' hour to minute -- !query 19 schema -struct +struct<1 2:03:00:interval> -- !query 19 output -interval 1 days 2 hours 3 minutes +1 2:03:00 -- !query 20 SELECT interval '1 2:03' hour to second -- !query 20 schema -struct +struct<1 2:03:00:interval> -- !query 20 output -interval 1 days 2 hours 3 minutes +1 2:03:00 -- !query 21 SELECT interval '1 2:03:04' hour to second -- !query 21 schema -struct +struct<1 2:03:04:interval> -- !query 21 output -interval 1 days 2 hours 3 minutes 4 seconds +1 2:03:04 -- !query 22 SELECT interval '1 2:03' minute to second -- !query 22 schema -struct +struct<1 0:02:03:interval> -- !query 22 output -interval 1 days 2 minutes 3 seconds +1 0:02:03 -- !query 23 SELECT interval '1 2:03:04' minute to second -- !query 23 schema -struct +struct<1 2:03:04:interval> -- !query 23 output -interval 1 days 2 hours 3 minutes 4 seconds +1 2:03:04 From fff17d5a28acda7f60f87b8dcf133a5252159516 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 7 Nov 2019 20:56:38 +0800 Subject: [PATCH 02/16] add a sep conf --- .../spark/sql/catalyst/expressions/Cast.scala | 6 +- .../sql/catalyst/expressions/literals.scala | 2 - .../apache/spark/sql/internal/SQLConf.scala | 16 ++- .../spark/sql/execution/HiveResult.scala | 8 +- .../resources/sql-tests/inputs/datetime.sql | 11 -- .../resources/sql-tests/inputs/interval.sql | 40 ++++++ .../sql-tests/results/datetime.sql.out | 58 +-------- .../sql-tests/results/interval.sql.out | 114 ++++++++++++++++++ 8 files changed, 179 insertions(+), 76 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/interval.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/interval.sql.out diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index ae9203de9590e..9e54e08b74128 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeUtils._ import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf.IntervalStyle import org.apache.spark.sql.types._ import org.apache.spark.unsafe.UTF8StringBuilder import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} @@ -281,8 +282,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit // UDFToString private[this] def castToString(from: DataType): Any => Any = from match { - case CalendarIntervalType if ansiEnabled => buildCast[CalendarInterval](_, - i => UTF8String.fromString(IntervalUtils.toSqlStandardString(i))) + case CalendarIntervalType if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD => + buildCast[CalendarInterval](_, i => + UTF8String.fromString(IntervalUtils.toSqlStandardString(i))) case BinaryType => buildCast[Array[Byte]](_, UTF8String.fromBytes) case DateType => buildCast[Int](_, d => UTF8String.fromString(dateFormatter.format(d))) case TimestampType => buildCast[Long](_, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 960035af640c9..5a5d7a17acd99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -409,8 +409,6 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression { DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)) s"TIMESTAMP('${formatter.format(v)}')" case (v: Array[Byte], BinaryType) => s"X'${DatatypeConverter.printHexBinary(v)}'" - case (v: CalendarInterval, CalendarIntervalType) if SQLConf.get.ansiEnabled => - IntervalUtils.toSqlStandardString(v) case _ => value.toString } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index a228d9f064a1e..d6293328fd4f4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -37,7 +37,6 @@ import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator import org.apache.spark.sql.catalyst.plans.logical.HintErrorHandler import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME -import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.unsafe.array.ByteArrayMethods import org.apache.spark.util.Utils @@ -1774,6 +1773,19 @@ object SQLConf { .booleanConf .createWithDefault(false) + object IntervalStyle extends Enumeration { + val SQL_STANDARD, MULTI_UNITS = Value + } + + val INTERVAL_STYLE = buildConf("spark.sql.IntervalOutputStyle") + .doc("Display format for interval values. The value SQL_STANDARD will produce output" + + " matching SQL standard interval literals. The value MULTI_UNITS (which is the default)" + + " will produce output in form of value unit pairs, i.e. '3 year 2 months 10 days'") + .stringConf + .transform(_.toUpperCase(Locale.ROOT)) + .checkValues(IntervalStyle.values.map(_.toString)) + .createWithDefault(IntervalStyle.MULTI_UNITS.toString) + val SORT_BEFORE_REPARTITION = buildConf("spark.sql.execution.sortBeforeRepartition") .internal() @@ -2502,6 +2514,8 @@ class SQLConf extends Serializable with Logging { def storeAssignmentPolicy: StoreAssignmentPolicy.Value = StoreAssignmentPolicy.withName(getConf(STORE_ASSIGNMENT_POLICY)) + def intervalOutputStyle: IntervalStyle.Value = IntervalStyle.withName(getConf(INTERVAL_STYLE)) + def ansiEnabled: Boolean = getConf(ANSI_ENABLED) def usePostgreSQLDialect: Boolean = getConf(DIALECT) == Dialect.POSTGRESQL.toString() diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala index d0bdb2fa552e1..66f673b4af12c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, IntervalUtils, TimestampFormatter} import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand} import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf.IntervalStyle import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval @@ -81,7 +82,6 @@ object HiveResult { private lazy val zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone) private lazy val dateFormatter = DateFormatter(zoneId) private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId) - private lazy val ansiEnabled = SQLConf.get.ansiEnabled /** Hive outputs fields of structs slightly differently than top level attributes. */ private def toHiveStructString(a: (Any, DataType)): String = a match { @@ -99,7 +99,8 @@ object HiveResult { case (null, _) => "null" case (s: String, StringType) => "\"" + s + "\"" case (decimal, DecimalType()) => decimal.toString - case (interval: CalendarInterval, CalendarIntervalType) if ansiEnabled => + case (interval: CalendarInterval, CalendarIntervalType) + if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD => IntervalUtils.toSqlStandardString(interval) case (interval, CalendarIntervalType) => interval.toString case (other, tpe) if primitiveTypes contains tpe => other.toString @@ -124,7 +125,8 @@ object HiveResult { DateTimeUtils.timestampToString(timestampFormatter, DateTimeUtils.fromJavaTimestamp(t)) case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8) case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal) - case (interval: CalendarInterval, CalendarIntervalType) if ansiEnabled => + case (interval: CalendarInterval, CalendarIntervalType) + if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD => IntervalUtils.toSqlStandardString(interval) case (interval, CalendarIntervalType) => interval.toString case (other, _ : UserDefinedType[_]) => other.toString diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 09eb185bd4c02..0e22af1fbdf29 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -36,14 +36,3 @@ select date '2001-10-01' - 7; select date '2001-10-01' - date '2001-09-28'; select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'; select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'; - --- interval operations -select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15'); -select interval 4 month 2 weeks 3 microseconds * 1.5; -select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5; - --- interval operation with null and zero case -select interval '2 seconds' / 0; -select interval '2 seconds' / null; -select interval '2 seconds' * null; -select null * interval '2 seconds'; diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql new file mode 100644 index 0000000000000..445428ee88440 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql @@ -0,0 +1,40 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- test for interacting with intervals + +-- using SQL_STANDARD output style +set spark.sql.IntervalOutputStyle=SQL_STANDARD; +select interval 4 month 2 weeks 3 microseconds; +select interval '1 year 20 month'; +select interval '-1 year -20 month'; +select interval '20 month 30 days -21 hours 10 minutes 999 milliseconds'; +select date'2019-10-15' - timestamp'2019-10-15 10:11:12.001002'; + +-- using MULTI_UNITS (which is default) output style +set spark.sql.IntervalOutputStyle=MULTI_UNITS; + +-- interval operations +select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15'); +select interval 4 month 2 weeks 3 microseconds * 1.5; +select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5; + +-- interval operation with null and zero case +select interval '2 seconds' / 0; +select interval '2 seconds' / null; +select interval '2 seconds' * null; +select null * interval '2 seconds'; diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index b5ea7d66fd205..a40f5acb6d538 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 24 +-- Number of queries: 17 -- !query 0 @@ -145,59 +145,3 @@ select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01' struct -- !query 16 output -2078 hours -48 minutes -47.654322 seconds - - --- !query 17 -select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15') --- !query 17 schema -struct --- !query 17 output -30 hours 33 minutes 36.003006 seconds - - --- !query 18 -select interval 4 month 2 weeks 3 microseconds * 1.5 --- !query 18 schema -struct --- !query 18 output -6 months 21 days 0.000005 seconds - - --- !query 19 -select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5 --- !query 19 schema -struct --- !query 19 output -16 hours - - --- !query 20 -select interval '2 seconds' / 0 --- !query 20 schema -struct --- !query 20 output -NULL - - --- !query 21 -select interval '2 seconds' / null --- !query 21 schema -struct --- !query 21 output -NULL - - --- !query 22 -select interval '2 seconds' * null --- !query 22 schema -struct --- !query 22 output -NULL - - --- !query 23 -select null * interval '2 seconds' --- !query 23 schema -struct --- !query 23 output -NULL diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out new file mode 100644 index 0000000000000..4d6d6ca4fec85 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -0,0 +1,114 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 14 + + +-- !query 0 +set spark.sql.IntervalOutputStyle=SQL_STANDARD +-- !query 0 schema +struct +-- !query 0 output +spark.sql.IntervalOutputStyle SQL_STANDARD + + +-- !query 1 +select interval 4 month 2 weeks 3 microseconds +-- !query 1 schema +struct<4 months 14 days 0.000003 seconds:interval> +-- !query 1 output +0-4 14 0:00:00.000003 + + +-- !query 2 +select interval '1 year 20 month' +-- !query 2 schema +struct<2 years 8 months:interval> +-- !query 2 output +2-8 + + +-- !query 3 +select interval '-1 year -20 month' +-- !query 3 schema +struct<-2 years -8 months:interval> +-- !query 3 output +-2-8 + + +-- !query 4 +select interval '20 month 30 days -21 hours 10 minutes 999 milliseconds' +-- !query 4 schema +struct<1 years 8 months 30 days -20 hours -49 minutes -59.001 seconds:interval> +-- !query 4 output +1-8 30 -20:49:59.001 + + +-- !query 5 +select date'2019-10-15' - timestamp'2019-10-15 10:11:12.001002' +-- !query 5 schema +struct +-- !query 5 output +-10:11:12.001002 + + +-- !query 6 +set spark.sql.IntervalOutputStyle=MULTI_UNITS +-- !query 6 schema +struct +-- !query 6 output +spark.sql.IntervalOutputStyle MULTI_UNITS + + +-- !query 7 +select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15') +-- !query 7 schema +struct +-- !query 7 output +30 hours 33 minutes 36.003006 seconds + + +-- !query 8 +select interval 4 month 2 weeks 3 microseconds * 1.5 +-- !query 8 schema +struct +-- !query 8 output +6 months 21 days 0.000005 seconds + + +-- !query 9 +select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5 +-- !query 9 schema +struct +-- !query 9 output +16 hours + + +-- !query 10 +select interval '2 seconds' / 0 +-- !query 10 schema +struct +-- !query 10 output +NULL + + +-- !query 11 +select interval '2 seconds' / null +-- !query 11 schema +struct +-- !query 11 output +NULL + + +-- !query 12 +select interval '2 seconds' * null +-- !query 12 schema +struct +-- !query 12 output +NULL + + +-- !query 13 +select null * interval '2 seconds' +-- !query 13 schema +struct +-- !query 13 output +NULL From b3c9e08d2e93e76217fdd1e23b87589307f80c9d Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 12 Nov 2019 00:42:38 +0800 Subject: [PATCH 03/16] add support for iso_8601 output --- .../spark/sql/catalyst/expressions/Cast.scala | 11 ++- .../sql/catalyst/util/IntervalUtils.scala | 35 ++++++++- .../apache/spark/sql/internal/SQLConf.scala | 9 ++- .../spark/sql/execution/HiveResult.scala | 21 +++-- .../resources/sql-tests/inputs/datetime.sql | 11 +++ .../resources/sql-tests/inputs/interval.sql | 37 +++++++++ .../sql-tests/inputs/postgreSQL/interval.sql | 10 ++- .../sql-tests/results/datetime.sql.out | 58 +++++++++++++- .../sql-tests/results/interval.sql.out | 77 ++++++++++++++++++- .../results/postgreSQL/interval.sql.out | 29 ++++++- 10 files changed, 273 insertions(+), 25 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 9e54e08b74128..cf702eee8013a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeUtils._ import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.IntervalStyle +import org.apache.spark.sql.internal.SQLConf.IntervalStyle._ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.UTF8StringBuilder import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} @@ -282,9 +282,14 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit // UDFToString private[this] def castToString(from: DataType): Any => Any = from match { - case CalendarIntervalType if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD => - buildCast[CalendarInterval](_, i => + case CalendarIntervalType => SQLConf.get.intervalOutputStyle match { + case SQL_STANDARD => buildCast[CalendarInterval](_, i => UTF8String.fromString(IntervalUtils.toSqlStandardString(i))) + case ISO_8601 => buildCast[CalendarInterval](_, i => + UTF8String.fromString(IntervalUtils.toIso8601String(i))) + case _ => buildCast[CalendarInterval](_, i => + UTF8String.fromString(i.toString)) + } case BinaryType => buildCast[Array[Byte]](_, UTF8String.fromBytes) case DateType => buildCast[Int](_, d => UTF8String.fromString(dateFormatter.format(d))) case TimestampType => buildCast[Long](_, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index d82a336b4895c..d4f5517e64e30 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -447,11 +447,11 @@ object IntervalUtils { sb.append(minutes) sb.append(':') rest %= MICROS_PER_MINUTE - val db = BigDecimal.valueOf(rest, 6) - if (db.compareTo(new BigDecimal(10)) < 0) { + val bd = BigDecimal.valueOf(rest, 6) + if (bd.compareTo(new BigDecimal(10)) < 0) { sb.append(0) } - val s = db.stripTrailingZeros().toPlainString + val s = bd.stripTrailingZeros().toPlainString sb.append(s) sb.toString() } else { @@ -462,6 +462,35 @@ object IntervalUtils { if (intervalList.nonEmpty) intervalList.mkString(" ") else "0" } + def toIso8601String(interval: CalendarInterval): String = { + val sb = new StringBuilder("P") + + val year = interval.months / 12 + if (year != 0) sb.append(year + "Y") + val month = interval.months % 12 + if (month != 0) sb.append(month + "M") + + if (interval.days != 0) sb.append(interval.days + "D") + + if (interval.microseconds != 0) { + sb.append('T') + var rest = interval.microseconds + val hour = rest / MICROS_PER_HOUR + if (hour != 0) sb.append(hour + "H") + rest %= MICROS_PER_HOUR + val minute = rest / MICROS_PER_MINUTE + if (minute != 0) sb.append(minute + "M") + rest %= MICROS_PER_MINUTE + if (rest != 0) { + val bd = BigDecimal.valueOf(rest, 6) + sb.append(bd.stripTrailingZeros().toPlainString + "S") + } + } else if (interval.days == 0 && interval.months == 0) { + sb.append("T0S") + } + sb.toString() + } + private object ParseState extends Enumeration { type ParseState = Value diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index d6293328fd4f4..cd7308a8161ac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1774,13 +1774,14 @@ object SQLConf { .createWithDefault(false) object IntervalStyle extends Enumeration { - val SQL_STANDARD, MULTI_UNITS = Value + val SQL_STANDARD, ISO_8601, MULTI_UNITS = Value } - val INTERVAL_STYLE = buildConf("spark.sql.IntervalOutputStyle") + val INTERVAL_STYLE = buildConf("spark.sql.intervalOutputStyle") .doc("Display format for interval values. The value SQL_STANDARD will produce output" + - " matching SQL standard interval literals. The value MULTI_UNITS (which is the default)" + - " will produce output in form of value unit pairs, i.e. '3 year 2 months 10 days'") + " matching SQL standard interval literals. The value ISO_8601 will produce output matching" + + " the ISO 8601 standard. The value MULTI_UNITS (which is the default) will produce output" + + " in form of value unit pairs, i.e. '3 year 2 months 10 days'") .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValues(IntervalStyle.values.map(_.toString)) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala index 66f673b4af12c..e2b4f0a59f1cf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, IntervalUtils, TimestampFormatter} import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.IntervalStyle +import org.apache.spark.sql.internal.SQLConf.IntervalStyle._ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval @@ -99,10 +99,12 @@ object HiveResult { case (null, _) => "null" case (s: String, StringType) => "\"" + s + "\"" case (decimal, DecimalType()) => decimal.toString - case (interval: CalendarInterval, CalendarIntervalType) - if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD => - IntervalUtils.toSqlStandardString(interval) - case (interval, CalendarIntervalType) => interval.toString + case (interval: CalendarInterval, CalendarIntervalType) => + SQLConf.get.intervalOutputStyle match { + case SQL_STANDARD => IntervalUtils.toSqlStandardString(interval) + case ISO_8601 => IntervalUtils.toIso8601String(interval) + case _ => interval.toString + } case (other, tpe) if primitiveTypes contains tpe => other.toString } @@ -125,9 +127,12 @@ object HiveResult { DateTimeUtils.timestampToString(timestampFormatter, DateTimeUtils.fromJavaTimestamp(t)) case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8) case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal) - case (interval: CalendarInterval, CalendarIntervalType) - if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD => - IntervalUtils.toSqlStandardString(interval) + case (interval: CalendarInterval, CalendarIntervalType) => + SQLConf.get.intervalOutputStyle match { + case SQL_STANDARD => IntervalUtils.toSqlStandardString(interval) + case ISO_8601 => IntervalUtils.toIso8601String(interval) + case _ => interval.toString + } case (interval, CalendarIntervalType) => interval.toString case (other, _ : UserDefinedType[_]) => other.toString case (other, tpe) if primitiveTypes.contains(tpe) => other.toString diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 0e22af1fbdf29..09eb185bd4c02 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -36,3 +36,14 @@ select date '2001-10-01' - 7; select date '2001-10-01' - date '2001-09-28'; select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'; select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'; + +-- interval operations +select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15'); +select interval 4 month 2 weeks 3 microseconds * 1.5; +select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5; + +-- interval operation with null and zero case +select interval '2 seconds' / 0; +select interval '2 seconds' / null; +select interval '2 seconds' * null; +select null * interval '2 seconds'; diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql index 148d84942a682..1406bf184aeb5 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql @@ -74,3 +74,40 @@ select cast('+ 1 second' as interval); select cast('- 1 second' as interval); select cast('- -1 second' as interval); select cast('- +1 second' as interval); + +-- interval output style +set spark.sql.intervalOutputStyle=SQL_STANDARD; +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; + +set spark.sql.intervalOutputStyle=ISO_8601; +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; + +set spark.sql.intervalOutputStyle=MULTI_UNITS; +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql index eb8cc34419519..7edcae59c49ff 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql @@ -272,10 +272,12 @@ SELECT interval '1 2:03:04' minute to second; -- test output of couple non-standard interval values in the sql style -- [SPARK-29406] Interval output styles -- SET IntervalStyle TO sql_standard; --- SELECT interval '1 day -1 hours', --- interval '-1 days +1 hours', --- interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', --- - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; +set spark.sql.intervalOutputStyle=SQL_STANDARD; +SELECT interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; +set spark.sql.intervalOutputStyle=MULTI_UNITS; -- test outputting iso8601 intervals -- [SPARK-29406] Interval output styles diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index a40f5acb6d538..b5ea7d66fd205 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 17 +-- Number of queries: 24 -- !query 0 @@ -145,3 +145,59 @@ select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01' struct -- !query 16 output -2078 hours -48 minutes -47.654322 seconds + + +-- !query 17 +select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15') +-- !query 17 schema +struct +-- !query 17 output +30 hours 33 minutes 36.003006 seconds + + +-- !query 18 +select interval 4 month 2 weeks 3 microseconds * 1.5 +-- !query 18 schema +struct +-- !query 18 output +6 months 21 days 0.000005 seconds + + +-- !query 19 +select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5 +-- !query 19 schema +struct +-- !query 19 output +16 hours + + +-- !query 20 +select interval '2 seconds' / 0 +-- !query 20 schema +struct +-- !query 20 output +NULL + + +-- !query 21 +select interval '2 seconds' / null +-- !query 21 schema +struct +-- !query 21 output +NULL + + +-- !query 22 +select interval '2 seconds' * null +-- !query 22 schema +struct +-- !query 22 output +NULL + + +-- !query 23 +select null * interval '2 seconds' +-- !query 23 schema +struct +-- !query 23 output +NULL diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 1bbeeb2085e43..d0e58c7e50274 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 45 +-- Number of queries: 51 -- !query 0 @@ -362,3 +362,78 @@ select cast('- +1 second' as interval) struct -- !query 44 output NULL + + +-- !query 45 +set spark.sql.intervalOutputStyle=SQL_STANDARD +-- !query 45 schema +struct +-- !query 45 output +spark.sql.intervalOutputStyle SQL_STANDARD + + +-- !query 46 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 46 schema +struct +-- !query 46 output +NULL 0 1-0 0-1 1-2 1 -1:00:00 -1 1:00:00 1-2 -3 4:05:06.789 -1-2 3 -4:05:06.789 + + +-- !query 47 +set spark.sql.intervalOutputStyle=ISO_8601 +-- !query 47 schema +struct +-- !query 47 output +spark.sql.intervalOutputStyle ISO_8601 + + +-- !query 48 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 48 schema +struct +-- !query 48 output +NULL PT0S P1Y P1M P1Y2M P1DT-1H P-1DT1H P1Y2M-3DT4H5M6.789S P-1Y-2M3DT-4H-5M-6.789S + + +-- !query 49 +set spark.sql.intervalOutputStyle=MULTI_UNITS +-- !query 49 schema +struct +-- !query 49 output +spark.sql.intervalOutputStyle MULTI_UNITS + + +-- !query 50 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 50 schema +struct +-- !query 50 output +NULL 0 seconds 1 years 1 months 1 years 2 months 1 days -1 hours -1 days 1 hours 1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds -1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out index 019068c9b4f96..5fdf3b1606bb2 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 24 +-- Number of queries: 27 -- !query 0 @@ -192,3 +192,30 @@ SELECT interval '1 2:03:04' minute to second struct<1 days 2 hours 3 minutes 4 seconds:interval> -- !query 23 output 1 days 2 hours 3 minutes 4 seconds + + +-- !query 24 +set spark.sql.intervalOutputStyle=SQL_STANDARD +-- !query 24 schema +struct +-- !query 24 output +spark.sql.intervalOutputStyle SQL_STANDARD + + +-- !query 25 +SELECT interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 25 schema +struct<1 days -1 hours:interval,-1 days 1 hours:interval,1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds:interval,-1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds:interval> +-- !query 25 output +1 -1:00:00 -1 1:00:00 1-2 -3 4:05:06.789 -1-2 3 -4:05:06.789 + + +-- !query 26 +set spark.sql.intervalOutputStyle=MULTI_UNITS +-- !query 26 schema +struct +-- !query 26 output +spark.sql.intervalOutputStyle MULTI_UNITS From 005a28d59cc536ad52e1de592d6479c21fda5385 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 12 Nov 2019 00:51:35 +0800 Subject: [PATCH 04/16] add ut --- .../catalyst/util/IntervalUtilsSuite.scala | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala index 9d0b070e68da0..e98463847f52e 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala @@ -287,4 +287,25 @@ class IntervalUtilsSuite extends SparkFunSuite { val i9 = new CalendarInterval(0, 0, -3000 * MICROS_PER_HOUR) assert(IntervalUtils.toSqlStandardString(i9) === "-3000:00:00") } + + test("to iso 8601 string") { + val i1 = new CalendarInterval(0, 0, 0) + assert(IntervalUtils.toIso8601String(i1) === "PT0S") + val i2 = new CalendarInterval(34, 0, 0) + assert(IntervalUtils.toIso8601String(i2) === "P2Y10M") + val i3 = new CalendarInterval(-34, 0, 0) + assert(IntervalUtils.toIso8601String(i3) === "P-2Y-10M") + val i4 = new CalendarInterval(0, 31, 0) + assert(IntervalUtils.toIso8601String(i4) === "P31D") + val i5 = new CalendarInterval(0, -31, 0) + assert(IntervalUtils.toIso8601String(i5) === "P-31D") + val i6 = new CalendarInterval(0, 0, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123) + assert(IntervalUtils.toIso8601String(i6) === "PT3H13M0.000123S") + val i7 = new CalendarInterval(0, 0, -3 * MICROS_PER_HOUR - 13 * MICROS_PER_MINUTE - 123) + assert(IntervalUtils.toIso8601String(i7) === "PT-3H-13M-0.000123S") + val i8 = new CalendarInterval(-34, 31, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123) + assert(IntervalUtils.toIso8601String(i8) === "P-2Y-10M31DT3H13M0.000123S") + val i9 = new CalendarInterval(0, 0, -3000 * MICROS_PER_HOUR) + assert(IntervalUtils.toIso8601String(i9) === "PT-3000H") + } } From 5ea0ff8045c1a4313070d6ecac755ac84d9fea55 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 12 Nov 2019 10:26:26 +0800 Subject: [PATCH 05/16] mv calendar interval toString to utils --- .../spark/unsafe/types/CalendarInterval.java | 35 +---------- .../spark/sql/catalyst/expressions/Cast.scala | 23 ++++++-- .../sql/catalyst/expressions/literals.scala | 1 + .../sql/catalyst/util/IntervalUtils.scala | 29 ++++++++++ .../catalyst/util/IntervalUtilsSuite.scala | 22 +++++++ .../spark/sql/execution/HiveResult.scala | 15 ++--- .../resources/sql-tests/inputs/datetime.sql | 11 ---- .../sql-tests/results/datetime.sql.out | 58 +------------------ 8 files changed, 79 insertions(+), 115 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java index d2abb36b0727c..0b4d25dfc0ea6 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java @@ -18,7 +18,6 @@ package org.apache.spark.unsafe.types; import java.io.Serializable; -import java.math.BigDecimal; import java.time.Duration; import java.time.Period; import java.time.temporal.ChronoUnit; @@ -80,39 +79,7 @@ public int compareTo(CalendarInterval that) { @Override public String toString() { - if (months == 0 && days == 0 && microseconds == 0) { - return "0 seconds"; - } - - StringBuilder sb = new StringBuilder(); - - if (months != 0) { - appendUnit(sb, months / 12, "years"); - appendUnit(sb, months % 12, "months"); - } - - appendUnit(sb, days, "days"); - - if (microseconds != 0) { - long rest = microseconds; - appendUnit(sb, rest / MICROS_PER_HOUR, "hours"); - rest %= MICROS_PER_HOUR; - appendUnit(sb, rest / MICROS_PER_MINUTE, "minutes"); - rest %= MICROS_PER_MINUTE; - if (rest != 0) { - String s = BigDecimal.valueOf(rest, 6).stripTrailingZeros().toPlainString(); - sb.append(s).append(" seconds "); - } - } - - sb.setLength(sb.length() - 1); - return sb.toString(); - } - - private void appendUnit(StringBuilder sb, long value, String unit) { - if (value != 0) { - sb.append(value).append(' ').append(unit).append(' '); - } + return "CalendarInterval(months= " + months + ", days = " + days + ", microsecond = " + microseconds + ")"; } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index cf702eee8013a..51fb39630e4ad 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeUtils._ +import org.apache.spark.sql.catalyst.util.IntervalUtils._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.IntervalStyle._ import org.apache.spark.sql.types._ @@ -283,12 +284,12 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit // UDFToString private[this] def castToString(from: DataType): Any => Any = from match { case CalendarIntervalType => SQLConf.get.intervalOutputStyle match { - case SQL_STANDARD => buildCast[CalendarInterval](_, i => - UTF8String.fromString(IntervalUtils.toSqlStandardString(i))) - case ISO_8601 => buildCast[CalendarInterval](_, i => - UTF8String.fromString(IntervalUtils.toIso8601String(i))) - case _ => buildCast[CalendarInterval](_, i => - UTF8String.fromString(i.toString)) + case SQL_STANDARD => + buildCast[CalendarInterval](_, i => UTF8String.fromString(toSqlStandardString(i))) + case ISO_8601 => + buildCast[CalendarInterval](_, i => UTF8String.fromString(toIso8601String(i))) + case _ => + buildCast[CalendarInterval](_, i => UTF8String.fromString(toMultiUnitsString(i))) } case BinaryType => buildCast[Array[Byte]](_, UTF8String.fromBytes) case DateType => buildCast[Int](_, d => UTF8String.fromString(dateFormatter.format(d))) @@ -994,6 +995,16 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit timestampFormatter.getClass) (c, evPrim, evNull) => code"""$evPrim = UTF8String.fromString( org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($tf, $c));""" + case CalendarIntervalType => + val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$") + SQLConf.get.intervalOutputStyle match { + case SQL_STANDARD => + (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toSqlStandardString($c));""" + case ISO_8601 => + (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toIso8601String($c));""" + case _ => + (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toMultiUnitsString($c));""" + } case ArrayType(et, _) => (c, evPrim, evNull) => { val buffer = ctx.freshVariable("buffer", classOf[UTF8StringBuilder]) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 5a5d7a17acd99..f426863de41a1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -409,6 +409,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression { DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)) s"TIMESTAMP('${formatter.format(v)}')" case (v: Array[Byte], BinaryType) => s"X'${DatatypeConverter.printHexBinary(v)}'" + case (v: CalendarInterval, CalendarIntervalType) => IntervalUtils.toMultiUnitsString(v) case _ => value.toString } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index d4f5517e64e30..fc6affc14d364 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -425,6 +425,35 @@ object IntervalUtils { fromDoubles(interval.months / num, interval.days / num, interval.microseconds / num) } + def toMultiUnitsString(interval: CalendarInterval): String = { + if (interval.months == 0 && interval.days == 0 && interval.microseconds == 0) { + return "0 seconds" + } + val sb = new StringBuilder + if (interval.months != 0) { + appendUnit(sb, interval.months / 12, "years") + appendUnit(sb, interval.months % 12, "months") + } + appendUnit(sb, interval.days, "days") + if (interval.microseconds != 0) { + var rest = interval.microseconds + appendUnit(sb, rest / MICROS_PER_HOUR, "hours") + rest %= MICROS_PER_HOUR + appendUnit(sb, rest / MICROS_PER_MINUTE, "minutes") + rest %= MICROS_PER_MINUTE + if (rest != 0) { + val s = BigDecimal.valueOf(rest, 6).stripTrailingZeros.toPlainString + sb.append(s).append(" seconds ") + } + } + sb.setLength(sb.length - 1) + sb.toString + } + + private def appendUnit(sb: StringBuilder, value: Long, unit: String): Unit = { + if (value != 0) sb.append(value).append(' ').append(unit).append(' ') + } + def toSqlStandardString(interval: CalendarInterval): String = { val yearMonthPart = if (interval.months != 0) { interval.months / 12 + "-" + math.abs(interval.months) % 12 diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala index e98463847f52e..326d45406bbfc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala @@ -308,4 +308,26 @@ class IntervalUtilsSuite extends SparkFunSuite { val i9 = new CalendarInterval(0, 0, -3000 * MICROS_PER_HOUR) assert(IntervalUtils.toIso8601String(i9) === "PT-3000H") } + + test("to multi units string") { + val i1 = new CalendarInterval(0, 0, 0) + assert(IntervalUtils.toMultiUnitsString(i1) === "0 seconds") + val i2 = new CalendarInterval(34, 0, 0) + assert(IntervalUtils.toMultiUnitsString(i2) === "2 years 10 months") + val i3 = new CalendarInterval(-34, 0, 0) + assert(IntervalUtils.toMultiUnitsString(i3) === "-2 years -10 months") + val i4 = new CalendarInterval(0, 31, 0) + assert(IntervalUtils.toMultiUnitsString(i4) === "31 days") + val i5 = new CalendarInterval(0, -31, 0) + assert(IntervalUtils.toMultiUnitsString(i5) === "-31 days") + val i6 = new CalendarInterval(0, 0, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123) + assert(IntervalUtils.toMultiUnitsString(i6) === "3 hours 13 minutes 0.000123 seconds") + val i7 = new CalendarInterval(0, 0, -3 * MICROS_PER_HOUR - 13 * MICROS_PER_MINUTE - 123) + assert(IntervalUtils.toMultiUnitsString(i7) === "-3 hours -13 minutes -0.000123 seconds") + val i8 = new CalendarInterval(-34, 31, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123) + assert(IntervalUtils.toMultiUnitsString(i8) === + "-2 years -10 months 31 days 3 hours 13 minutes 0.000123 seconds") + val i9 = new CalendarInterval(0, 0, -3000 * MICROS_PER_HOUR) + assert(IntervalUtils.toMultiUnitsString(i9) === "-3000 hours") + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala index e2b4f0a59f1cf..401ed61c2b6fa 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala @@ -21,7 +21,8 @@ import java.nio.charset.StandardCharsets import java.sql.{Date, Timestamp} import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, IntervalUtils, TimestampFormatter} +import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter} +import org.apache.spark.sql.catalyst.util.IntervalUtils._ import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.IntervalStyle._ @@ -101,9 +102,9 @@ object HiveResult { case (decimal, DecimalType()) => decimal.toString case (interval: CalendarInterval, CalendarIntervalType) => SQLConf.get.intervalOutputStyle match { - case SQL_STANDARD => IntervalUtils.toSqlStandardString(interval) - case ISO_8601 => IntervalUtils.toIso8601String(interval) - case _ => interval.toString + case SQL_STANDARD => toSqlStandardString(interval) + case ISO_8601 => toIso8601String(interval) + case _ => toMultiUnitsString(interval) } case (other, tpe) if primitiveTypes contains tpe => other.toString } @@ -129,9 +130,9 @@ object HiveResult { case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal) case (interval: CalendarInterval, CalendarIntervalType) => SQLConf.get.intervalOutputStyle match { - case SQL_STANDARD => IntervalUtils.toSqlStandardString(interval) - case ISO_8601 => IntervalUtils.toIso8601String(interval) - case _ => interval.toString + case SQL_STANDARD => toSqlStandardString(interval) + case ISO_8601 => toIso8601String(interval) + case _ => toMultiUnitsString(interval) } case (interval, CalendarIntervalType) => interval.toString case (other, _ : UserDefinedType[_]) => other.toString diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 09eb185bd4c02..0e22af1fbdf29 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -36,14 +36,3 @@ select date '2001-10-01' - 7; select date '2001-10-01' - date '2001-09-28'; select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'; select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'; - --- interval operations -select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15'); -select interval 4 month 2 weeks 3 microseconds * 1.5; -select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5; - --- interval operation with null and zero case -select interval '2 seconds' / 0; -select interval '2 seconds' / null; -select interval '2 seconds' * null; -select null * interval '2 seconds'; diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index b5ea7d66fd205..a40f5acb6d538 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 24 +-- Number of queries: 17 -- !query 0 @@ -145,59 +145,3 @@ select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01' struct -- !query 16 output -2078 hours -48 minutes -47.654322 seconds - - --- !query 17 -select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15') --- !query 17 schema -struct --- !query 17 output -30 hours 33 minutes 36.003006 seconds - - --- !query 18 -select interval 4 month 2 weeks 3 microseconds * 1.5 --- !query 18 schema -struct --- !query 18 output -6 months 21 days 0.000005 seconds - - --- !query 19 -select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5 --- !query 19 schema -struct --- !query 19 output -16 hours - - --- !query 20 -select interval '2 seconds' / 0 --- !query 20 schema -struct --- !query 20 output -NULL - - --- !query 21 -select interval '2 seconds' / null --- !query 21 schema -struct --- !query 21 output -NULL - - --- !query 22 -select interval '2 seconds' * null --- !query 22 schema -struct --- !query 22 output -NULL - - --- !query 23 -select null * interval '2 seconds' --- !query 23 schema -struct --- !query 23 output -NULL From fa9c41eb85a589c7f4f55a6e55784f617427aaba Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 12 Nov 2019 10:48:57 +0800 Subject: [PATCH 06/16] fix java style --- .../java/org/apache/spark/unsafe/types/CalendarInterval.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java index 0b4d25dfc0ea6..0464e886e1441 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java @@ -79,7 +79,8 @@ public int compareTo(CalendarInterval that) { @Override public String toString() { - return "CalendarInterval(months= " + months + ", days = " + days + ", microsecond = " + microseconds + ")"; + return "CalendarInterval(months= " + months + ", days = " + days + ", microsecond = " + + microseconds + ")"; } /** From f89e7c1d6b0763346175c32e44ceff94a45e8c03 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 12 Nov 2019 15:44:42 +0800 Subject: [PATCH 07/16] fix test case --- .../unsafe/types/CalendarIntervalSuite.java | 30 ------------------- .../sql/catalyst/json/JacksonGenerator.scala | 4 +++ .../apache/spark/sql/DateFunctionsSuite.scala | 10 +++---- 3 files changed, 9 insertions(+), 35 deletions(-) diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java index 6397f26c02f3a..01bf7eb2438ad 100644 --- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java +++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java @@ -46,36 +46,6 @@ public void equalsTest() { assertEquals(i1, i6); } - @Test - public void toStringTest() { - CalendarInterval i; - - i = new CalendarInterval(0, 0, 0); - assertEquals("0 seconds", i.toString()); - - i = new CalendarInterval(34, 0, 0); - assertEquals("2 years 10 months", i.toString()); - - i = new CalendarInterval(-34, 0, 0); - assertEquals("-2 years -10 months", i.toString()); - - i = new CalendarInterval(0, 31, 0); - assertEquals("31 days", i.toString()); - - i = new CalendarInterval(0, -31, 0); - assertEquals("-31 days", i.toString()); - - i = new CalendarInterval(0, 0, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123); - assertEquals("3 hours 13 minutes 0.000123 seconds", i.toString()); - - i = new CalendarInterval(0, 0, -3 * MICROS_PER_HOUR - 13 * MICROS_PER_MINUTE - 123); - assertEquals("-3 hours -13 minutes -0.000123 seconds", i.toString()); - - i = new CalendarInterval(34, 31, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123); - assertEquals("2 years 10 months 31 days 3 hours 13 minutes 0.000123 seconds", - i.toString()); - } - @Test public void periodAndDurationTest() { CalendarInterval interval = new CalendarInterval(120, -40, 123456); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala index aaf2ecf7923ce..e94ba53625668 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala @@ -119,6 +119,10 @@ private[sql] class JacksonGenerator( (row: SpecializedGetters, ordinal: Int) => gen.writeNumber(row.getDouble(ordinal)) + case _: CalendarIntervalType => + (row: SpecializedGetters, ordinal: Int) => + gen.writeString(IntervalUtils.toMultiUnitsString(row.getInterval(ordinal))) + case StringType => (row: SpecializedGetters, ordinal: Int) => gen.writeString(row.getUTF8String(ordinal).toString) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala index 6ea37baeaf24c..c80e675b149d0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala @@ -23,7 +23,7 @@ import java.time.Instant import java.util.Locale import java.util.concurrent.TimeUnit -import org.apache.spark.sql.catalyst.util.DateTimeUtils +import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils} import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession @@ -293,10 +293,10 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession { val i = new CalendarInterval(2, 2, 2000000L) val df = Seq((1, t1, d1), (3, t2, d2)).toDF("n", "t", "d") checkAnswer( - df.selectExpr(s"d + INTERVAL'$i'"), + df.selectExpr(s"d + INTERVAL'${IntervalUtils.toMultiUnitsString(i)}'"), Seq(Row(Date.valueOf("2015-10-02")), Row(Date.valueOf("2016-03-02")))) checkAnswer( - df.selectExpr(s"t + INTERVAL'$i'"), + df.selectExpr(s"t + INTERVAL'${IntervalUtils.toMultiUnitsString(i)}'"), Seq(Row(Timestamp.valueOf("2015-10-03 00:00:01")), Row(Timestamp.valueOf("2016-03-02 00:00:02")))) } @@ -309,10 +309,10 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession { val i = new CalendarInterval(2, 2, 2000000L) val df = Seq((1, t1, d1), (3, t2, d2)).toDF("n", "t", "d") checkAnswer( - df.selectExpr(s"d - INTERVAL'$i'"), + df.selectExpr(s"d - INTERVAL'${IntervalUtils.toMultiUnitsString(i)}'"), Seq(Row(Date.valueOf("2015-07-27")), Row(Date.valueOf("2015-12-26")))) checkAnswer( - df.selectExpr(s"t - INTERVAL'$i'"), + df.selectExpr(s"t - INTERVAL'${IntervalUtils.toMultiUnitsString(i)}'"), Seq(Row(Timestamp.valueOf("2015-07-29 23:59:59")), Row(Timestamp.valueOf("2015-12-27 00:00:00")))) } From 981eae52ca7cb064ba260a2682ca01a58b74a1c9 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 12 Nov 2019 17:31:09 +0800 Subject: [PATCH 08/16] to json support and fix toSqlStandardString bug --- .../spark/sql/catalyst/json/JSONOptions.scala | 5 ++++ .../sql/catalyst/json/JacksonGenerator.scala | 27 +++++++++++++++---- .../sql/catalyst/util/IntervalUtils.scala | 7 +++-- .../apache/spark/sql/internal/SQLConf.scala | 1 + .../apache/spark/sql/JsonFunctionsSuite.scala | 8 ++++++ 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala index 4952540f1132d..bc21b377e5aca 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala @@ -26,6 +26,8 @@ import com.fasterxml.jackson.core.{JsonFactory, JsonParser} import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf.IntervalStyle +import org.apache.spark.sql.internal.SQLConf.IntervalStyle.IntervalStyle /** * Options for parsing JSON data into Spark SQL rows. @@ -92,6 +94,9 @@ private[sql] class JSONOptions( val timestampFormat: String = parameters.getOrElse("timestampFormat", "uuuu-MM-dd'T'HH:mm:ss.SSSXXX") + val intervalOutputStyle: IntervalStyle = parameters.get("intervalOutputStyle") + .map(IntervalStyle.withName).getOrElse(SQLConf.get.intervalOutputStyle) + val multiLine = parameters.get("multiLine").map(_.toBoolean).getOrElse(false) /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala index e94ba53625668..95fede0a2e530 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala @@ -24,8 +24,8 @@ import com.fasterxml.jackson.core._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.SpecializedGetters import org.apache.spark.sql.catalyst.util._ +import org.apache.spark.sql.internal.SQLConf.IntervalStyle._ import org.apache.spark.sql.types._ - /** * `JackGenerator` can only be initialized with a `StructType`, a `MapType` or an `ArrayType`. * Once it is initialized with `StructType`, it can be used to write out a struct or an array of @@ -119,9 +119,15 @@ private[sql] class JacksonGenerator( (row: SpecializedGetters, ordinal: Int) => gen.writeNumber(row.getDouble(ordinal)) - case _: CalendarIntervalType => - (row: SpecializedGetters, ordinal: Int) => - gen.writeString(IntervalUtils.toMultiUnitsString(row.getInterval(ordinal))) + case CalendarIntervalType => + (row: SpecializedGetters, ordinal: Int) => options.intervalOutputStyle match { + case SQL_STANDARD => + gen.writeString(IntervalUtils.toSqlStandardString(row.getInterval(ordinal))) + case ISO_8601 => + gen.writeString(IntervalUtils.toIso8601String(row.getInterval(ordinal))) + case _ => + gen.writeString(IntervalUtils.toMultiUnitsString(row.getInterval(ordinal))) + } case StringType => (row: SpecializedGetters, ordinal: Int) => @@ -218,10 +224,21 @@ private[sql] class JacksonGenerator( private def writeMapData( map: MapData, mapType: MapType, fieldWriter: ValueWriter): Unit = { val keyArray = map.keyArray() + val keyString = mapType.keyType match { + case CalendarIntervalType => options.intervalOutputStyle match { + case SQL_STANDARD => + (i: Int) => IntervalUtils.toSqlStandardString(keyArray.getInterval(i)) + case ISO_8601 => + (i: Int) => IntervalUtils.toIso8601String(keyArray.getInterval(i)) + case _ => + (i: Int) => IntervalUtils.toMultiUnitsString(keyArray.getInterval(i)) + } + case _ => (i: Int) => keyArray.get(i, mapType.keyType).toString + } val valueArray = map.valueArray() var i = 0 while (i < map.numElements()) { - gen.writeFieldName(keyArray.get(i, mapType.keyType).toString) + gen.writeFieldName(keyString(i)) if (!valueArray.isNullAt(i)) { fieldWriter.apply(valueArray, i) } else { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index fc6affc14d364..dd0e28980ad00 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -455,8 +455,11 @@ object IntervalUtils { } def toSqlStandardString(interval: CalendarInterval): String = { - val yearMonthPart = if (interval.months != 0) { - interval.months / 12 + "-" + math.abs(interval.months) % 12 + val yearMonthPart = if (interval.months < 0) { + val ma = math.abs(interval.months) + "-" + ma / 12 + "-" + ma % 12 + } else if (interval.months > 0) { + interval.months / 12 + "-" + interval.months % 12 } else { "" } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index cd7308a8161ac..37af007ce15de 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1774,6 +1774,7 @@ object SQLConf { .createWithDefault(false) object IntervalStyle extends Enumeration { + type IntervalStyle = Value val SQL_STANDARD, ISO_8601, MULTI_UNITS = Value } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index c41ca925721bf..4b89cb68ccff6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -223,6 +223,14 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { checkAnswer( df.select(to_json($"c")), Row("""{"col1":{"-3 months 7 hours":"a"}}""") :: Nil) + + checkAnswer( + df.select(to_json($"c", Map("intervalOutputStyle" -> "SQL_STANDARD"))), + Row("""{"col1":{"-0-3 7:00:00":"a"}}""") :: Nil) + + checkAnswer( + df.select(to_json($"c", Map("intervalOutputStyle" -> "ISO_8601"))), + Row("""{"col1":{"P-3MT7H":"a"}}""") :: Nil) } test("to_json unsupported type") { From be65a4d3a9dfebea27456cb3878235ea461de9a1 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 12 Nov 2019 22:13:23 +0800 Subject: [PATCH 09/16] make sql standard output format explicitly --- .../spark/sql/catalyst/util/IntervalUtils.scala | 17 ++++++++++++----- .../sql/catalyst/util/IntervalUtilsSuite.scala | 8 ++++---- .../sql-tests/results/interval.sql.out | 12 ++++++++++-- .../results/postgreSQL/interval.sql.out | 2 +- .../apache/spark/sql/JsonFunctionsSuite.scala | 2 +- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index dd0e28980ad00..ecdd9fd44084f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -459,19 +459,26 @@ object IntervalUtils { val ma = math.abs(interval.months) "-" + ma / 12 + "-" + ma % 12 } else if (interval.months > 0) { - interval.months / 12 + "-" + interval.months % 12 + "+" + interval.months / 12 + "-" + interval.months % 12 } else { "" } - val dayPart = if (interval.days != 0) interval.days.toString else "" + val dayPart = if (interval.days < 0) { + interval.days.toString + } else if (interval.days > 0) { + "+" + interval.days + } else { + "" + } val timePart = if (interval.microseconds != 0) { - val sb = new StringBuilder() - var rest = interval.microseconds + val sign = if (interval.microseconds > 0) "+" else "-" + val sb = new StringBuilder(sign) + var rest = math.abs(interval.microseconds) sb.append(rest / MICROS_PER_HOUR) sb.append(':') - rest = math.abs(rest % MICROS_PER_HOUR) + rest = rest % MICROS_PER_HOUR val minutes = rest / MICROS_PER_MINUTE; if (minutes < 10) { sb.append(0) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala index 326d45406bbfc..aee1eaef3729a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala @@ -271,19 +271,19 @@ class IntervalUtilsSuite extends SparkFunSuite { val i1 = new CalendarInterval(0, 0, 0) assert(IntervalUtils.toSqlStandardString(i1) === "0") val i2 = new CalendarInterval(34, 0, 0) - assert(IntervalUtils.toSqlStandardString(i2) === "2-10") + assert(IntervalUtils.toSqlStandardString(i2) === "+2-10") val i3 = new CalendarInterval(-34, 0, 0) assert(IntervalUtils.toSqlStandardString(i3) === "-2-10") val i4 = new CalendarInterval(0, 31, 0) - assert(IntervalUtils.toSqlStandardString(i4) === "31") + assert(IntervalUtils.toSqlStandardString(i4) === "+31") val i5 = new CalendarInterval(0, -31, 0) assert(IntervalUtils.toSqlStandardString(i5) === "-31") val i6 = new CalendarInterval(0, 0, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123) - assert(IntervalUtils.toSqlStandardString(i6) === "3:13:00.000123") + assert(IntervalUtils.toSqlStandardString(i6) === "+3:13:00.000123") val i7 = new CalendarInterval(0, 0, -3 * MICROS_PER_HOUR - 13 * MICROS_PER_MINUTE - 123) assert(IntervalUtils.toSqlStandardString(i7) === "-3:13:00.000123") val i8 = new CalendarInterval(-34, 31, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123) - assert(IntervalUtils.toSqlStandardString(i8) === "-2-10 31 3:13:00.000123") + assert(IntervalUtils.toSqlStandardString(i8) === "-2-10 +31 +3:13:00.000123") val i9 = new CalendarInterval(0, 0, -3000 * MICROS_PER_HOUR) assert(IntervalUtils.toSqlStandardString(i9) === "-3000:00:00") } diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index d0e58c7e50274..31a730afe1a1d 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 51 +-- Number of queries: 52 -- !query 0 @@ -386,7 +386,7 @@ SELECT -- !query 46 schema struct -- !query 46 output -NULL 0 1-0 0-1 1-2 1 -1:00:00 -1 1:00:00 1-2 -3 4:05:06.789 -1-2 3 -4:05:06.789 +NULL 0 +1-0 +0-1 +1-2 +1 -1:00:00 -1 +1:00:00 +1-2 -3 +4:05:06.789 -1-2 +3 -4:05:06.789 -- !query 47 @@ -437,3 +437,11 @@ SELECT struct -- !query 50 output NULL 0 seconds 1 years 1 months 1 years 2 months 1 days -1 hours -1 days 1 hours 1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds -1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds + + +-- !query 51 +select interval '-1 year 1 day 1 minute' +-- !query 51 schema +struct<-1 years 1 days 1 minutes:interval> +-- !query 51 output +-1 years 1 days 1 minutes diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out index 5fdf3b1606bb2..e32e9926de8df 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out @@ -210,7 +210,7 @@ SELECT interval '1 day -1 hours', -- !query 25 schema struct<1 days -1 hours:interval,-1 days 1 hours:interval,1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds:interval,-1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds:interval> -- !query 25 output -1 -1:00:00 -1 1:00:00 1-2 -3 4:05:06.789 -1-2 3 -4:05:06.789 ++1 -1:00:00 -1 +1:00:00 +1-2 -3 +4:05:06.789 -1-2 +3 -4:05:06.789 -- !query 26 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index 4b89cb68ccff6..07dcbc62b7454 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -226,7 +226,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { checkAnswer( df.select(to_json($"c", Map("intervalOutputStyle" -> "SQL_STANDARD"))), - Row("""{"col1":{"-0-3 7:00:00":"a"}}""") :: Nil) + Row("""{"col1":{"-0-3 +7:00:00":"a"}}""") :: Nil) checkAnswer( df.select(to_json($"c", Map("intervalOutputStyle" -> "ISO_8601"))), From 748fbad55436a48cf532192344ebd36b2d230e14 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 13 Nov 2019 00:11:48 +0800 Subject: [PATCH 10/16] regen golden file --- .../resources/sql-tests/inputs/interval.sql | 6 ++++ .../sql-tests/results/interval.sql.out | 28 +++++++++---------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql index 1406bf184aeb5..4d96cafee79de 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql @@ -84,6 +84,8 @@ SELECT interval '1 month', -- month only interval '1 year 2 month', -- year month only interval '1 day -1 hours', + interval '-1 day -1 hours', + interval '-1 day 1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; @@ -95,6 +97,8 @@ SELECT interval '1 year', -- year only interval '1 month', -- month only interval '1 year 2 month', -- year month only + interval '-1 day -1 hours', + interval '-1 day 1 hours', interval '1 day -1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', @@ -107,6 +111,8 @@ SELECT interval '1 year', -- year only interval '1 month', -- month only interval '1 year 2 month', -- year month only + interval '-1 day -1 hours', + interval '-1 day 1 hours', interval '1 day -1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 31a730afe1a1d..a2f4c89ab49e6 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 52 +-- Number of queries: 51 -- !query 0 @@ -380,13 +380,15 @@ SELECT interval '1 month', -- month only interval '1 year 2 month', -- year month only interval '1 day -1 hours', + interval '-1 day -1 hours', + interval '-1 day 1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' -- !query 46 schema -struct +struct -- !query 46 output -NULL 0 +1-0 +0-1 +1-2 +1 -1:00:00 -1 +1:00:00 +1-2 -3 +4:05:06.789 -1-2 +3 -4:05:06.789 +NULL 0 +1-0 +0-1 +1-2 +1 -1:00:00 -1 -1:00:00 -1 +1:00:00 -1 +1:00:00 +1-2 -3 +4:05:06.789 -1-2 +3 -4:05:06.789 -- !query 47 @@ -404,14 +406,16 @@ SELECT interval '1 year', -- year only interval '1 month', -- month only interval '1 year 2 month', -- year month only + interval '-1 day -1 hours', + interval '-1 day 1 hours', interval '1 day -1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' -- !query 48 schema -struct +struct -- !query 48 output -NULL PT0S P1Y P1M P1Y2M P1DT-1H P-1DT1H P1Y2M-3DT4H5M6.789S P-1Y-2M3DT-4H-5M-6.789S +NULL PT0S P1Y P1M P1Y2M P-1DT-1H P-1DT1H P1DT-1H P-1DT1H P1Y2M-3DT4H5M6.789S P-1Y-2M3DT-4H-5M-6.789S -- !query 49 @@ -429,19 +433,13 @@ SELECT interval '1 year', -- year only interval '1 month', -- month only interval '1 year 2 month', -- year month only + interval '-1 day -1 hours', + interval '-1 day 1 hours', interval '1 day -1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' -- !query 50 schema -struct +struct -- !query 50 output -NULL 0 seconds 1 years 1 months 1 years 2 months 1 days -1 hours -1 days 1 hours 1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds -1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds - - --- !query 51 -select interval '-1 year 1 day 1 minute' --- !query 51 schema -struct<-1 years 1 days 1 minutes:interval> --- !query 51 output --1 years 1 days 1 minutes +NULL 0 seconds 1 years 1 months 1 years 2 months -1 days -1 hours -1 days 1 hours 1 days -1 hours -1 days 1 hours 1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds -1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds From 3aa723ca6a811898ea4f8c2d6100c7e4045b0c30 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 13 Nov 2019 10:58:38 +0800 Subject: [PATCH 11/16] rm json support / change config name / other minor revisions --- .../spark/sql/catalyst/expressions/Cast.scala | 4 ++-- .../sql/catalyst/json/JacksonGenerator.scala | 22 +++++-------------- .../sql/catalyst/util/IntervalUtils.scala | 2 +- .../apache/spark/sql/internal/SQLConf.scala | 12 +++++----- .../spark/sql/execution/HiveResult.scala | 4 ++-- .../resources/sql-tests/inputs/interval.sql | 6 ++--- .../sql-tests/inputs/postgreSQL/interval.sql | 4 ++-- .../sql-tests/results/interval.sql.out | 12 +++++----- .../results/postgreSQL/interval.sql.out | 8 +++---- .../apache/spark/sql/JsonFunctionsSuite.scala | 8 ------- 10 files changed, 32 insertions(+), 50 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 51fb39630e4ad..c214397c3ad9b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -288,7 +288,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit buildCast[CalendarInterval](_, i => UTF8String.fromString(toSqlStandardString(i))) case ISO_8601 => buildCast[CalendarInterval](_, i => UTF8String.fromString(toIso8601String(i))) - case _ => + case MULTI_UNITS => buildCast[CalendarInterval](_, i => UTF8String.fromString(toMultiUnitsString(i))) } case BinaryType => buildCast[Array[Byte]](_, UTF8String.fromBytes) @@ -1002,7 +1002,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toSqlStandardString($c));""" case ISO_8601 => (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toIso8601String($c));""" - case _ => + case MULTI_UNITS => (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toMultiUnitsString($c));""" } case ArrayType(et, _) => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala index 95fede0a2e530..544f6e9108325 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala @@ -24,8 +24,8 @@ import com.fasterxml.jackson.core._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.SpecializedGetters import org.apache.spark.sql.catalyst.util._ -import org.apache.spark.sql.internal.SQLConf.IntervalStyle._ import org.apache.spark.sql.types._ + /** * `JackGenerator` can only be initialized with a `StructType`, a `MapType` or an `ArrayType`. * Once it is initialized with `StructType`, it can be used to write out a struct or an array of @@ -120,14 +120,8 @@ private[sql] class JacksonGenerator( gen.writeNumber(row.getDouble(ordinal)) case CalendarIntervalType => - (row: SpecializedGetters, ordinal: Int) => options.intervalOutputStyle match { - case SQL_STANDARD => - gen.writeString(IntervalUtils.toSqlStandardString(row.getInterval(ordinal))) - case ISO_8601 => - gen.writeString(IntervalUtils.toIso8601String(row.getInterval(ordinal))) - case _ => - gen.writeString(IntervalUtils.toMultiUnitsString(row.getInterval(ordinal))) - } + (row: SpecializedGetters, ordinal: Int) => + gen.writeString(IntervalUtils.toMultiUnitsString(row.getInterval(ordinal))) case StringType => (row: SpecializedGetters, ordinal: Int) => @@ -225,14 +219,8 @@ private[sql] class JacksonGenerator( map: MapData, mapType: MapType, fieldWriter: ValueWriter): Unit = { val keyArray = map.keyArray() val keyString = mapType.keyType match { - case CalendarIntervalType => options.intervalOutputStyle match { - case SQL_STANDARD => - (i: Int) => IntervalUtils.toSqlStandardString(keyArray.getInterval(i)) - case ISO_8601 => - (i: Int) => IntervalUtils.toIso8601String(keyArray.getInterval(i)) - case _ => - (i: Int) => IntervalUtils.toMultiUnitsString(keyArray.getInterval(i)) - } + case CalendarIntervalType => + (i: Int) => IntervalUtils.toMultiUnitsString(keyArray.getInterval(i)) case _ => (i: Int) => keyArray.get(i, mapType.keyType).toString } val valueArray = map.valueArray() diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index ecdd9fd44084f..e4deaaa398504 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -478,7 +478,7 @@ object IntervalUtils { var rest = math.abs(interval.microseconds) sb.append(rest / MICROS_PER_HOUR) sb.append(':') - rest = rest % MICROS_PER_HOUR + rest %= MICROS_PER_HOUR val minutes = rest / MICROS_PER_MINUTE; if (minutes < 10) { sb.append(0) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 37af007ce15de..1f9f2fa4cbb92 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1778,11 +1778,13 @@ object SQLConf { val SQL_STANDARD, ISO_8601, MULTI_UNITS = Value } - val INTERVAL_STYLE = buildConf("spark.sql.intervalOutputStyle") - .doc("Display format for interval values. The value SQL_STANDARD will produce output" + - " matching SQL standard interval literals. The value ISO_8601 will produce output matching" + - " the ISO 8601 standard. The value MULTI_UNITS (which is the default) will produce output" + - " in form of value unit pairs, i.e. '3 year 2 months 10 days'") + val INTERVAL_STYLE = buildConf("spark.sql.dialect.intervalOutputStyle") + .doc("When converting interval values to strings (i.e. for display), this config decides the" + + " interval string format. The value SQL_STANDARD will produce output matching SQL standard" + + " interval literals (i.e. '+3-2 +10 -00:00:01'). The value ISO_8601 will produce output" + + " matching the ISO 8601 standard (i.e. 'P3Y2M10DT-1S'). The value MULTI_UNITS (which is the" + + " default) will produce output in form of value unit pairs, (i.e. '3 year 2 months 10 days" + + " -1 seconds'") .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValues(IntervalStyle.values.map(_.toString)) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala index 401ed61c2b6fa..d4e10b3ffc733 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala @@ -104,7 +104,7 @@ object HiveResult { SQLConf.get.intervalOutputStyle match { case SQL_STANDARD => toSqlStandardString(interval) case ISO_8601 => toIso8601String(interval) - case _ => toMultiUnitsString(interval) + case MULTI_UNITS => toMultiUnitsString(interval) } case (other, tpe) if primitiveTypes contains tpe => other.toString } @@ -132,7 +132,7 @@ object HiveResult { SQLConf.get.intervalOutputStyle match { case SQL_STANDARD => toSqlStandardString(interval) case ISO_8601 => toIso8601String(interval) - case _ => toMultiUnitsString(interval) + case MULTI_UNITS => toMultiUnitsString(interval) } case (interval, CalendarIntervalType) => interval.toString case (other, _ : UserDefinedType[_]) => other.toString diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql index 4d96cafee79de..7704a9c5adb35 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql @@ -76,7 +76,7 @@ select cast('- -1 second' as interval); select cast('- +1 second' as interval); -- interval output style -set spark.sql.intervalOutputStyle=SQL_STANDARD; +set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD; SELECT cast(null as interval), -- null interval '0 day', -- 0 @@ -90,7 +90,7 @@ SELECT interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; -set spark.sql.intervalOutputStyle=ISO_8601; +set spark.sql.dialect.intervalOutputStyle=ISO_8601; SELECT cast(null as interval), -- null interval '0 day', -- 0 @@ -104,7 +104,7 @@ SELECT interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; -set spark.sql.intervalOutputStyle=MULTI_UNITS; +set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS; SELECT cast(null as interval), -- null interval '0 day', -- 0 diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql index 7edcae59c49ff..17dc901835a02 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql @@ -272,12 +272,12 @@ SELECT interval '1 2:03:04' minute to second; -- test output of couple non-standard interval values in the sql style -- [SPARK-29406] Interval output styles -- SET IntervalStyle TO sql_standard; -set spark.sql.intervalOutputStyle=SQL_STANDARD; +set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD; SELECT interval '1 day -1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; -set spark.sql.intervalOutputStyle=MULTI_UNITS; +set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS; -- test outputting iso8601 intervals -- [SPARK-29406] Interval output styles diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index a2f4c89ab49e6..9601876b51837 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -365,11 +365,11 @@ NULL -- !query 45 -set spark.sql.intervalOutputStyle=SQL_STANDARD +set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD -- !query 45 schema struct -- !query 45 output -spark.sql.intervalOutputStyle SQL_STANDARD +spark.sql.dialect.intervalOutputStyle SQL_STANDARD -- !query 46 @@ -392,11 +392,11 @@ NULL 0 +1-0 +0-1 +1-2 +1 -1:00:00 -1 -1:00:00 -1 +1:00:00 -1 +1:00:00 +1-2 -3 +4 -- !query 47 -set spark.sql.intervalOutputStyle=ISO_8601 +set spark.sql.dialect.intervalOutputStyle=ISO_8601 -- !query 47 schema struct -- !query 47 output -spark.sql.intervalOutputStyle ISO_8601 +spark.sql.dialect.intervalOutputStyle ISO_8601 -- !query 48 @@ -419,11 +419,11 @@ NULL PT0S P1Y P1M P1Y2M P-1DT-1H P-1DT1H P1DT-1H P-1DT1H P1Y2M-3DT4H5M6.789S P-1 -- !query 49 -set spark.sql.intervalOutputStyle=MULTI_UNITS +set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS -- !query 49 schema struct -- !query 49 output -spark.sql.intervalOutputStyle MULTI_UNITS +spark.sql.dialect.intervalOutputStyle MULTI_UNITS -- !query 50 diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out index e32e9926de8df..6aa5dab1fa43f 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out @@ -195,11 +195,11 @@ struct<1 days 2 hours 3 minutes 4 seconds:interval> -- !query 24 -set spark.sql.intervalOutputStyle=SQL_STANDARD +set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD -- !query 24 schema struct -- !query 24 output -spark.sql.intervalOutputStyle SQL_STANDARD +spark.sql.dialect.intervalOutputStyle SQL_STANDARD -- !query 25 @@ -214,8 +214,8 @@ struct<1 days -1 hours:interval,-1 days 1 hours:interval,1 years 2 months -3 day -- !query 26 -set spark.sql.intervalOutputStyle=MULTI_UNITS +set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS -- !query 26 schema struct -- !query 26 output -spark.sql.intervalOutputStyle MULTI_UNITS +spark.sql.dialect.intervalOutputStyle MULTI_UNITS diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index 07dcbc62b7454..c41ca925721bf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -223,14 +223,6 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { checkAnswer( df.select(to_json($"c")), Row("""{"col1":{"-3 months 7 hours":"a"}}""") :: Nil) - - checkAnswer( - df.select(to_json($"c", Map("intervalOutputStyle" -> "SQL_STANDARD"))), - Row("""{"col1":{"-0-3 +7:00:00":"a"}}""") :: Nil) - - checkAnswer( - df.select(to_json($"c", Map("intervalOutputStyle" -> "ISO_8601"))), - Row("""{"col1":{"P-3MT7H":"a"}}""") :: Nil) } test("to_json unsupported type") { From 3cf71e9caf584096a42fc9b88c1047845f44d0c3 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 13 Nov 2019 19:48:47 +0800 Subject: [PATCH 12/16] regen golden file --- .../sql-tests/results/interval.sql.out | 83 ++++++++++++++++++- .../results/postgreSQL/interval.sql.out | 20 ++--- 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 372f7d9e0b96d..1b76cb9fa434e 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 57 +-- Number of queries: 63 -- !query 0 @@ -458,3 +458,84 @@ select justify_interval(interval '1 month 59 day -25 hour') struct -- !query 56 output 2 months 27 days 23 hours + + +-- !query 57 +set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD +-- !query 57 schema +struct +-- !query 57 output +spark.sql.dialect.intervalOutputStyle SQL_STANDARD + + +-- !query 58 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 day -1 hours', + interval '-1 day 1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 58 schema +struct +-- !query 58 output +NULL 0 +1-0 +0-1 +1-2 +1 -1:00:00 -1 -1:00:00 -1 +1:00:00 -1 +1:00:00 +1-2 -3 +4:05:06.789 -1-2 +3 -4:05:06.789 + + +-- !query 59 +set spark.sql.dialect.intervalOutputStyle=ISO_8601 +-- !query 59 schema +struct +-- !query 59 output +spark.sql.dialect.intervalOutputStyle ISO_8601 + + +-- !query 60 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '-1 day -1 hours', + interval '-1 day 1 hours', + interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 60 schema +struct +-- !query 60 output +NULL PT0S P1Y P1M P1Y2M P-1DT-1H P-1DT1H P1DT-1H P-1DT1H P1Y2M-3DT4H5M6.789S P-1Y-2M3DT-4H-5M-6.789S + + +-- !query 61 +set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS +-- !query 61 schema +struct +-- !query 61 output +spark.sql.dialect.intervalOutputStyle MULTI_UNITS + + +-- !query 62 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '-1 day -1 hours', + interval '-1 day 1 hours', + interval '1 day -1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 62 schema +struct +-- !query 62 output +NULL 0 seconds 1 years 1 months 1 years 2 months -1 days -1 hours -1 days 1 hours 1 days -1 hours -1 days 1 hours 1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds -1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out index d9e7a4836e7ee..47d6d2f797b51 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 27 +-- Number of queries: 30 -- !query 0 @@ -218,28 +218,28 @@ struct<1 days 2 hours 3 minutes 4 seconds:interval> 1 days 2 hours 3 minutes 4 seconds --- !query 24 +-- !query 27 set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD --- !query 24 schema +-- !query 27 schema struct --- !query 24 output +-- !query 27 output spark.sql.dialect.intervalOutputStyle SQL_STANDARD --- !query 25 +-- !query 28 SELECT interval '1 day -1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' --- !query 25 schema +-- !query 28 schema struct<1 days -1 hours:interval,-1 days 1 hours:interval,1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds:interval,-1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds:interval> --- !query 25 output +-- !query 28 output +1 -1:00:00 -1 +1:00:00 +1-2 -3 +4:05:06.789 -1-2 +3 -4:05:06.789 --- !query 26 +-- !query 29 set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS --- !query 26 schema +-- !query 29 schema struct --- !query 26 output +-- !query 29 output spark.sql.dialect.intervalOutputStyle MULTI_UNITS From 8d18facca29b7a9bf625a991ba9bd6e4a83829c8 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 14 Nov 2019 00:43:30 +0800 Subject: [PATCH 13/16] rm json part change and change config back --- .../apache/spark/sql/catalyst/expressions/Cast.scala | 12 +++++------- .../apache/spark/sql/catalyst/json/JSONOptions.scala | 3 --- .../org/apache/spark/sql/internal/SQLConf.scala | 2 +- .../src/test/resources/sql-tests/inputs/interval.sql | 6 +++--- .../sql-tests/inputs/postgreSQL/interval.sql | 4 ++-- .../resources/sql-tests/results/interval.sql.out | 12 ++++++------ .../sql-tests/results/postgreSQL/interval.sql.out | 8 ++++---- 7 files changed, 21 insertions(+), 26 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index c214397c3ad9b..c9b30292e18f0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -997,14 +997,12 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit org.apache.spark.sql.catalyst.util.DateTimeUtils.timestampToString($tf, $c));""" case CalendarIntervalType => val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$") - SQLConf.get.intervalOutputStyle match { - case SQL_STANDARD => - (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toSqlStandardString($c));""" - case ISO_8601 => - (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toIso8601String($c));""" - case MULTI_UNITS => - (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.toMultiUnitsString($c));""" + val funcName = SQLConf.get.intervalOutputStyle match { + case SQL_STANDARD => "toSqlStandardString" + case ISO_8601 => "toIso8601String" + case MULTI_UNITS => "toMultiUnitsString" } + (c, evPrim, _) => code"""$evPrim = UTF8String.fromString($iu.$funcName($c));""" case ArrayType(et, _) => (c, evPrim, evNull) => { val buffer = ctx.freshVariable("buffer", classOf[UTF8StringBuilder]) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala index bc21b377e5aca..6cdc9d7b1bd83 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala @@ -94,9 +94,6 @@ private[sql] class JSONOptions( val timestampFormat: String = parameters.getOrElse("timestampFormat", "uuuu-MM-dd'T'HH:mm:ss.SSSXXX") - val intervalOutputStyle: IntervalStyle = parameters.get("intervalOutputStyle") - .map(IntervalStyle.withName).getOrElse(SQLConf.get.intervalOutputStyle) - val multiLine = parameters.get("multiLine").map(_.toBoolean).getOrElse(false) /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 398e0f27c8888..8699e2e1f8f31 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1788,7 +1788,7 @@ object SQLConf { val SQL_STANDARD, ISO_8601, MULTI_UNITS = Value } - val INTERVAL_STYLE = buildConf("spark.sql.dialect.intervalOutputStyle") + val INTERVAL_STYLE = buildConf("spark.sql.intervalOutputStyle") .doc("When converting interval values to strings (i.e. for display), this config decides the" + " interval string format. The value SQL_STANDARD will produce output matching SQL standard" + " interval literals (i.e. '+3-2 +10 -00:00:01'). The value ISO_8601 will produce output" + diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql index 4dfaf8f09e65c..7692ad6002f68 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql @@ -90,7 +90,7 @@ select justify_hours(interval '1 month 59 day -25 hour'); select justify_interval(interval '1 month 59 day -25 hour'); -- interval output style -set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD; +set spark.sql.intervalOutputStyle=SQL_STANDARD; SELECT cast(null as interval), -- null interval '0 day', -- 0 @@ -104,7 +104,7 @@ SELECT interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; -set spark.sql.dialect.intervalOutputStyle=ISO_8601; +set spark.sql.intervalOutputStyle=ISO_8601; SELECT cast(null as interval), -- null interval '0 day', -- 0 @@ -118,7 +118,7 @@ SELECT interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; -set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS; +set spark.sql.intervalOutputStyle=MULTI_UNITS; SELECT cast(null as interval), -- null interval '0 day', -- 0 diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql index ae4eb4884acb0..3b25ef7334c0a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/interval.sql @@ -270,12 +270,12 @@ SELECT interval '1 2:03:04' minute to second; -- test output of couple non-standard interval values in the sql style -- [SPARK-29406] Interval output styles -- SET IntervalStyle TO sql_standard; -set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD; +set spark.sql.intervalOutputStyle=SQL_STANDARD; SELECT interval '1 day -1 hours', interval '-1 days +1 hours', interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; -set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS; +set spark.sql.intervalOutputStyle=MULTI_UNITS; -- test outputting iso8601 intervals -- [SPARK-29406] Interval output styles diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 1b76cb9fa434e..a0c6a64d38881 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -461,11 +461,11 @@ struct -- !query 57 -set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD +set spark.sql.intervalOutputStyle=SQL_STANDARD -- !query 57 schema struct -- !query 57 output -spark.sql.dialect.intervalOutputStyle SQL_STANDARD +spark.sql.intervalOutputStyle SQL_STANDARD -- !query 58 @@ -488,11 +488,11 @@ NULL 0 +1-0 +0-1 +1-2 +1 -1:00:00 -1 -1:00:00 -1 +1:00:00 -1 +1:00:00 +1-2 -3 +4 -- !query 59 -set spark.sql.dialect.intervalOutputStyle=ISO_8601 +set spark.sql.intervalOutputStyle=ISO_8601 -- !query 59 schema struct -- !query 59 output -spark.sql.dialect.intervalOutputStyle ISO_8601 +spark.sql.intervalOutputStyle ISO_8601 -- !query 60 @@ -515,11 +515,11 @@ NULL PT0S P1Y P1M P1Y2M P-1DT-1H P-1DT1H P1DT-1H P-1DT1H P1Y2M-3DT4H5M6.789S P-1 -- !query 61 -set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS +set spark.sql.intervalOutputStyle=MULTI_UNITS -- !query 61 schema struct -- !query 61 output -spark.sql.dialect.intervalOutputStyle MULTI_UNITS +spark.sql.intervalOutputStyle MULTI_UNITS -- !query 62 diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out index 47d6d2f797b51..d981ed15e37f4 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out @@ -219,11 +219,11 @@ struct<1 days 2 hours 3 minutes 4 seconds:interval> -- !query 27 -set spark.sql.dialect.intervalOutputStyle=SQL_STANDARD +set spark.sql.intervalOutputStyle=SQL_STANDARD -- !query 27 schema struct -- !query 27 output -spark.sql.dialect.intervalOutputStyle SQL_STANDARD +spark.sql.intervalOutputStyle SQL_STANDARD -- !query 28 @@ -238,8 +238,8 @@ struct<1 days -1 hours:interval,-1 days 1 hours:interval,1 years 2 months -3 day -- !query 29 -set spark.sql.dialect.intervalOutputStyle=MULTI_UNITS +set spark.sql.intervalOutputStyle=MULTI_UNITS -- !query 29 schema struct -- !query 29 output -spark.sql.dialect.intervalOutputStyle MULTI_UNITS +spark.sql.intervalOutputStyle MULTI_UNITS From 0c530cceb920b4fcd68049d94843e4c19a817a7d Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 14 Nov 2019 01:37:40 +0800 Subject: [PATCH 14/16] import --- .../scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala index 6cdc9d7b1bd83..4952540f1132d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala @@ -26,8 +26,6 @@ import com.fasterxml.jackson.core.{JsonFactory, JsonParser} import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.IntervalStyle -import org.apache.spark.sql.internal.SQLConf.IntervalStyle.IntervalStyle /** * Options for parsing JSON data into Spark SQL rows. From 7641e5e154a54307f9055a15ed7064c2a1ed6128 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 16 Nov 2019 20:30:12 +0800 Subject: [PATCH 15/16] create interval-display*.sql --- .../inputs/interval-display-iso_8601.sql | 3 +++ .../inputs/interval-display-sql_standard.sql | 3 +++ .../sql-tests/inputs/interval-display.sql | 14 +++++++++++++ .../results/interval-display-iso_8601.sql.out | 21 +++++++++++++++++++ .../interval-display-sql_standard.sql.out | 21 +++++++++++++++++++ .../results/interval-display.sql.out | 21 +++++++++++++++++++ 6 files changed, 83 insertions(+) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/interval-display-iso_8601.sql create mode 100644 sql/core/src/test/resources/sql-tests/inputs/interval-display-sql_standard.sql create mode 100644 sql/core/src/test/resources/sql-tests/inputs/interval-display.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/interval-display-iso_8601.sql.out create mode 100644 sql/core/src/test/resources/sql-tests/results/interval-display-sql_standard.sql.out create mode 100644 sql/core/src/test/resources/sql-tests/results/interval-display.sql.out diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval-display-iso_8601.sql b/sql/core/src/test/resources/sql-tests/inputs/interval-display-iso_8601.sql new file mode 100644 index 0000000000000..62f3f43bd2c48 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/interval-display-iso_8601.sql @@ -0,0 +1,3 @@ +-- tests for interval output style with iso_8601 format +--SET spark.sql.intervalOutputStyle = ISO_8601 +--import interval-display.sql diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval-display-sql_standard.sql b/sql/core/src/test/resources/sql-tests/inputs/interval-display-sql_standard.sql new file mode 100644 index 0000000000000..375b4899e760e --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/interval-display-sql_standard.sql @@ -0,0 +1,3 @@ +-- tests for interval output style with sql standard format +--SET spark.sql.intervalOutputStyle = SQL_STANDARD +--import interval-display.sql diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval-display.sql b/sql/core/src/test/resources/sql-tests/inputs/interval-display.sql new file mode 100644 index 0000000000000..ae19f1b6374ba --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/interval-display.sql @@ -0,0 +1,14 @@ +-- tests for interval output style + +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 day -1 hours', + interval '-1 day 1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds'; diff --git a/sql/core/src/test/resources/sql-tests/results/interval-display-iso_8601.sql.out b/sql/core/src/test/resources/sql-tests/results/interval-display-iso_8601.sql.out new file mode 100644 index 0000000000000..57fe8a3f4fcc6 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/interval-display-iso_8601.sql.out @@ -0,0 +1,21 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 1 + + +-- !query 0 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 day -1 hours', + interval '-1 day 1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 0 schema +struct +-- !query 0 output +NULL PT0S P1Y P1M P1Y2M P1DT-1H P-1DT-1H P-1DT1H P-1DT1H P1Y2M-3DT4H5M6.789S P-1Y-2M3DT-4H-5M-6.789S diff --git a/sql/core/src/test/resources/sql-tests/results/interval-display-sql_standard.sql.out b/sql/core/src/test/resources/sql-tests/results/interval-display-sql_standard.sql.out new file mode 100644 index 0000000000000..9e40f52151475 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/interval-display-sql_standard.sql.out @@ -0,0 +1,21 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 1 + + +-- !query 0 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 day -1 hours', + interval '-1 day 1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 0 schema +struct +-- !query 0 output +NULL 0 +1-0 +0-1 +1-2 +1 -1:00:00 -1 -1:00:00 -1 +1:00:00 -1 +1:00:00 +1-2 -3 +4:05:06.789 -1-2 +3 -4:05:06.789 diff --git a/sql/core/src/test/resources/sql-tests/results/interval-display.sql.out b/sql/core/src/test/resources/sql-tests/results/interval-display.sql.out new file mode 100644 index 0000000000000..340496e404326 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/interval-display.sql.out @@ -0,0 +1,21 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 1 + + +-- !query 0 +SELECT + cast(null as interval), -- null + interval '0 day', -- 0 + interval '1 year', -- year only + interval '1 month', -- month only + interval '1 year 2 month', -- year month only + interval '1 day -1 hours', + interval '-1 day -1 hours', + interval '-1 day 1 hours', + interval '-1 days +1 hours', + interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds', + - interval '1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds' +-- !query 0 schema +struct +-- !query 0 output +NULL 0 seconds 1 years 1 months 1 years 2 months 1 days -1 hours -1 days -1 hours -1 days 1 hours -1 days 1 hours 1 years 2 months -3 days 4 hours 5 minutes 6.789 seconds -1 years -2 months 3 days -4 hours -5 minutes -6.789 seconds From 0f54af89b8641ee7df80991dcca63e183a03d5ff Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 16 Nov 2019 23:44:57 +0800 Subject: [PATCH 16/16] blacklisting new tests in thrift test --- .../sql/hive/thriftserver/ThriftServerQueryTestSuite.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala index 82da4c049fd41..b50a6045f5378 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala @@ -93,7 +93,10 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite { "subquery/in-subquery/in-group-by.sql", "subquery/in-subquery/simple-in.sql", "subquery/in-subquery/in-order-by.sql", - "subquery/in-subquery/in-set-operations.sql" + "subquery/in-subquery/in-set-operations.sql", + // SPARK-29783: need to set conf + "interval-display-iso_8601.sql", + "interval-display-sql_standard.sql" ) override def runQueries(