Skip to content

Commit 55fe5cc

Browse files
committed
Refactor tests
1 parent ef298a0 commit 55fe5cc

1 file changed

Lines changed: 22 additions & 7 deletions

File tree

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1033,15 +1033,31 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
10331033
val N = 8
10341034
Seq(false, true).foreach { dictionaryEncoding =>
10351035
Seq(
1036-
("TIMESTAMP_MILLIS", "1001-01-01 01:02:03.123", "1001-01-07 01:09:05.123"),
1037-
("TIMESTAMP_MICROS", "1001-01-01 01:02:03.123456", "1001-01-07 01:09:05.123456"),
1038-
("INT96", "1001-01-01 01:02:03.123456", "1001-01-07 01:09:05.123456")
1039-
).foreach { case (outType, tsStr, nonRebased) =>
1036+
(
1037+
"TIMESTAMP_MILLIS",
1038+
"1001-01-01 01:02:03.123",
1039+
"1001-01-07 01:09:05.123",
1040+
SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key,
1041+
SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key),
1042+
(
1043+
"TIMESTAMP_MICROS",
1044+
"1001-01-01 01:02:03.123456",
1045+
"1001-01-07 01:09:05.123456",
1046+
SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key,
1047+
SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key),
1048+
(
1049+
"INT96",
1050+
"1001-01-01 01:02:03.123456",
1051+
"1001-01-07 01:09:05.123456",
1052+
SQLConf.LEGACY_PARQUET_INT96_REBASE_MODE_IN_WRITE.key,
1053+
SQLConf.LEGACY_PARQUET_INT96_REBASE_MODE_IN_READ.key
1054+
)
1055+
).foreach { case (outType, tsStr, nonRebased, inWriteConf, inReadConf) =>
10401056
withClue(s"output type $outType") {
10411057
withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
10421058
withTempPath { dir =>
10431059
val path = dir.getAbsolutePath
1044-
withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
1060+
withSQLConf(inWriteConf -> LEGACY.toString) {
10451061
Seq.tabulate(N)(_ => tsStr).toDF("tsS")
10461062
.select($"tsS".cast("timestamp").as("ts"))
10471063
.repartition(1)
@@ -1054,8 +1070,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
10541070
// The file metadata indicates if it needs rebase or not, so we can always get the
10551071
// correct result regardless of the "rebase mode" config.
10561072
Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
1057-
withSQLConf(
1058-
SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> mode.toString) {
1073+
withSQLConf(inReadConf -> mode.toString) {
10591074
checkAnswer(
10601075
spark.read.parquet(path),
10611076
Seq.tabulate(N)(_ => Row(Timestamp.valueOf(tsStr))))

0 commit comments

Comments
 (0)