Skip to content

Commit 141cb1d

Browse files
committed
Add more test.
1 parent 2bb8368 commit 141cb1d

1 file changed

Lines changed: 20 additions & 3 deletions

File tree

sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration
2424
import org.apache.hadoop.hive.ql.io.orc.{OrcStruct, SparkOrcNewRecordReader}
2525
import org.scalatest.BeforeAndAfterAll
2626

27+
import org.apache.spark.SparkException
2728
import org.apache.spark.sql._
2829
import org.apache.spark.sql.catalyst.TableIdentifier
2930
import org.apache.spark.sql.execution.datasources.{LogicalRelation, RecordReaderIterator}
@@ -596,7 +597,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest {
596597
singleRowDF.createOrReplaceTempView("single")
597598

598599
withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true") {
599-
withTable("dummy_orc") {
600+
withTable("dummy_orc", "dummy_orc2", "dummy_orc3") {
600601
withTempPath { dir =>
601602
val path = dir.getCanonicalPath
602603

@@ -619,7 +620,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest {
619620
val df = spark.sql("SELECT key, value FROM dummy_orc WHERE key=0")
620621
checkAnswer(df, singleRowDF)
621622

622-
// Create a Metastore ORC table with different schema.
623+
// Create a Metastore ORC table with the schema of different column names.
623624
spark.sql(
624625
s"""
625626
|CREATE EXTERNAL TABLE dummy_orc2(value2 STRING)
@@ -630,7 +631,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest {
630631

631632
spark.sql("ALTER TABLE dummy_orc2 ADD PARTITION(key=0)")
632633

633-
// The output of the relation is the schema from the Metastore, not the file.
634+
// The output of the relation is the schema from the Metastore, not from the orc file.
634635
val df2 = spark.sql("SELECT key, value2 FROM dummy_orc2 WHERE key=0 AND value2='foo'")
635636
checkAnswer(df2, singleRowDF)
636637

@@ -641,6 +642,22 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest {
641642
fail(s"Expecting the query plan to convert orc to data sources, " +
642643
s"but got:\n$queryExecution")
643644
}
645+
646+
// When the column types of Orc files are not matching with metastore schema,
647+
// we can't convert Hive metastore Orc table to datasource table.
648+
spark.sql(
649+
s"""
650+
|CREATE EXTERNAL TABLE dummy_orc3(value2 INT)
651+
|PARTITIONED BY (key INT)
652+
|STORED AS ORC
653+
|LOCATION '$path'
654+
""".stripMargin)
655+
656+
spark.sql("ALTER TABLE dummy_orc3 ADD PARTITION(key=0)")
657+
val errorMessage = intercept[SparkException] {
658+
spark.sql("SELECT key, value2 FROM dummy_orc3 WHERE key=0 AND value2=1").count()
659+
}.getMessage
660+
assert(errorMessage.contains("please disable spark.sql.hive.convertMetastoreOrc"))
644661
}
645662
}
646663
}

0 commit comments

Comments
 (0)