Skip to content

Commit 119fa64

Browse files
committed
modify some test case code
1 parent b4caca7 commit 119fa64

2 files changed

Lines changed: 20 additions & 17 deletions

File tree

sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ class HadoopTableReader(
115115
val inputPathStr = applyFilterIfNeeded(tablePath, filterOpt)
116116

117117
val locationPath = new Path(inputPathStr)
118-
val fs = locationPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
118+
val fs = locationPath.getFileSystem(broadcastedHadoopConf.value.value)
119119

120120
// if the table location does not exist, return an empty RDD
121121
if (!fs.exists(locationPath)) {

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1499,14 +1499,15 @@ class HiveDDLSuite
14991499
test("insert data to a hive serde table which has a not existed location should succeed") {
15001500
withTable("t") {
15011501
withTempDir { dir =>
1502+
val dirPath = dir.getAbsolutePath.stripSuffix("/")
15021503
spark.sql(
15031504
s"""
15041505
|CREATE TABLE t(a string, b int)
15051506
|USING hive
1506-
|OPTIONS(path "file:${dir.getCanonicalPath}")
1507+
|OPTIONS(path "file:$dirPath")
15071508
""".stripMargin)
15081509
val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
1509-
val expectedPath = s"file:${dir.getAbsolutePath.stripSuffix("/")}"
1510+
val expectedPath = s"file:$dirPath"
15101511
assert(table.location.stripSuffix("/") == expectedPath)
15111512

15121513
val tableLocFile = new File(table.location.stripPrefix("file:"))
@@ -1523,11 +1524,12 @@ class HiveDDLSuite
15231524
checkAnswer(spark.table("t"), Row("c", 1) :: Nil)
15241525

15251526
val newDirFile = new File(dir, "x")
1526-
spark.sql(s"ALTER TABLE t SET LOCATION '${newDirFile.getAbsolutePath}'")
1527+
val newDirPath = newDirFile.getAbsolutePath.stripSuffix("/")
1528+
spark.sql(s"ALTER TABLE t SET LOCATION '$newDirPath'")
15271529
spark.sessionState.catalog.refreshTable(TableIdentifier("t"))
15281530

15291531
val table1 = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
1530-
assert(table1.location.stripSuffix("/") == newDirFile.getAbsolutePath.stripSuffix("/"))
1532+
assert(table1.location.stripSuffix("/") == newDirPath)
15311533
assert(!newDirFile.exists())
15321534

15331535
spark.sql("INSERT INTO TABLE t SELECT 'c', 1")
@@ -1540,21 +1542,22 @@ class HiveDDLSuite
15401542
test("insert into a hive serde table with no existed partition location should succeed") {
15411543
withTable("t") {
15421544
withTempDir { dir =>
1545+
val dirPath = dir.getAbsolutePath.stripSuffix("/")
15431546
spark.sql(
15441547
s"""
15451548
|CREATE TABLE t(a int, b int, c int, d int)
15461549
|USING hive
15471550
|PARTITIONED BY(a, b)
1548-
|LOCATION "file:${dir.getCanonicalPath}"
1551+
|LOCATION "file:$dirPath"
15491552
""".stripMargin)
15501553
val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
1551-
val expectedPath = s"file:${dir.getAbsolutePath.stripSuffix("/")}"
1554+
val expectedPath = s"file:$dirPath"
15521555
assert(table.location.stripSuffix("/") == expectedPath)
15531556

15541557
spark.sql("INSERT INTO TABLE t PARTITION(a=1, b=2) SELECT 3, 4")
15551558
checkAnswer(spark.table("t"), Row(3, 4, 1, 2) :: Nil)
15561559

1557-
val partLoc = new File(s"${dir.getAbsolutePath}/a=1")
1560+
val partLoc = new File(s"$dirPath/a=1")
15581561
Utils.deleteRecursively(partLoc)
15591562
assert(!partLoc.exists())
15601563
// insert overwrite into a partition which location has been deleted.
@@ -1563,8 +1566,8 @@ class HiveDDLSuite
15631566
checkAnswer(spark.table("t"), Row(7, 8, 1, 2) :: Nil)
15641567

15651568
val newDirFile = new File(dir, "x")
1566-
spark.sql(s"ALTER TABLE t PARTITION(a=1, b=2) SET LOCATION " +
1567-
s"'${newDirFile.getAbsolutePath}'")
1569+
val newDirPath = newDirFile.getAbsolutePath.stripSuffix("/")
1570+
spark.sql(s"ALTER TABLE t PARTITION(a=1, b=2) SET LOCATION '$newDirPath'")
15681571
assert(!newDirFile.exists())
15691572

15701573
// insert into a partition which location does not exists.
@@ -1578,24 +1581,26 @@ class HiveDDLSuite
15781581
test("read data from a hive serde table which has a not existed location should succeed") {
15791582
withTable("t") {
15801583
withTempDir { dir =>
1584+
val dirPath = dir.getAbsolutePath.stripSuffix("/")
15811585
spark.sql(
15821586
s"""
15831587
|CREATE TABLE t(a string, b int)
15841588
|USING hive
15851589
|OPTIONS(path "file:${dir.getAbsolutePath}")
15861590
""".stripMargin)
15871591
val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
1588-
val expectedPath = s"file:${dir.getAbsolutePath.stripSuffix("/")}"
1592+
val expectedPath = s"file:$dirPath"
15891593
assert(table.location.stripSuffix("/") == expectedPath)
15901594

15911595
dir.delete()
15921596
checkAnswer(spark.table("t"), Nil)
15931597

15941598
val newDirFile = new File(dir, "x")
1595-
spark.sql(s"ALTER TABLE t SET LOCATION '${newDirFile.getAbsolutePath}'")
1599+
val newDirPath = newDirFile.getAbsolutePath.stripSuffix("/")
1600+
spark.sql(s"ALTER TABLE t SET LOCATION '$newDirPath'")
15961601

15971602
val table1 = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
1598-
assert(table1.location.stripSuffix("/") == newDirFile.getAbsolutePath.stripSuffix("/"))
1603+
assert(table1.location.stripSuffix("/") == newDirPath)
15991604
assert(!newDirFile.exists())
16001605
checkAnswer(spark.table("t"), Nil)
16011606
}
@@ -1612,14 +1617,12 @@ class HiveDDLSuite
16121617
|PARTITIONED BY(a, b)
16131618
|LOCATION "file:${dir.getCanonicalPath}"
16141619
""".stripMargin)
1615-
val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
1616-
16171620
spark.sql("INSERT INTO TABLE t PARTITION(a=1, b=2) SELECT 3, 4")
16181621
checkAnswer(spark.table("t"), Row(3, 4, 1, 2) :: Nil)
16191622

16201623
val newDirFile = new File(dir, "x")
1621-
spark.sql(s"ALTER TABLE t PARTITION(a=1, b=2) SET LOCATION " +
1622-
s"'${newDirFile.getAbsolutePath}'")
1624+
val newDirPath = newDirFile.getAbsolutePath
1625+
spark.sql(s"ALTER TABLE t PARTITION(a=1, b=2) SET LOCATION '$newDirPath'")
16231626
assert(!newDirFile.exists())
16241627
// select from a partition which location has changed to a not existed location
16251628
withSQLConf(SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "true") {

0 commit comments

Comments
 (0)