Skip to content

Commit a7ba238

Browse files
dongjoon-hyundongjoon-hyun
committed
[SPARK-20954][SQL][BRANCH-2.2] DESCRIBE [EXTENDED] result should be compatible with previous Spark
After [SPARK-20067](https://issues.apache.org/jira/browse/SPARK-20067), `DESCRIBE` and `DESCRIBE EXTENDED` shows the following result. This is incompatible with Spark 2.1.1. This PR removes the column header line in case of those command. **MASTER** and **BRANCH-2.2** ```scala scala> sql("desc t").show(false) +----------+---------+-------+ |col_name |data_type|comment| +----------+---------+-------+ |# col_name|data_type|comment| |a |int |null | +----------+---------+-------+ ``` **SPARK 2.1.1** and **this PR** ```scala scala> sql("desc t").show(false) +--------+---------+-------+ |col_name|data_type|comment| +--------+---------+-------+ |a |int |null | +--------+---------+-------+ ``` Pass the Jenkins with the updated test suites.
1 parent 02cf178 commit a7ba238

4 files changed

Lines changed: 12 additions & 33 deletions

File tree

‎sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -522,15 +522,15 @@ case class DescribeTableCommand(
522522
throw new AnalysisException(
523523
s"DESC PARTITION is not allowed on a temporary view: ${table.identifier}")
524524
}
525-
describeSchema(catalog.lookupRelation(table).schema, result)
525+
describeSchema(catalog.lookupRelation(table).schema, result, header = false)
526526
} else {
527527
val metadata = catalog.getTableMetadata(table)
528528
if (metadata.schema.isEmpty) {
529529
// In older version(prior to 2.1) of Spark, the table schema can be empty and should be
530530
// inferred at runtime. We should still support it.
531-
describeSchema(sparkSession.table(metadata.identifier).schema, result)
531+
describeSchema(sparkSession.table(metadata.identifier).schema, result, header = false)
532532
} else {
533-
describeSchema(metadata.schema, result)
533+
describeSchema(metadata.schema, result, header = false)
534534
}
535535

536536
describePartitionInfo(metadata, result)
@@ -550,7 +550,7 @@ case class DescribeTableCommand(
550550
private def describePartitionInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = {
551551
if (table.partitionColumnNames.nonEmpty) {
552552
append(buffer, "# Partition Information", "", "")
553-
describeSchema(table.partitionSchema, buffer)
553+
describeSchema(table.partitionSchema, buffer, header = true)
554554
}
555555
}
556556

@@ -601,8 +601,13 @@ case class DescribeTableCommand(
601601
table.storage.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, ""))
602602
}
603603

604-
private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = {
605-
append(buffer, s"# ${output.head.name}", output(1).name, output(2).name)
604+
private def describeSchema(
605+
schema: StructType,
606+
buffer: ArrayBuffer[Row],
607+
header: Boolean): Unit = {
608+
if (header) {
609+
append(buffer, s"# ${output.head.name}", output(1).name, output(2).name)
610+
}
606611
schema.foreach { column =>
607612
append(buffer, column.name, column.dataType.simpleString, column.getComment().orNull)
608613
}

‎sql/core/src/test/resources/sql-tests/results/change-column.sql.out‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ DESC test_change
1515
-- !query 1 schema
1616
struct<col_name:string,data_type:string,comment:string>
1717
-- !query 1 output
18-
# col_name data_type comment
1918
a int
2019
b string
2120
c int
@@ -35,7 +34,6 @@ DESC test_change
3534
-- !query 3 schema
3635
struct<col_name:string,data_type:string,comment:string>
3736
-- !query 3 output
38-
# col_name data_type comment
3937
a int
4038
b string
4139
c int
@@ -55,7 +53,6 @@ DESC test_change
5553
-- !query 5 schema
5654
struct<col_name:string,data_type:string,comment:string>
5755
-- !query 5 output
58-
# col_name data_type comment
5956
a int
6057
b string
6158
c int
@@ -94,7 +91,6 @@ DESC test_change
9491
-- !query 8 schema
9592
struct<col_name:string,data_type:string,comment:string>
9693
-- !query 8 output
97-
# col_name data_type comment
9894
a int
9995
b string
10096
c int
@@ -129,7 +125,6 @@ DESC test_change
129125
-- !query 12 schema
130126
struct<col_name:string,data_type:string,comment:string>
131127
-- !query 12 output
132-
# col_name data_type comment
133128
a int this is column a
134129
b string #*02?`
135130
c int
@@ -148,7 +143,6 @@ DESC test_change
148143
-- !query 14 schema
149144
struct<col_name:string,data_type:string,comment:string>
150145
-- !query 14 output
151-
# col_name data_type comment
152146
a int this is column a
153147
b string #*02?`
154148
c int
@@ -168,7 +162,6 @@ DESC test_change
168162
-- !query 16 schema
169163
struct<col_name:string,data_type:string,comment:string>
170164
-- !query 16 output
171-
# col_name data_type comment
172165
a int this is column a
173166
b string #*02?`
174167
c int
@@ -193,7 +186,6 @@ DESC test_change
193186
-- !query 18 schema
194187
struct<col_name:string,data_type:string,comment:string>
195188
-- !query 18 output
196-
# col_name data_type comment
197189
a int this is column a
198190
b string #*02?`
199191
c int
@@ -237,7 +229,6 @@ DESC test_change
237229
-- !query 23 schema
238230
struct<col_name:string,data_type:string,comment:string>
239231
-- !query 23 output
240-
# col_name data_type comment
241232
a int this is column A
242233
b string #*02?`
243234
c int

‎sql/core/src/test/resources/sql-tests/results/describe.sql.out‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ DESCRIBE t
5454
-- !query 5 schema
5555
struct<col_name:string,data_type:string,comment:string>
5656
-- !query 5 output
57-
# col_name data_type comment
5857
a string
5958
b int
6059
c string
@@ -70,7 +69,6 @@ DESC default.t
7069
-- !query 6 schema
7170
struct<col_name:string,data_type:string,comment:string>
7271
-- !query 6 output
73-
# col_name data_type comment
7472
a string
7573
b int
7674
c string
@@ -86,7 +84,6 @@ DESC TABLE t
8684
-- !query 7 schema
8785
struct<col_name:string,data_type:string,comment:string>
8886
-- !query 7 output
89-
# col_name data_type comment
9087
a string
9188
b int
9289
c string
@@ -102,7 +99,6 @@ DESC FORMATTED t
10299
-- !query 8 schema
103100
struct<col_name:string,data_type:string,comment:string>
104101
-- !query 8 output
105-
# col_name data_type comment
106102
a string
107103
b int
108104
c string
@@ -132,7 +128,6 @@ DESC EXTENDED t
132128
-- !query 9 schema
133129
struct<col_name:string,data_type:string,comment:string>
134130
-- !query 9 output
135-
# col_name data_type comment
136131
a string
137132
b int
138133
c string
@@ -162,7 +157,6 @@ DESC t PARTITION (c='Us', d=1)
162157
-- !query 10 schema
163158
struct<col_name:string,data_type:string,comment:string>
164159
-- !query 10 output
165-
# col_name data_type comment
166160
a string
167161
b int
168162
c string
@@ -178,7 +172,6 @@ DESC EXTENDED t PARTITION (c='Us', d=1)
178172
-- !query 11 schema
179173
struct<col_name:string,data_type:string,comment:string>
180174
-- !query 11 output
181-
# col_name data_type comment
182175
a string
183176
b int
184177
c string
@@ -206,7 +199,6 @@ DESC FORMATTED t PARTITION (c='Us', d=1)
206199
-- !query 12 schema
207200
struct<col_name:string,data_type:string,comment:string>
208201
-- !query 12 output
209-
# col_name data_type comment
210202
a string
211203
b int
212204
c string
@@ -268,7 +260,6 @@ DESC temp_v
268260
-- !query 16 schema
269261
struct<col_name:string,data_type:string,comment:string>
270262
-- !query 16 output
271-
# col_name data_type comment
272263
a string
273264
b int
274265
c string
@@ -280,7 +271,6 @@ DESC TABLE temp_v
280271
-- !query 17 schema
281272
struct<col_name:string,data_type:string,comment:string>
282273
-- !query 17 output
283-
# col_name data_type comment
284274
a string
285275
b int
286276
c string
@@ -292,7 +282,6 @@ DESC FORMATTED temp_v
292282
-- !query 18 schema
293283
struct<col_name:string,data_type:string,comment:string>
294284
-- !query 18 output
295-
# col_name data_type comment
296285
a string
297286
b int
298287
c string
@@ -304,7 +293,6 @@ DESC EXTENDED temp_v
304293
-- !query 19 schema
305294
struct<col_name:string,data_type:string,comment:string>
306295
-- !query 19 output
307-
# col_name data_type comment
308296
a string
309297
b int
310298
c string
@@ -316,7 +304,6 @@ DESC temp_Data_Source_View
316304
-- !query 20 schema
317305
struct<col_name:string,data_type:string,comment:string>
318306
-- !query 20 output
319-
# col_name data_type comment
320307
intType int test comment test1
321308
stringType string
322309
dateType date
@@ -349,7 +336,6 @@ DESC v
349336
-- !query 22 schema
350337
struct<col_name:string,data_type:string,comment:string>
351338
-- !query 22 output
352-
# col_name data_type comment
353339
a string
354340
b int
355341
c string
@@ -361,7 +347,6 @@ DESC TABLE v
361347
-- !query 23 schema
362348
struct<col_name:string,data_type:string,comment:string>
363349
-- !query 23 output
364-
# col_name data_type comment
365350
a string
366351
b int
367352
c string
@@ -373,7 +358,6 @@ DESC FORMATTED v
373358
-- !query 24 schema
374359
struct<col_name:string,data_type:string,comment:string>
375360
-- !query 24 output
376-
# col_name data_type comment
377361
a string
378362
b int
379363
c string
@@ -396,7 +380,6 @@ DESC EXTENDED v
396380
-- !query 25 schema
397381
struct<col_name:string,data_type:string,comment:string>
398382
-- !query 25 output
399-
# col_name data_type comment
400383
a string
401384
b int
402385
c string

‎sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -786,7 +786,7 @@ class HiveDDLSuite
786786

787787
checkAnswer(
788788
sql(s"DESC $tabName").select("col_name", "data_type", "comment"),
789-
Row("# col_name", "data_type", "comment") :: Row("a", "int", "test") :: Nil
789+
Row("a", "int", "test") :: Nil
790790
)
791791
}
792792
}

0 commit comments

Comments
 (0)