From 77953558117bf2f93e914bdfb43a5d985ac72fa9 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Sat, 22 Jun 2019 17:51:29 +0800 Subject: [PATCH 01/16] Add ShowCreateTableAsSparkCommand. --- .../spark/sql/catalyst/parser/SqlBase.g4 | 5 +- .../spark/sql/execution/SparkSqlParser.scala | 8 +- .../spark/sql/execution/command/tables.scala | 224 ++++++++++++------ .../apache/spark/sql/internal/HiveSerDe.scala | 16 ++ .../spark/sql/ShowCreateTableSuite.scala | 2 +- .../sql/hive/HiveShowCreateTableSuite.scala | 218 +++++++++++++++++ 6 files changed, 398 insertions(+), 75 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index dcb793921d6bb..5e6eac40adfcf 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -183,7 +183,7 @@ statement | SHOW PARTITIONS tableIdentifier partitionSpec? #showPartitions | SHOW identifier? FUNCTIONS (LIKE? (qualifiedName | pattern=STRING))? #showFunctions - | SHOW CREATE TABLE tableIdentifier #showCreateTable + | SHOW CREATE TABLE tableIdentifier (AS SPARK)? #showCreateTable | (DESC | DESCRIBE) FUNCTION EXTENDED? describeFuncName #describeFunction | (DESC | DESCRIBE) database EXTENDED? identifier #describeDatabase | (DESC | DESCRIBE) TABLE? option=(EXTENDED | FORMATTED)? @@ -1027,6 +1027,7 @@ ansiNonReserved | SKEWED | SORT | SORTED + | SPARK | START | STATISTICS | STORED @@ -1283,6 +1284,7 @@ nonReserved | SOME | SORT | SORTED + | SPARK | START | STATISTICS | STORED @@ -1541,6 +1543,7 @@ SKEWED: 'SKEWED'; SOME: 'SOME'; SORT: 'SORT'; SORTED: 'SORTED'; +SPARK: 'SPARK'; START: 'START'; STATISTICS: 'STATISTICS'; STORED: 'STORED'; diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index de1fbc0f99b8a..7d6f8b30ce805 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -235,11 +235,15 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { } /** - * Creates a [[ShowCreateTableCommand]] + * Creates a [[ShowCreateTableCommand]] or [[ShowCreateTableAsSparkCommand]] */ override def visitShowCreateTable(ctx: ShowCreateTableContext): LogicalPlan = withOrigin(ctx) { val table = visitTableIdentifier(ctx.tableIdentifier()) - ShowCreateTableCommand(table) + if (ctx.SPARK != null) { + ShowCreateTableAsSparkCommand(table) + } else { + ShowCreateTableCommand(table) + } } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 03aca89bc642e..13aa78d8c9d84 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -17,9 +17,7 @@ package org.apache.spark.sql.execution.command -import java.io.File import java.net.{URI, URISyntaxException} -import java.nio.file.FileSystems import scala.collection.mutable.ArrayBuffer import scala.util.Try @@ -29,7 +27,7 @@ import org.apache.hadoop.fs.{FileContext, FsConstants, Path} import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, UnresolvedAttribute, UnresolvedRelation} +import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, UnresolvedAttribute} import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.catalog.CatalogTableType._ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec @@ -44,7 +42,7 @@ import org.apache.spark.sql.execution.datasources.v2.csv.CSVDataSourceV2 import org.apache.spark.sql.execution.datasources.v2.json.JsonDataSourceV2 import org.apache.spark.sql.execution.datasources.v2.orc.OrcDataSourceV2 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2 -import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.{HiveSerDe, SQLConf} import org.apache.spark.sql.types._ import org.apache.spark.sql.util.SchemaUtils @@ -942,7 +940,95 @@ case class ShowPartitionsCommand( } } -case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableCommand { +/** + * Provides common utilities between `ShowCreateTableCommand` and `ShowCreateTableAsSparkCommand`. + */ +trait ShowCreateTableCommandBase { + + protected val table: TableIdentifier + + protected def showTableLocation(metadata: CatalogTable, builder: StringBuilder): Unit = { + if (metadata.tableType == EXTERNAL) { + metadata.storage.locationUri.foreach { location => + builder ++= s"LOCATION '${escapeSingleQuotedString(CatalogUtils.URIToString(location))}'\n" + } + } + } + + protected def showTableComment(metadata: CatalogTable, builder: StringBuilder): Unit = { + metadata + .comment + .map("COMMENT '" + escapeSingleQuotedString(_) + "'\n") + .foreach(builder.append) + } + + protected def showTableProperties(metadata: CatalogTable, builder: StringBuilder): Unit = { + if (metadata.properties.nonEmpty) { + val props = metadata.properties.map { case (key, value) => + s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'" + } + + builder ++= props.mkString("TBLPROPERTIES (\n ", ",\n ", "\n)\n") + } + } + + protected def showDataSourceTableDataColumns( + metadata: CatalogTable, builder: StringBuilder): Unit = { + val columns = metadata.schema.fields.map(_.toDDL) + builder ++= columns.mkString("(", ", ", ")\n") + } + + protected def showDataSourceTableOptions(metadata: CatalogTable, builder: StringBuilder): Unit = { + builder ++= s"USING ${metadata.provider.get}\n" + + val dataSourceOptions = metadata.storage.properties.map { + case (key, value) => s"${quoteIdentifier(key)} '${escapeSingleQuotedString(value)}'" + } + + if (dataSourceOptions.nonEmpty) { + builder ++= "OPTIONS (\n" + builder ++= dataSourceOptions.mkString(" ", ",\n ", "\n") + builder ++= ")\n" + } + } + + protected def showDataSourceTableNonDataColumns( + metadata: CatalogTable, builder: StringBuilder): Unit = { + val partCols = metadata.partitionColumnNames + if (partCols.nonEmpty) { + builder ++= s"PARTITIONED BY ${partCols.mkString("(", ", ", ")")}\n" + } + + metadata.bucketSpec.foreach { spec => + if (spec.bucketColumnNames.nonEmpty) { + builder ++= s"CLUSTERED BY ${spec.bucketColumnNames.mkString("(", ", ", ")")}\n" + + if (spec.sortColumnNames.nonEmpty) { + builder ++= s"SORTED BY ${spec.sortColumnNames.mkString("(", ", ", ")")}\n" + } + + builder ++= s"INTO ${spec.numBuckets} BUCKETS\n" + } + } + } + + protected def showCreateDataSourceTable(metadata: CatalogTable): String = { + val builder = StringBuilder.newBuilder + + builder ++= s"CREATE TABLE ${table.quotedString} " + showDataSourceTableDataColumns(metadata, builder) + showDataSourceTableOptions(metadata, builder) + showDataSourceTableNonDataColumns(metadata, builder) + showTableComment(metadata, builder) + showTableLocation(metadata, builder) + showTableProperties(metadata, builder) + + builder.toString() + } +} + +case class ShowCreateTableCommand(table: TableIdentifier) + extends RunnableCommand with ShowCreateTableCommandBase { override val output: Seq[Attribute] = Seq( AttributeReference("createtab_stmt", StringType, nullable = false)() ) @@ -1057,83 +1143,79 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman } } } +} - private def showTableLocation(metadata: CatalogTable, builder: StringBuilder): Unit = { - if (metadata.tableType == EXTERNAL) { - metadata.storage.locationUri.foreach { location => - builder ++= s"LOCATION '${escapeSingleQuotedString(CatalogUtils.URIToString(location))}'\n" - } - } - } +/** + * This commands generates Spark DDL for Hive table. + * + * The syntax of using this command in SQL is: + * {{{ + * SHOW CREATE TABLE table_identifier AS SPARK; + * }}} + */ +case class ShowCreateTableAsSparkCommand(table: TableIdentifier) + extends RunnableCommand with ShowCreateTableCommandBase { + override val output: Seq[Attribute] = Seq( + AttributeReference("sparktab_stmt", StringType, nullable = false)() + ) - private def showTableComment(metadata: CatalogTable, builder: StringBuilder): Unit = { - metadata - .comment - .map("COMMENT '" + escapeSingleQuotedString(_) + "'\n") - .foreach(builder.append) - } + override def run(sparkSession: SparkSession): Seq[Row] = { + val catalog = sparkSession.sessionState.catalog + val tableMetadata = catalog.getTableMetadata(table) - private def showTableProperties(metadata: CatalogTable, builder: StringBuilder): Unit = { - if (metadata.properties.nonEmpty) { - val props = metadata.properties.map { case (key, value) => - s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'" + val stmt = if (DDLUtils.isDatasourceTable(tableMetadata)) { + throw new AnalysisException( + s"$table is already a Spark data source table. Using `SHOW CREATE TABLE` instead.") + } else { + if (tableMetadata.unsupportedFeatures.nonEmpty) { + throw new AnalysisException( + "Failed to execute SHOW CREATE TABLE AS SPARK against table " + + s"${tableMetadata.identifier}, which is created by Hive and uses the " + + "following unsupported feature(s)\n" + + tableMetadata.unsupportedFeatures.map(" - " + _).mkString("\n") + ) } - builder ++= props.mkString("TBLPROPERTIES (\n ", ",\n ", "\n)\n") - } - } - - private def showCreateDataSourceTable(metadata: CatalogTable): String = { - val builder = StringBuilder.newBuilder - - builder ++= s"CREATE TABLE ${table.quotedString} " - showDataSourceTableDataColumns(metadata, builder) - showDataSourceTableOptions(metadata, builder) - showDataSourceTableNonDataColumns(metadata, builder) - showTableComment(metadata, builder) - showTableLocation(metadata, builder) - showTableProperties(metadata, builder) - - builder.toString() - } - - private def showDataSourceTableDataColumns( - metadata: CatalogTable, builder: StringBuilder): Unit = { - val columns = metadata.schema.fields.map(_.toDDL) - builder ++= columns.mkString("(", ", ", ")\n") - } - - private def showDataSourceTableOptions(metadata: CatalogTable, builder: StringBuilder): Unit = { - builder ++= s"USING ${metadata.provider.get}\n" + if (tableMetadata.tableType == VIEW) { + throw new AnalysisException("Hive view isn't supported by SHOW CREATE TABLE AS SPARK") + } - val dataSourceOptions = metadata.storage.properties.map { - case (key, value) => s"${quoteIdentifier(key)} '${escapeSingleQuotedString(value)}'" + showCreateDataSourceTable(convertTableMetadata(tableMetadata)) } - if (dataSourceOptions.nonEmpty) { - builder ++= "OPTIONS (\n" - builder ++= dataSourceOptions.mkString(" ", ",\n ", "\n") - builder ++= ")\n" - } + Seq(Row(stmt)) } - private def showDataSourceTableNonDataColumns( - metadata: CatalogTable, builder: StringBuilder): Unit = { - val partCols = metadata.partitionColumnNames - if (partCols.nonEmpty) { - builder ++= s"PARTITIONED BY ${partCols.mkString("(", ", ", ")")}\n" - } - - metadata.bucketSpec.foreach { spec => - if (spec.bucketColumnNames.nonEmpty) { - builder ++= s"CLUSTERED BY ${spec.bucketColumnNames.mkString("(", ", ", ")")}\n" - - if (spec.sortColumnNames.nonEmpty) { - builder ++= s"SORTED BY ${spec.sortColumnNames.mkString("(", ", ", ")")}\n" - } - - builder ++= s"INTO ${spec.numBuckets} BUCKETS\n" + private def convertTableMetadata(tableMetadata: CatalogTable): CatalogTable = { + val hiveSerde = HiveSerDe( + serde = tableMetadata.storage.serde, + inputFormat = tableMetadata.storage.inputFormat, + outputFormat = tableMetadata.storage.outputFormat) + + // Looking for Spark data source that maps to to the Hive serde. + // TODO: some Hive fileformat + row serde might be mapped to Spark data source, e.g. CSV. + val source = HiveSerDe.serdeToSource(hiveSerde) + if (source.isEmpty) { + val builder = StringBuilder.newBuilder + hiveSerde.serde.foreach { serde => + builder ++= s" SERDE: $serde" + } + hiveSerde.inputFormat.foreach { format => + builder ++= s" INPUTFORMAT: $format" } + hiveSerde.outputFormat.foreach { format => + builder ++= s" OUTPUTFORMAT: $format" + } + throw new AnalysisException( + "Failed to execute SHOW CREATE TABLE AS SPARK against table " + + s"${tableMetadata.identifier}, which is created by Hive and uses the " + + "following unsupported serde configuration\n" + + builder.toString() + ) + } else { + // TODO: should we keep Hive serde properties? + val newStorage = tableMetadata.storage.copy(properties = Map.empty) + tableMetadata.copy(provider = source, storage = newStorage) } } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala index 4921e3ca903c4..64b7e7fe7923a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala @@ -65,6 +65,14 @@ object HiveSerDe { outputFormat = Option("org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat"), serde = Option("org.apache.hadoop.hive.serde2.avro.AvroSerDe"))) + // `HiveSerDe` in `serdeMap` should be dintinct. + val serdeInverseMap: Map[HiveSerDe, String] = serdeMap.flatMap { + case ("sequencefile", _) => None + case ("rcfile", _) => None + case ("textfile", serde) => Some((serde, "text")) + case pair => Some(pair.swap) + } + /** * Get the Hive SerDe information from the data source abbreviation string or classname. * @@ -88,6 +96,14 @@ object HiveSerDe { serdeMap.get(key) } + /** + * Get the Spark data source name from the Hive SerDe information. + * + * @param serde Hive SerDe information. + * @return Spark data source name associated with the specified Hive Serde. + */ + def serdeToSource(serde: HiveSerDe): Option[String] = serdeInverseMap.get(serde) + def getDefaultStorage(conf: SQLConf): CatalogStorageFormat = { // To respect hive-site.xml, it peeks Hadoop configuration from existing Spark session, // as an easy workaround. See SPARK-27555. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala index 5c347d2677d5e..91a35b6cd58c9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala @@ -200,7 +200,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { } } - private def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { + protected def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { def normalize(table: CatalogTable): CatalogTable = { val nondeterministicProps = Set( "CreateTime", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala index 0386dc79804c6..86a0fb184d18c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala @@ -18,6 +18,8 @@ package org.apache.spark.sql.hive import org.apache.spark.sql.{AnalysisException, ShowCreateTableSuite} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable} import org.apache.spark.sql.hive.test.TestHiveSingleton class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSingleton { @@ -172,6 +174,12 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet } assert(cause.getMessage.contains(" - partitioned view")) + + val causeForSpark = intercept[AnalysisException] { + sql("SHOW CREATE TABLE v1 AS SPARK") + } + + assert(causeForSpark.getMessage.contains(" - partitioned view")) } } } @@ -195,4 +203,214 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet hiveContext.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog] .client.runSqlHive(ddl) } + + private def checkCreateSparkTable(tableName: String): Unit = { + val table = TableIdentifier(tableName, Some("default")) + val db = table.database.get + val hiveTable = spark.sharedState.externalCatalog.getTable(db, table.table) + val shownSparkDDL = sql(s"SHOW CREATE TABLE ${table.quotedString} AS SPARK").head().getString(0) + // Drops original Hive table. + sql(s"DROP TABLE ${table.quotedString}") + + try { + sql(shownSparkDDL) + val actual = spark.sharedState.externalCatalog.getTable(db, table.table) + val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0) + + // Drops created Spark table using `SHOW CREATE TABLE AS SPARK`. + sql(s"DROP TABLE ${table.quotedString}") + + sql(shownDDL) + val expected = spark.sharedState.externalCatalog.getTable(db, table.table) + + checkCatalogTables(expected, actual) + checkHiveCatalogTables(hiveTable, actual) + } finally { + sql(s"DROP TABLE IF EXISTS ${table.table}") + } + } + + private def checkHiveCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { + def normalize(table: CatalogTable): CatalogTable = { + val nondeterministicProps = Set( + "CreateTime", + "transient_lastDdlTime", + "grantTime", + "lastUpdateTime", + "last_modified_by", + "last_modified_time", + "Owner:", + // The following are hive specific schema parameters which we do not need to match exactly. + "totalNumberFiles", + "maxFileSize", + "minFileSize" + ) + + table.copy( + createTime = 0L, + lastAccessTime = 0L, + properties = table.properties.filterKeys(!nondeterministicProps.contains(_)), + stats = None, + ignoredProperties = Map.empty, + storage = CatalogStorageFormat.empty, + provider = None, + tracksPartitionsInCatalog = false + ) + } + assert(normalize(actual) == normalize(expected)) + } + + test("simple hive table as spark") { + withTable("t1") { + sql( + s"""CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |TBLPROPERTIES ( + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + """.stripMargin + ) + + checkCreateSparkTable("t1") + } + } + + test("show create table as spark can't work on data source table") { + withTable("t1") { + sql( + s"""CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |USING orc + """.stripMargin + ) + + val cause = intercept[AnalysisException] { + checkCreateSparkTable("t1") + } + + assert(cause.getMessage.contains("Using `SHOW CREATE TABLE` instead")) + } + } + + test("simple external hive table as spark") { + withTempDir { dir => + withTable("t1") { + sql( + s"""CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |LOCATION '${dir.toURI}' + |TBLPROPERTIES ( + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + """.stripMargin + ) + + checkCreateSparkTable("t1") + } + } + } + + test("hive table with STORED AS clause as spark") { + withTable("t1") { + sql( + s"""CREATE TABLE t1 ( + | c1 INT COMMENT 'bla', + | c2 STRING + |) + |STORED AS PARQUET + """.stripMargin + ) + + checkCreateSparkTable("t1") + } + } + + test("hive table with unsupported fileformat as spark") { + withTable("t1") { + sql( + s"""CREATE TABLE t1 ( + | c1 INT COMMENT 'bla', + | c2 STRING + |) + |STORED AS RCFILE + """.stripMargin + ) + + val cause = intercept[AnalysisException] { + checkCreateSparkTable("t1") + } + + assert(cause.getMessage.contains("unsupported serde configuration")) + } + } + + test("hive table with serde info as spark") { + withTable("t1") { + sql( + s"""CREATE TABLE t1 ( + | c1 INT COMMENT 'bla', + | c2 STRING + |) + |ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' + |STORED AS + | INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' + | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' + """.stripMargin + ) + + checkCreateSparkTable("t1") + } + } + + test("hive view is not supported as spark") { + withTable("t1") { + withView("v1") { + sql( + s""" + |CREATE TABLE t1 (c1 STRING, c2 STRING) + """.stripMargin) + + createRawHiveTable( + s""" + |CREATE VIEW v1 + |AS SELECT * from t1 + """.stripMargin + ) + + val cause = intercept[AnalysisException] { + sql("SHOW CREATE TABLE v1 AS SPARK") + } + + assert(cause.getMessage.contains("view isn't supported")) + } + } + } + + test("partitioned, bucketed hive table as spark") { + withTable("t1") { + sql( + s"""CREATE TABLE t1 ( + | emp_id INT COMMENT 'employee id', emp_name STRING, + | emp_dob STRING COMMENT 'employee date of birth', emp_sex STRING COMMENT 'M/F' + |) + |COMMENT 'employee table' + |PARTITIONED BY ( + | emp_country STRING COMMENT '2-char code', emp_state STRING COMMENT '2-char code' + |) + |CLUSTERED BY (emp_sex) SORTED BY (emp_id ASC) INTO 10 BUCKETS + |STORED AS ORC + """.stripMargin + ) + + checkCreateSparkTable("t1") + } + } } From b9dacc5d229937043f01fced22726c38bdb365d9 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 2 Aug 2019 14:36:06 +0800 Subject: [PATCH 02/16] Address comments. --- .../spark/sql/execution/command/tables.scala | 9 +++- .../sql/hive/HiveShowCreateTableSuite.scala | 54 ++++++++++++++----- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 13aa78d8c9d84..2e3db95bec489 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -1165,7 +1165,7 @@ case class ShowCreateTableAsSparkCommand(table: TableIdentifier) val stmt = if (DDLUtils.isDatasourceTable(tableMetadata)) { throw new AnalysisException( - s"$table is already a Spark data source table. Using `SHOW CREATE TABLE` instead.") + s"$table is already a Spark data source table. Use `SHOW CREATE TABLE` instead.") } else { if (tableMetadata.unsupportedFeatures.nonEmpty) { throw new AnalysisException( @@ -1180,6 +1180,13 @@ case class ShowCreateTableAsSparkCommand(table: TableIdentifier) throw new AnalysisException("Hive view isn't supported by SHOW CREATE TABLE AS SPARK") } + // scalastyle:off caselocale + if (tableMetadata.properties.getOrElse("transactional", "false").toLowerCase.equals("true")) { + throw new AnalysisException( + "SHOW CRETE TABLE AS SPARK doesn't support transactional Hive table") + } + // scalastyle:on caselocale + showCreateDataSourceTable(convertTableMetadata(tableMetadata)) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala index 86a0fb184d18c..ac915ded3cc2b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala @@ -263,7 +263,8 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet test("simple hive table as spark") { withTable("t1") { sql( - s"""CREATE TABLE t1 ( + s""" + |CREATE TABLE t1 ( | c1 STRING COMMENT 'bla', | c2 STRING |) @@ -281,7 +282,8 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet test("show create table as spark can't work on data source table") { withTable("t1") { sql( - s"""CREATE TABLE t1 ( + s""" + |CREATE TABLE t1 ( | c1 STRING COMMENT 'bla', | c2 STRING |) @@ -293,7 +295,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet checkCreateSparkTable("t1") } - assert(cause.getMessage.contains("Using `SHOW CREATE TABLE` instead")) + assert(cause.getMessage.contains("Use `SHOW CREATE TABLE` instead")) } } @@ -301,7 +303,8 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet withTempDir { dir => withTable("t1") { sql( - s"""CREATE TABLE t1 ( + s""" + |CREATE TABLE t1 ( | c1 STRING COMMENT 'bla', | c2 STRING |) @@ -321,7 +324,8 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet test("hive table with STORED AS clause as spark") { withTable("t1") { sql( - s"""CREATE TABLE t1 ( + s""" + |CREATE TABLE t1 ( | c1 INT COMMENT 'bla', | c2 STRING |) @@ -336,7 +340,8 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet test("hive table with unsupported fileformat as spark") { withTable("t1") { sql( - s"""CREATE TABLE t1 ( + s""" + |CREATE TABLE t1 ( | c1 INT COMMENT 'bla', | c2 STRING |) @@ -355,7 +360,8 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet test("hive table with serde info as spark") { withTable("t1") { sql( - s"""CREATE TABLE t1 ( + s""" + |CREATE TABLE t1 ( | c1 INT COMMENT 'bla', | c2 STRING |) @@ -373,10 +379,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet test("hive view is not supported as spark") { withTable("t1") { withView("v1") { - sql( - s""" - |CREATE TABLE t1 (c1 STRING, c2 STRING) - """.stripMargin) + sql("CREATE TABLE t1 (c1 STRING, c2 STRING)") createRawHiveTable( s""" @@ -397,7 +400,8 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet test("partitioned, bucketed hive table as spark") { withTable("t1") { sql( - s"""CREATE TABLE t1 ( + s""" + |CREATE TABLE t1 ( | emp_id INT COMMENT 'employee id', emp_name STRING, | emp_dob STRING COMMENT 'employee date of birth', emp_sex STRING COMMENT 'M/F' |) @@ -413,4 +417,30 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet checkCreateSparkTable("t1") } } + + test("transactional hive table as spark") { + withTable("t1") { + sql( + s""" + |CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |TBLPROPERTIES ( + | 'transactional' = 'true', + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + """.stripMargin + ) + + + val cause = intercept[AnalysisException] { + sql("SHOW CREATE TABLE t1 AS SPARK") + } + + assert(cause.getMessage.contains( + "SHOW CRETE TABLE AS SPARK doesn't support transactional Hive table")) + } + } } From a909790d5e25c656e0ec61dc139b22f4b8702796 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Sat, 16 Nov 2019 17:02:55 -0800 Subject: [PATCH 03/16] Updated. --- .../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 2 +- .../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 2 +- .../spark/sql/catalyst/plans/logical/statements.scala | 4 +++- .../spark/sql/catalyst/analysis/ResolveSessionCatalog.scala | 6 +++++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index cd14d60a54f53..25099c0cea3ef 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -196,7 +196,7 @@ statement | SHOW PARTITIONS multipartIdentifier partitionSpec? #showPartitions | SHOW identifier? FUNCTIONS (LIKE? (multipartIdentifier | pattern=STRING))? #showFunctions - | SHOW CREATE TABLE multipartIdentifier (AS SPARK) #showCreateTable + | SHOW CREATE TABLE multipartIdentifier (AS SPARK)? #showCreateTable | SHOW CURRENT NAMESPACE #showCurrentNamespace | (DESC | DESCRIBE) FUNCTION EXTENDED? describeFuncName #describeFunction | (DESC | DESCRIBE) database EXTENDED? db=errorCapturingIdentifier #describeDatabase diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index c623b5c4d36a5..61613adddb1c6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -2982,7 +2982,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Creates a [[ShowCreateTableStatement]] */ override def visitShowCreateTable(ctx: ShowCreateTableContext): LogicalPlan = withOrigin(ctx) { - ShowCreateTableStatement(visitMultipartIdentifier(ctx.multipartIdentifier())) + ShowCreateTableStatement(visitMultipartIdentifier(ctx.multipartIdentifier()), ctx.SPARK != null) } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala index ec373d95fad88..1fbcc41d6a352 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala @@ -382,7 +382,9 @@ case class LoadDataStatement( /** * A SHOW CREATE TABLE statement, as parsed from SQL. */ -case class ShowCreateTableStatement(tableName: Seq[String]) extends ParsedStatement +case class ShowCreateTableStatement( + tableName: Seq[String], + asSpark: Boolean = false) extends ParsedStatement /** * A CACHE TABLE statement, as parsed from SQL diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index 340e09ae66adb..5c973c96490b2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -328,10 +328,14 @@ class ResolveSessionCatalog( isOverwrite, partition) - case ShowCreateTableStatement(tableName) => + case ShowCreateTableStatement(tableName, asSpark) if !asSpark => val v1TableName = parseV1Table(tableName, "SHOW CREATE TABLE") ShowCreateTableCommand(v1TableName.asTableIdentifier) + case ShowCreateTableStatement(tableName, asSpark) if asSpark => + val v1TableName = parseV1Table(tableName, "SHOW CREATE TABLE AS SPARK") + ShowCreateTableAsSparkCommand(v1TableName.asTableIdentifier) + case CacheTableStatement(tableName, plan, isLazy, options) => val v1TableName = parseV1Table(tableName, "CACHE TABLE") CacheTableCommand(v1TableName.asTableIdentifier, plan, isLazy, options) From 5f925327a9ed26e355158bc4e5884214d3a85438 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 27 Jan 2020 17:43:53 -0800 Subject: [PATCH 04/16] Fix test. --- .../scala/org/apache/spark/sql/ShowCreateTableSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala index 6b3ee28ebe804..21203c74c35ae 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala @@ -187,7 +187,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>)" sql(s"$createTable USING json") val shownDDL = getShowDDL("SHOW CREATE TABLE t1") - assert(shownDDL == createTable) + assert(shownDDL == s"$createTable USING json") checkCreateTable("t1") } @@ -199,7 +199,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { .getString(0) .split("\n") .map(_.trim) - if (result.length > 1) result(0) + result(1) else result.head + if (result.length > 1) result(0) + " " + result(1) else result.head } protected def checkCreateTable(table: String): Unit = { From 8dec28bbcd12d026a6249c23bd33222b4b873bb9 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 28 Jan 2020 00:42:00 -0800 Subject: [PATCH 05/16] Fix test. --- .../spark/sql/execution/command/tables.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index b8dfb37467db6..7dca496791e3c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -1030,14 +1030,15 @@ trait ShowCreateTableCommandBase { s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'" } - builder ++= props.mkString("TBLPROPERTIES (\n ", ",\n ", "\n)\n") + builder ++= "TBLPROPERTIES " + builder ++= concatByMultiLines(props) } } protected def showDataSourceTableDataColumns( metadata: CatalogTable, builder: StringBuilder): Unit = { val columns = metadata.schema.fields.map(_.toDDL) - builder ++= columns.mkString("(", ", ", ")\n") + builder ++= concatByMultiLines(columns) } protected def showDataSourceTableOptions(metadata: CatalogTable, builder: StringBuilder): Unit = { @@ -1048,9 +1049,8 @@ trait ShowCreateTableCommandBase { } if (dataSourceOptions.nonEmpty) { - builder ++= "OPTIONS (\n" - builder ++= dataSourceOptions.mkString(" ", ",\n ", "\n") - builder ++= ")\n" + builder ++= "OPTIONS " + builder ++= concatByMultiLines(dataSourceOptions) } } @@ -1087,6 +1087,10 @@ trait ShowCreateTableCommandBase { builder.toString() } + + protected def concatByMultiLines(iter: Iterable[String]): String = { + iter.mkString("(\n ", ",\n ", ")\n") + } } case class ShowCreateTableCommand(table: TableIdentifier) @@ -1172,10 +1176,6 @@ case class ShowCreateTableCommand(table: TableIdentifier) } } - private def concatByMultiLines(iter: Iterable[String]): String = { - iter.mkString("(\n ", ",\n ", ")\n") - } - private def showViewProperties(metadata: CatalogTable, builder: StringBuilder): Unit = { val viewProps = metadata.properties.filterKeys(!_.startsWith(CatalogTable.VIEW_PREFIX)) if (viewProps.nonEmpty) { From 9228d6afbcdc6d22544e01f2248723d00dbd6e35 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 29 Jan 2020 21:38:56 -0800 Subject: [PATCH 06/16] Make sure creating Hive table. --- .../spark/sql/ShowCreateTableSuite.scala | 1 + .../sql/hive/HiveShowCreateTableSuite.scala | 110 ++++++++++-------- 2 files changed, 61 insertions(+), 50 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala index 21203c74c35ae..e86e06a9ed6c2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala @@ -213,6 +213,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { private def checkCreateTableOrView(table: TableIdentifier, checkType: String): Unit = { val db = table.database.getOrElse("default") val expected = spark.sharedState.externalCatalog.getTable(db, table.table) + println(s"expected: ${expected.provider}") val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0) sql(s"DROP $checkType ${table.quotedString}") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala index 0d6d49e9068cf..6fb618da93a12 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.{AnalysisException, ShowCreateTableSuite} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable} import org.apache.spark.sql.hive.test.TestHiveSingleton +import org.apache.spark.sql.internal.SQLConf class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSingleton { @@ -257,21 +258,24 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet } test("simple hive table as spark") { - withTable("t1") { - sql( - s""" - |CREATE TABLE t1 ( - | c1 STRING COMMENT 'bla', - | c2 STRING - |) - |TBLPROPERTIES ( - | 'prop1' = 'value1', - | 'prop2' = 'value2' - |) - """.stripMargin - ) + withSQLConf( + SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") { + withTable("t1") { + sql( + s""" + |CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |TBLPROPERTIES ( + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + """.stripMargin + ) - checkCreateSparkTable("t1") + checkCreateSparkTable("t1") + } } } @@ -296,23 +300,26 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet } test("simple external hive table as spark") { - withTempDir { dir => - withTable("t1") { - sql( - s""" - |CREATE TABLE t1 ( - | c1 STRING COMMENT 'bla', - | c2 STRING - |) - |LOCATION '${dir.toURI}' - |TBLPROPERTIES ( - | 'prop1' = 'value1', - | 'prop2' = 'value2' - |) - """.stripMargin - ) - - checkCreateSparkTable("t1") + withSQLConf( + SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") { + withTempDir { dir => + withTable("t1") { + sql( + s""" + |CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |LOCATION '${dir.toURI}' + |TBLPROPERTIES ( + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + """.stripMargin + ) + + checkCreateSparkTable("t1") + } } } } @@ -415,28 +422,31 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet } test("transactional hive table as spark") { - withTable("t1") { - sql( - s""" - |CREATE TABLE t1 ( - | c1 STRING COMMENT 'bla', - | c2 STRING - |) - |TBLPROPERTIES ( - | 'transactional' = 'true', - | 'prop1' = 'value1', - | 'prop2' = 'value2' - |) - """.stripMargin - ) + withSQLConf( + SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") { + withTable("t1") { + sql( + s""" + |CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |TBLPROPERTIES ( + | 'transactional' = 'true', + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + """.stripMargin + ) - val cause = intercept[AnalysisException] { - sql("SHOW CREATE TABLE t1 AS SPARK") - } + val cause = intercept[AnalysisException] { + sql("SHOW CREATE TABLE t1 AS SPARK") + } - assert(cause.getMessage.contains( - "SHOW CRETE TABLE AS SPARK doesn't support transactional Hive table")) + assert(cause.getMessage.contains( + "SHOW CRETE TABLE AS SPARK doesn't support transactional Hive table")) + } } } } From bbe6fa7a8f4d4d689d3e021a277e3c0e8e18474c Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 29 Jan 2020 23:07:08 -0800 Subject: [PATCH 07/16] Fix merging conflit. --- .../scala/org/apache/spark/sql/ShowCreateTableSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala index e86e06a9ed6c2..7252721029b66 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala @@ -187,7 +187,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>)" sql(s"$createTable USING json") val shownDDL = getShowDDL("SHOW CREATE TABLE t1") - assert(shownDDL == s"$createTable USING json") + assert(shownDDL == createTable) checkCreateTable("t1") } @@ -199,7 +199,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { .getString(0) .split("\n") .map(_.trim) - if (result.length > 1) result(0) + " " + result(1) else result.head + if (result.length > 1) result(0) + result(1) else result.head } protected def checkCreateTable(table: String): Unit = { From 4f33fd8a60400f9fd6455eed928f8fe9af4dd7c5 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 29 Jan 2020 23:12:32 -0800 Subject: [PATCH 08/16] Fix merging conflict. --- .../scala/org/apache/spark/sql/execution/command/tables.scala | 2 +- .../scala/org/apache/spark/sql/ShowCreateTableSuite.scala | 1 - .../org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 7dca496791e3c..2736e35019798 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -1284,7 +1284,7 @@ case class ShowCreateTableAsSparkCommand(table: TableIdentifier) // scalastyle:off caselocale if (tableMetadata.properties.getOrElse("transactional", "false").toLowerCase.equals("true")) { throw new AnalysisException( - "SHOW CRETE TABLE AS SPARK doesn't support transactional Hive table") + "SHOW CREATE TABLE AS SPARK doesn't support transactional Hive table") } // scalastyle:on caselocale diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala index 7252721029b66..6b3ee28ebe804 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala @@ -213,7 +213,6 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { private def checkCreateTableOrView(table: TableIdentifier, checkType: String): Unit = { val db = table.database.getOrElse("default") val expected = spark.sharedState.externalCatalog.getTable(db, table.table) - println(s"expected: ${expected.provider}") val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0) sql(s"DROP $checkType ${table.quotedString}") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala index 6fb618da93a12..47e624d04ff77 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala @@ -436,6 +436,8 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet | 'prop1' = 'value1', | 'prop2' = 'value2' |) + |CLUSTERED BY (c1) INTO 10 BUCKETS + |STORED AS ORC """.stripMargin ) @@ -445,7 +447,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet } assert(cause.getMessage.contains( - "SHOW CRETE TABLE AS SPARK doesn't support transactional Hive table")) + "SHOW CREATE TABLE AS SPARK doesn't support transactional Hive table")) } } } From a4b0ce643fb862edf0a6165f7d6ed434c04257fc Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 30 Jan 2020 23:02:08 -0800 Subject: [PATCH 09/16] Add AS SERDE option. --- .../spark/sql/catalyst/parser/SqlBase.g4 | 5 +- .../sql/catalyst/parser/AstBuilder.scala | 2 +- .../catalyst/plans/logical/statements.scala | 2 +- .../analysis/ResolveSessionCatalog.scala | 8 +- .../spark/sql/execution/command/tables.scala | 184 +++++++------ .../spark/sql/ShowCreateTableSuite.scala | 14 - .../sql/hive/HiveShowCreateTableSuite.scala | 241 ++++++++++-------- 7 files changed, 242 insertions(+), 214 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 704ea250ca439..6f2bb7a9a7536 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -210,7 +210,7 @@ statement | SHOW PARTITIONS multipartIdentifier partitionSpec? #showPartitions | SHOW identifier? FUNCTIONS (LIKE? (multipartIdentifier | pattern=STRING))? #showFunctions - | SHOW CREATE TABLE multipartIdentifier (AS SPARK)? #showCreateTable + | SHOW CREATE TABLE multipartIdentifier (AS SERDE)? #showCreateTable | SHOW CURRENT NAMESPACE #showCurrentNamespace | (DESC | DESCRIBE) FUNCTION EXTENDED? describeFuncName #describeFunction | (DESC | DESCRIBE) namespace EXTENDED? @@ -1140,7 +1140,6 @@ ansiNonReserved | SKEWED | SORT | SORTED - | SPARK | START | STATISTICS | STORED @@ -1399,7 +1398,6 @@ nonReserved | SOME | SORT | SORTED - | SPARK | START | STATISTICS | STORED @@ -1661,7 +1659,6 @@ SKEWED: 'SKEWED'; SOME: 'SOME'; SORT: 'SORT'; SORTED: 'SORTED'; -SPARK: 'SPARK'; START: 'START'; STATISTICS: 'STATISTICS'; STORED: 'STORED'; diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 0a4ae38014493..e9ad84472904d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -3215,7 +3215,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Creates a [[ShowCreateTableStatement]] */ override def visitShowCreateTable(ctx: ShowCreateTableContext): LogicalPlan = withOrigin(ctx) { - ShowCreateTableStatement(visitMultipartIdentifier(ctx.multipartIdentifier()), ctx.SPARK != null) + ShowCreateTableStatement(visitMultipartIdentifier(ctx.multipartIdentifier()), ctx.SERDE != null) } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala index 1ce0c46c9d149..1e6b67bf78b70 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala @@ -391,7 +391,7 @@ case class LoadDataStatement( */ case class ShowCreateTableStatement( tableName: Seq[String], - asSpark: Boolean = false) extends ParsedStatement + asSerde: Boolean = false) extends ParsedStatement /** * A CACHE TABLE statement, as parsed from SQL diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index 77859d31a8911..486e7f1f84b46 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -378,13 +378,13 @@ class ResolveSessionCatalog( isOverwrite, partition) - case ShowCreateTableStatement(tbl, asSpark) if !asSpark => + case ShowCreateTableStatement(tbl, asSerde) if !asSerde => val v1TableName = parseV1Table(tbl, "SHOW CREATE TABLE") ShowCreateTableCommand(v1TableName.asTableIdentifier) - case ShowCreateTableStatement(tbl, asSpark) if asSpark => - val v1TableName = parseV1Table(tbl, "SHOW CREATE TABLE AS SPARK") - ShowCreateTableAsSparkCommand(v1TableName.asTableIdentifier) + case ShowCreateTableStatement(tbl, asSerde) if asSerde => + val v1TableName = parseV1Table(tbl, "SHOW CREATE TABLE AS SERDE") + ShowCreateTableAsSerdeCommand(v1TableName.asTableIdentifier) case CacheTableStatement(tbl, plan, isLazy, options) => val v1TableName = parseV1Table(tbl, "CACHE TABLE") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 2736e35019798..efdc4cadebfd9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -1093,6 +1093,16 @@ trait ShowCreateTableCommandBase { } } +/** + * A command that shows the Spark DDL syntax that can be used to create a given table. + * For Hive serde table, this command will generate Spark DDL that can be used to + * create corresponding Spark table. + * + * The syntax of using this command in SQL is: + * {{{ + * SHOW CREATE TABLE [db_name.]table_name + * }}} + */ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableCommand with ShowCreateTableCommandBase { override val output: Seq[Attribute] = Seq( @@ -1109,16 +1119,102 @@ case class ShowCreateTableCommand(table: TableIdentifier) // TODO: [SPARK-28692] unify this after we unify the // CREATE TABLE syntax for hive serde and data source table. - val stmt = if (DDLUtils.isDatasourceTable(tableMetadata)) { - showCreateDataSourceTable(tableMetadata) + val metadata = if (DDLUtils.isDatasourceTable(tableMetadata)) { + tableMetadata } else { - showCreateHiveTable(tableMetadata) + // For a Hive serde table, we try to convert it to Spark DDL. + if (tableMetadata.unsupportedFeatures.nonEmpty) { + throw new AnalysisException( + "Failed to execute SHOW CREATE TABLE against table " + + s"${tableMetadata.identifier}, which is created by Hive and uses the " + + "following unsupported feature(s)\n" + + tableMetadata.unsupportedFeatures.map(" - " + _).mkString("\n") + ) + } + + if (tableMetadata.tableType == VIEW) { + throw new AnalysisException("Hive view isn't supported by SHOW CREATE TABLE") + } + + // scalastyle:off caselocale + if (tableMetadata.properties.getOrElse("transactional", "false") + .toLowerCase.equals("true")) { + throw new AnalysisException( + "SHOW CREATE TABLE doesn't support transactional Hive table") + } + // scalastyle:on caselocale + + convertTableMetadata(tableMetadata) } + val stmt = showCreateDataSourceTable(metadata) + Seq(Row(stmt)) } } + private def convertTableMetadata(tableMetadata: CatalogTable): CatalogTable = { + val hiveSerde = HiveSerDe( + serde = tableMetadata.storage.serde, + inputFormat = tableMetadata.storage.inputFormat, + outputFormat = tableMetadata.storage.outputFormat) + + // Looking for Spark data source that maps to to the Hive serde. + // TODO: some Hive fileformat + row serde might be mapped to Spark data source, e.g. CSV. + val source = HiveSerDe.serdeToSource(hiveSerde) + if (source.isEmpty) { + val builder = StringBuilder.newBuilder + hiveSerde.serde.foreach { serde => + builder ++= s" SERDE: $serde" + } + hiveSerde.inputFormat.foreach { format => + builder ++= s" INPUTFORMAT: $format" + } + hiveSerde.outputFormat.foreach { format => + builder ++= s" OUTPUTFORMAT: $format" + } + throw new AnalysisException( + "Failed to execute SHOW CREATE TABLE AS SPARK against table " + + s"${tableMetadata.identifier}, which is created by Hive and uses the " + + "following unsupported serde configuration\n" + + builder.toString() + ) + } else { + // TODO: should we keep Hive serde properties? + val newStorage = tableMetadata.storage.copy(properties = Map.empty) + tableMetadata.copy(provider = source, storage = newStorage) + } + } +} + +/** + * This commands generates the DDL for Hive serde table. + * + * The syntax of using this command in SQL is: + * {{{ + * SHOW CREATE TABLE table_identifier AS SERDE; + * }}} + */ +case class ShowCreateTableAsSerdeCommand(table: TableIdentifier) + extends RunnableCommand with ShowCreateTableCommandBase { + override val output: Seq[Attribute] = Seq( + AttributeReference("sparktab_stmt", StringType, nullable = false)() + ) + + override def run(sparkSession: SparkSession): Seq[Row] = { + val catalog = sparkSession.sessionState.catalog + val tableMetadata = catalog.getTableMetadata(table) + + val stmt = if (DDLUtils.isDatasourceTable(tableMetadata)) { + throw new AnalysisException( + s"$table is a Spark data source table. Use `SHOW CREATE TABLE` without `AS SERDE` instead.") + } else { + showCreateHiveTable(tableMetadata) + } + + Seq(Row(stmt)) + } + private def showCreateHiveTable(metadata: CatalogTable): String = { def reportUnsupportedError(features: Seq[String]): Unit = { throw new AnalysisException( @@ -1245,85 +1341,3 @@ case class ShowCreateTableCommand(table: TableIdentifier) } } } - -/** - * This commands generates Spark DDL for Hive table. - * - * The syntax of using this command in SQL is: - * {{{ - * SHOW CREATE TABLE table_identifier AS SPARK; - * }}} - */ -case class ShowCreateTableAsSparkCommand(table: TableIdentifier) - extends RunnableCommand with ShowCreateTableCommandBase { - override val output: Seq[Attribute] = Seq( - AttributeReference("sparktab_stmt", StringType, nullable = false)() - ) - - override def run(sparkSession: SparkSession): Seq[Row] = { - val catalog = sparkSession.sessionState.catalog - val tableMetadata = catalog.getTableMetadata(table) - - val stmt = if (DDLUtils.isDatasourceTable(tableMetadata)) { - throw new AnalysisException( - s"$table is already a Spark data source table. Use `SHOW CREATE TABLE` instead.") - } else { - if (tableMetadata.unsupportedFeatures.nonEmpty) { - throw new AnalysisException( - "Failed to execute SHOW CREATE TABLE AS SPARK against table " + - s"${tableMetadata.identifier}, which is created by Hive and uses the " + - "following unsupported feature(s)\n" + - tableMetadata.unsupportedFeatures.map(" - " + _).mkString("\n") - ) - } - - if (tableMetadata.tableType == VIEW) { - throw new AnalysisException("Hive view isn't supported by SHOW CREATE TABLE AS SPARK") - } - - // scalastyle:off caselocale - if (tableMetadata.properties.getOrElse("transactional", "false").toLowerCase.equals("true")) { - throw new AnalysisException( - "SHOW CREATE TABLE AS SPARK doesn't support transactional Hive table") - } - // scalastyle:on caselocale - - showCreateDataSourceTable(convertTableMetadata(tableMetadata)) - } - - Seq(Row(stmt)) - } - - private def convertTableMetadata(tableMetadata: CatalogTable): CatalogTable = { - val hiveSerde = HiveSerDe( - serde = tableMetadata.storage.serde, - inputFormat = tableMetadata.storage.inputFormat, - outputFormat = tableMetadata.storage.outputFormat) - - // Looking for Spark data source that maps to to the Hive serde. - // TODO: some Hive fileformat + row serde might be mapped to Spark data source, e.g. CSV. - val source = HiveSerDe.serdeToSource(hiveSerde) - if (source.isEmpty) { - val builder = StringBuilder.newBuilder - hiveSerde.serde.foreach { serde => - builder ++= s" SERDE: $serde" - } - hiveSerde.inputFormat.foreach { format => - builder ++= s" INPUTFORMAT: $format" - } - hiveSerde.outputFormat.foreach { format => - builder ++= s" OUTPUTFORMAT: $format" - } - throw new AnalysisException( - "Failed to execute SHOW CREATE TABLE AS SPARK against table " + - s"${tableMetadata.identifier}, which is created by Hive and uses the " + - "following unsupported serde configuration\n" + - builder.toString() - ) - } else { - // TODO: should we keep Hive serde properties? - val newStorage = tableMetadata.storage.copy(properties = Map.empty) - tableMetadata.copy(provider = source, storage = newStorage) - } - } -} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala index 6b3ee28ebe804..b3b94f8be0d17 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala @@ -148,20 +148,6 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { } } - test("view") { - withView("v1") { - sql("CREATE VIEW v1 AS SELECT 1 AS a") - checkCreateView("v1") - } - } - - test("view with output columns") { - withView("v1") { - sql("CREATE VIEW v1 (b) AS SELECT 1 AS a") - checkCreateView("v1") - } - } - test("temp view") { val viewName = "spark_28383" withTempView(viewName) { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala index 8dd451eaa2b07..38338824a49ad 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.{AnalysisException, ShowCreateTableSuite} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable} import org.apache.spark.sql.hive.test.TestHiveSingleton -import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.{HiveSerDe, SQLConf} class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSingleton { @@ -40,6 +40,20 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet super.afterAll() } + test("view") { + withView("v1") { + sql("CREATE VIEW v1 AS SELECT 1 AS a") + checkCreateHiveTableOrView("v1", "VIEW") + } + } + + test("view with output columns") { + withView("v1") { + sql("CREATE VIEW v1 (b) AS SELECT 1 AS a") + checkCreateHiveTableOrView("v1", "VIEW") + } + } + test("simple hive table") { withTable("t1") { sql( @@ -54,7 +68,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateTable("t1") + checkCreateHiveTableOrView("t1") } } @@ -74,7 +88,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateTable("t1") + checkCreateHiveTableOrView("t1") } } } @@ -94,7 +108,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateTable("t1") + checkCreateHiveTableOrView("t1") } } @@ -112,7 +126,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateTable("t1") + checkCreateHiveTableOrView("t1") } } @@ -127,7 +141,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateTable("t1") + checkCreateHiveTableOrView("t1") } } @@ -149,7 +163,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateTable("t1") + checkCreateHiveTableOrView("t1") } } @@ -162,7 +176,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet |INTO 2 BUCKETS """.stripMargin ) - checkCreateTable("t1") + checkCreateHiveTableOrView("t1") } } @@ -192,7 +206,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet assert(cause.getMessage.contains(" - partitioned view")) val causeForSpark = intercept[AnalysisException] { - sql("SHOW CREATE TABLE v1 AS SPARK") + sql("SHOW CREATE TABLE v1 AS SERDE") } assert(causeForSpark.getMessage.contains(" - partitioned view")) @@ -202,12 +216,32 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet test("SPARK-24911: keep quotes for nested fields in hive") { withTable("t1") { - val createTable = "CREATE TABLE `t1`(`a` STRUCT<`b`: STRING>) USING hive" + val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>) USING hive" sql(createTable) val shownDDL = getShowDDL("SHOW CREATE TABLE t1") assert(shownDDL == createTable.dropRight(" USING hive".length)) - checkCreateTable("t1") + checkCreateHiveTableOrView("t1") + } + } + + /** + * This method compares the given table with the table created by the DDL generated by + * `SHOW CREATE TABLE AS SERDE`. + */ + private def checkCreateHiveTableOrView(tableName: String, checkType: String = "TABLE"): Unit = { + val table = TableIdentifier(tableName, Some("default")) + val db = table.database.getOrElse("default") + val expected = spark.sharedState.externalCatalog.getTable(db, table.table) + val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString} AS SERDE").head().getString(0) + sql(s"DROP $checkType ${table.quotedString}") + + try { + sql(shownDDL) + val actual = spark.sharedState.externalCatalog.getTable(db, table.table) + checkCatalogTables(expected, actual) + } finally { + sql(s"DROP $checkType IF EXISTS ${table.table}") } } @@ -216,33 +250,25 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet .client.runSqlHive(ddl) } - private def checkCreateSparkTable(tableName: String): Unit = { + private def checkCreateSparkTableAsHive(tableName: String): Unit = { val table = TableIdentifier(tableName, Some("default")) val db = table.database.get val hiveTable = spark.sharedState.externalCatalog.getTable(db, table.table) - val shownSparkDDL = sql(s"SHOW CREATE TABLE ${table.quotedString} AS SPARK").head().getString(0) + val sparkDDL = sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0) // Drops original Hive table. sql(s"DROP TABLE ${table.quotedString}") try { - sql(shownSparkDDL) - val actual = spark.sharedState.externalCatalog.getTable(db, table.table) - val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0) - - // Drops created Spark table using `SHOW CREATE TABLE AS SPARK`. - sql(s"DROP TABLE ${table.quotedString}") - - sql(shownDDL) - val expected = spark.sharedState.externalCatalog.getTable(db, table.table) - - checkCatalogTables(expected, actual) - checkHiveCatalogTables(hiveTable, actual) + // Creates Spark datasource table using generated Spark DDL. + sql(sparkDDL) + val sparkTable = spark.sharedState.externalCatalog.getTable(db, table.table) + checkHiveCatalogTables(hiveTable, sparkTable) } finally { sql(s"DROP TABLE IF EXISTS ${table.table}") } } - private def checkHiveCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { + private def checkHiveCatalogTables(hiveTable: CatalogTable, sparkTable: CatalogTable): Unit = { def normalize(table: CatalogTable): CatalogTable = { val nondeterministicProps = Set( "CreateTime", @@ -264,37 +290,47 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet properties = table.properties.filterKeys(!nondeterministicProps.contains(_)), stats = None, ignoredProperties = Map.empty, - storage = CatalogStorageFormat.empty, + storage = table.storage.copy(properties = Map.empty), provider = None, tracksPartitionsInCatalog = false ) } - assert(normalize(actual) == normalize(expected)) - } - test("simple hive table as spark") { - withSQLConf( - SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") { - withTable("t1") { - sql( - s""" - |CREATE TABLE t1 ( - | c1 STRING COMMENT 'bla', - | c2 STRING - |) - |TBLPROPERTIES ( - | 'prop1' = 'value1', - | 'prop2' = 'value2' - |) - """.stripMargin + def fillSerdeFromProvider(table: CatalogTable): CatalogTable = { + table.provider.flatMap(HiveSerDe.sourceToSerDe(_)).map { hiveSerde => + val newStorage = table.storage.copy( + inputFormat = hiveSerde.inputFormat, + outputFormat = hiveSerde.outputFormat, + serde = hiveSerde.serde ) + table.copy(storage = newStorage) + }.getOrElse(table) + } - checkCreateSparkTable("t1") - } + assert(normalize(fillSerdeFromProvider(sparkTable)) == normalize(hiveTable)) + } + + test("simple hive table in Spark DDL") { + withTable("t1") { + sql( + s""" + |CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |TBLPROPERTIES ( + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + |STORED AS orc + """.stripMargin + ) + + checkCreateSparkTableAsHive("t1") } } - test("show create table as spark can't work on data source table") { + test("show create table as serde can't work on data source table") { withTable("t1") { sql( s""" @@ -307,39 +343,37 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet ) val cause = intercept[AnalysisException] { - checkCreateSparkTable("t1") + checkCreateHiveTableOrView("t1") } - assert(cause.getMessage.contains("Use `SHOW CREATE TABLE` instead")) + assert(cause.getMessage.contains("Use `SHOW CREATE TABLE` without `AS SERDE` instead")) } } - test("simple external hive table as spark") { - withSQLConf( - SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") { - withTempDir { dir => - withTable("t1") { - sql( - s""" - |CREATE TABLE t1 ( - | c1 STRING COMMENT 'bla', - | c2 STRING - |) - |LOCATION '${dir.toURI}' - |TBLPROPERTIES ( - | 'prop1' = 'value1', - | 'prop2' = 'value2' - |) - """.stripMargin - ) - - checkCreateSparkTable("t1") - } + test("simple external hive table in Spark DDL") { + withTempDir { dir => + withTable("t1") { + sql( + s""" + |CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |LOCATION '${dir.toURI}' + |TBLPROPERTIES ( + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + |STORED AS orc + """.stripMargin + ) + + checkCreateSparkTableAsHive("t1") } } } - test("hive table with STORED AS clause as spark") { + test("hive table with STORED AS clause in Spark DDL") { withTable("t1") { sql( s""" @@ -351,11 +385,11 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateSparkTable("t1") + checkCreateSparkTableAsHive("t1") } } - test("hive table with unsupported fileformat as spark") { + test("hive table with unsupported fileformat in Spark DDL") { withTable("t1") { sql( s""" @@ -368,14 +402,14 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet ) val cause = intercept[AnalysisException] { - checkCreateSparkTable("t1") + checkCreateSparkTableAsHive("t1") } assert(cause.getMessage.contains("unsupported serde configuration")) } } - test("hive table with serde info as spark") { + test("hive table with serde info in Spark DDL") { withTable("t1") { sql( s""" @@ -390,11 +424,11 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateSparkTable("t1") + checkCreateSparkTableAsHive("t1") } } - test("hive view is not supported as spark") { + test("hive view is not supported by show create table without as serde") { withTable("t1") { withView("v1") { sql("CREATE TABLE t1 (c1 STRING, c2 STRING)") @@ -407,7 +441,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet ) val cause = intercept[AnalysisException] { - sql("SHOW CREATE TABLE v1 AS SPARK") + sql("SHOW CREATE TABLE v1") } assert(cause.getMessage.contains("view isn't supported")) @@ -415,7 +449,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet } } - test("partitioned, bucketed hive table as spark") { + test("partitioned, bucketed hive table in Spark DDL") { withTable("t1") { sql( s""" @@ -432,38 +466,35 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet """.stripMargin ) - checkCreateSparkTable("t1") + checkCreateSparkTableAsHive("t1") } } - test("transactional hive table as spark") { - withSQLConf( - SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") { - withTable("t1") { - sql( - s""" - |CREATE TABLE t1 ( - | c1 STRING COMMENT 'bla', - | c2 STRING - |) - |TBLPROPERTIES ( - | 'transactional' = 'true', - | 'prop1' = 'value1', - | 'prop2' = 'value2' - |) - |CLUSTERED BY (c1) INTO 10 BUCKETS - |STORED AS ORC - """.stripMargin - ) - + test("show create table for transactional hive table") { + withTable("t1") { + sql( + s""" + |CREATE TABLE t1 ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |TBLPROPERTIES ( + | 'transactional' = 'true', + | 'prop1' = 'value1', + | 'prop2' = 'value2' + |) + |CLUSTERED BY (c1) INTO 10 BUCKETS + |STORED AS ORC + """.stripMargin + ) - val cause = intercept[AnalysisException] { - sql("SHOW CREATE TABLE t1 AS SPARK") - } - assert(cause.getMessage.contains( - "SHOW CREATE TABLE AS SPARK doesn't support transactional Hive table")) + val cause = intercept[AnalysisException] { + sql("SHOW CREATE TABLE t1") } + + assert(cause.getMessage.contains( + "SHOW CREATE TABLE doesn't support transactional Hive table")) } } } From 115dce32b1e819cb85278b3006843df455df0410 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Jan 2020 08:44:13 -0800 Subject: [PATCH 10/16] Address comment and update existing tests. --- .../spark/sql/execution/command/tables.scala | 7 ++-- .../sql-tests/inputs/show-create-table.sql | 11 ++++-- .../results/show-create-table.sql.out | 34 ++++++++++++++++--- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index efdc4cadebfd9..c5bb0c70fb845 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -1136,13 +1136,10 @@ case class ShowCreateTableCommand(table: TableIdentifier) throw new AnalysisException("Hive view isn't supported by SHOW CREATE TABLE") } - // scalastyle:off caselocale - if (tableMetadata.properties.getOrElse("transactional", "false") - .toLowerCase.equals("true")) { + if ("true".equalsIgnoreCase(tableMetadata.properties.getOrElse("transactional", "false"))) { throw new AnalysisException( "SHOW CREATE TABLE doesn't support transactional Hive table") } - // scalastyle:on caselocale convertTableMetadata(tableMetadata) } @@ -1198,7 +1195,7 @@ case class ShowCreateTableCommand(table: TableIdentifier) case class ShowCreateTableAsSerdeCommand(table: TableIdentifier) extends RunnableCommand with ShowCreateTableCommandBase { override val output: Seq[Attribute] = Seq( - AttributeReference("sparktab_stmt", StringType, nullable = false)() + AttributeReference("createtab_stmt", StringType, nullable = false)() ) override def run(sparkSession: SparkSession): Seq[Row] = { diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql index ccb40f8d991b4..dc77f87d9743a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql @@ -73,7 +73,7 @@ CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet; CREATE VIEW view_SPARK_30302 (aaa, bbb) AS SELECT a, b FROM tbl; -SHOW CREATE TABLE view_SPARK_30302; +SHOW CREATE TABLE view_SPARK_30302 AS SERDE; DROP VIEW view_SPARK_30302; @@ -82,7 +82,7 @@ CREATE VIEW view_SPARK_30302 (aaa COMMENT 'comment with \'quoted text\' for aaa' COMMENT 'This is a comment with \'quoted text\' for view' AS SELECT a, b FROM tbl; -SHOW CREATE TABLE view_SPARK_30302; +SHOW CREATE TABLE view_SPARK_30302 AS SERDE; DROP VIEW view_SPARK_30302; @@ -91,6 +91,13 @@ CREATE VIEW view_SPARK_30302 (aaa, bbb) TBLPROPERTIES ('a' = '1', 'b' = '2') AS SELECT a, b FROM tbl; +SHOW CREATE TABLE view_SPARK_30302 AS SERDE; +DROP VIEW view_SPARK_30302; + +-- SHOW CREATE TABLE does not support view +CREATE VIEW view_SPARK_30302 (aaa, bbb) +AS SELECT a, b FROM tbl; + SHOW CREATE TABLE view_SPARK_30302; DROP VIEW view_SPARK_30302; diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out index 5771f218e3b57..e8ee07171651d 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 38 +-- Number of queries: 41 -- !query @@ -291,7 +291,7 @@ struct<> -- !query -SHOW CREATE TABLE view_SPARK_30302 +SHOW CREATE TABLE view_SPARK_30302 AS SERDE -- !query schema struct -- !query output @@ -320,7 +320,7 @@ struct<> -- !query -SHOW CREATE TABLE view_SPARK_30302 +SHOW CREATE TABLE view_SPARK_30302 AS SERDE -- !query schema struct -- !query output @@ -350,7 +350,7 @@ struct<> -- !query -SHOW CREATE TABLE view_SPARK_30302 +SHOW CREATE TABLE view_SPARK_30302 AS SERDE -- !query schema struct -- !query output @@ -371,6 +371,32 @@ struct<> +-- !query +CREATE VIEW view_SPARK_30302 (aaa, bbb) +AS SELECT a, b FROM tbl +-- !query schema +struct<> +-- !query output + + + +-- !query +SHOW CREATE TABLE view_SPARK_30302 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +Hive view isn't supported by SHOW CREATE TABLE; + + +-- !query +DROP VIEW view_SPARK_30302 +-- !query schema +struct<> +-- !query output + + + -- !query DROP TABLE tbl -- !query schema From 88feda3b4bfcc0343ccb8072307ee82d4e091639 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Jan 2020 08:46:20 -0800 Subject: [PATCH 11/16] Revert unnecessary change. --- .../test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala index b3b94f8be0d17..2b441900391f7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala @@ -211,7 +211,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { } } - protected def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { + private def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { def normalize(table: CatalogTable): CatalogTable = { val nondeterministicProps = Set( "CreateTime", From 67cbea2266f7eef7577e07e5c3b328e8db02823a Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Jan 2020 09:04:22 -0800 Subject: [PATCH 12/16] oops. --- .../test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala index 2b441900391f7..b3b94f8be0d17 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala @@ -211,7 +211,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { } } - private def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { + protected def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { def normalize(table: CatalogTable): CatalogTable = { val nondeterministicProps = Set( "CreateTime", From 431195522186502bc62e516d871dcc055ed2db2b Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Jan 2020 14:40:50 -0800 Subject: [PATCH 13/16] Move some methods. --- .../spark/sql/execution/command/tables.scala | 105 +++++++++--------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index c5bb0c70fb845..8c8c25d2d64c7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -1035,58 +1035,6 @@ trait ShowCreateTableCommandBase { } } - protected def showDataSourceTableDataColumns( - metadata: CatalogTable, builder: StringBuilder): Unit = { - val columns = metadata.schema.fields.map(_.toDDL) - builder ++= concatByMultiLines(columns) - } - - protected def showDataSourceTableOptions(metadata: CatalogTable, builder: StringBuilder): Unit = { - builder ++= s"USING ${metadata.provider.get}\n" - - val dataSourceOptions = SQLConf.get.redactOptions(metadata.storage.properties).map { - case (key, value) => s"${quoteIdentifier(key)} '${escapeSingleQuotedString(value)}'" - } - - if (dataSourceOptions.nonEmpty) { - builder ++= "OPTIONS " - builder ++= concatByMultiLines(dataSourceOptions) - } - } - - protected def showDataSourceTableNonDataColumns( - metadata: CatalogTable, builder: StringBuilder): Unit = { - val partCols = metadata.partitionColumnNames - if (partCols.nonEmpty) { - builder ++= s"PARTITIONED BY ${partCols.mkString("(", ", ", ")")}\n" - } - - metadata.bucketSpec.foreach { spec => - if (spec.bucketColumnNames.nonEmpty) { - builder ++= s"CLUSTERED BY ${spec.bucketColumnNames.mkString("(", ", ", ")")}\n" - - if (spec.sortColumnNames.nonEmpty) { - builder ++= s"SORTED BY ${spec.sortColumnNames.mkString("(", ", ", ")")}\n" - } - - builder ++= s"INTO ${spec.numBuckets} BUCKETS\n" - } - } - } - - protected def showCreateDataSourceTable(metadata: CatalogTable): String = { - val builder = StringBuilder.newBuilder - - builder ++= s"CREATE TABLE ${table.quotedString} " - showDataSourceTableDataColumns(metadata, builder) - showDataSourceTableOptions(metadata, builder) - showDataSourceTableNonDataColumns(metadata, builder) - showTableComment(metadata, builder) - showTableLocation(metadata, builder) - showTableProperties(metadata, builder) - - builder.toString() - } protected def concatByMultiLines(iter: Iterable[String]): String = { iter.mkString("(\n ", ",\n ", ")\n") @@ -1182,6 +1130,59 @@ case class ShowCreateTableCommand(table: TableIdentifier) tableMetadata.copy(provider = source, storage = newStorage) } } + + private def showDataSourceTableDataColumns( + metadata: CatalogTable, builder: StringBuilder): Unit = { + val columns = metadata.schema.fields.map(_.toDDL) + builder ++= concatByMultiLines(columns) + } + + private def showDataSourceTableOptions(metadata: CatalogTable, builder: StringBuilder): Unit = { + builder ++= s"USING ${metadata.provider.get}\n" + + val dataSourceOptions = SQLConf.get.redactOptions(metadata.storage.properties).map { + case (key, value) => s"${quoteIdentifier(key)} '${escapeSingleQuotedString(value)}'" + } + + if (dataSourceOptions.nonEmpty) { + builder ++= "OPTIONS " + builder ++= concatByMultiLines(dataSourceOptions) + } + } + + private def showDataSourceTableNonDataColumns( + metadata: CatalogTable, builder: StringBuilder): Unit = { + val partCols = metadata.partitionColumnNames + if (partCols.nonEmpty) { + builder ++= s"PARTITIONED BY ${partCols.mkString("(", ", ", ")")}\n" + } + + metadata.bucketSpec.foreach { spec => + if (spec.bucketColumnNames.nonEmpty) { + builder ++= s"CLUSTERED BY ${spec.bucketColumnNames.mkString("(", ", ", ")")}\n" + + if (spec.sortColumnNames.nonEmpty) { + builder ++= s"SORTED BY ${spec.sortColumnNames.mkString("(", ", ", ")")}\n" + } + + builder ++= s"INTO ${spec.numBuckets} BUCKETS\n" + } + } + } + + private def showCreateDataSourceTable(metadata: CatalogTable): String = { + val builder = StringBuilder.newBuilder + + builder ++= s"CREATE TABLE ${table.quotedString} " + showDataSourceTableDataColumns(metadata, builder) + showDataSourceTableOptions(metadata, builder) + showDataSourceTableNonDataColumns(metadata, builder) + showTableComment(metadata, builder) + showTableLocation(metadata, builder) + showTableProperties(metadata, builder) + + builder.toString() + } } /** From f886f04f96136c1d85921777f6bdfe19f8840aba Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Jan 2020 15:04:47 -0800 Subject: [PATCH 14/16] Address comment. --- .../scala/org/apache/spark/sql/execution/command/tables.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 8c8c25d2d64c7..468ca505cce1f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -1119,7 +1119,7 @@ case class ShowCreateTableCommand(table: TableIdentifier) builder ++= s" OUTPUTFORMAT: $format" } throw new AnalysisException( - "Failed to execute SHOW CREATE TABLE AS SPARK against table " + + "Failed to execute SHOW CREATE TABLE against table " + s"${tableMetadata.identifier}, which is created by Hive and uses the " + "following unsupported serde configuration\n" + builder.toString() @@ -1138,6 +1138,8 @@ case class ShowCreateTableCommand(table: TableIdentifier) } private def showDataSourceTableOptions(metadata: CatalogTable, builder: StringBuilder): Unit = { + // For datasource table, there is a provider there in the metadata. + // If it is a Hive table, we already convert its metadata and fill in a provider. builder ++= s"USING ${metadata.provider.get}\n" val dataSourceOptions = SQLConf.get.redactOptions(metadata.storage.properties).map { From 9882932bb275d287dfc3cea3d52bb7903e25e73f Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Jan 2020 15:56:45 -0800 Subject: [PATCH 15/16] Add test and migration guide. --- docs/sql-migration-guide.md | 2 ++ .../sql/hive/HiveShowCreateTableSuite.scala | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index 18e7df7aca5aa..aa435e91cb7d0 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -328,6 +328,8 @@ license: | - Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if the table does not exist. In Spark version 2.4 and earlier, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned an empty result. + - Since Spark 3.0, `SHOW CREATE TABLE` will return Spark DDL if the given table is a Hive serde table. For Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead. + ## Upgrading from Spark SQL 2.4.4 to 2.4.5 - Since Spark 2.4.5, `TRUNCATE TABLE` command tries to set back original permission and ACLs during re-creating the table/partition paths. To restore the behaviour of earlier versions, set `spark.sql.truncateTable.ignorePermissionAcl.enabled` to `true`. diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala index 38338824a49ad..e5d572c90af38 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala @@ -389,6 +389,23 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet } } + test("hive table with nested fields with STORED AS clause in Spark DDL") { + withTable("t1") { + sql( + s""" + |CREATE TABLE t1 ( + | c1 INT COMMENT 'bla', + | c2 STRING, + | c3 STRUCT + |) + |STORED AS PARQUET + """.stripMargin + ) + + checkCreateSparkTableAsHive("t1") + } + } + test("hive table with unsupported fileformat in Spark DDL") { withTable("t1") { sql( From 27c76b3b5e9106f0fe7de1ccb2c8576064e625da Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 31 Jan 2020 16:13:19 -0800 Subject: [PATCH 16/16] Update migration guide. --- docs/sql-migration-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index aa435e91cb7d0..533c96a0832de 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -328,7 +328,7 @@ license: | - Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if the table does not exist. In Spark version 2.4 and earlier, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned an empty result. - - Since Spark 3.0, `SHOW CREATE TABLE` will return Spark DDL if the given table is a Hive serde table. For Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead. + - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead. ## Upgrading from Spark SQL 2.4.4 to 2.4.5