From 52579832aea7cabf21dcf3c5ead7c0d2cba71474 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Fri, 24 Feb 2017 13:54:28 -0800 Subject: [PATCH 01/19] Add tests to cover scenarios for the column resolution --- .../spark/sql/ColumnResolutionSuite.scala | 101 +++++ .../execution/ColumnResolutionSuite.scala | 346 ++++++++++++++++++ 2 files changed, 447 insertions(+) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala new file mode 100644 index 000000000000..8b611669f143 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import java.io.File + +import org.apache.spark.sql.test.SharedSQLContext + + +class ColumnResolutionSuite extends QueryTest with SharedSQLContext { + + import testImplicits._ + + def columnResolutionTests(db1: String, db2: String): Unit = { + spark.catalog.setCurrentDatabase(db1) + checkAnswer(spark.sql("SELECT i1 FROM t1"), Row(1)) + checkAnswer(spark.sql(s"SELECT i1 FROM $db1.t1"), Row(1)) + + checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(1)) + checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1") + } + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") + } + + // Change current database to db2 + spark.catalog.setCurrentDatabase(db2) + checkAnswer(spark.sql("SELECT i1 FROM t1"), Row(20)) + checkAnswer(spark.sql(s"SELECT i1 FROM $db1.t1"), Row(1)) + + checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(20)) + checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) + + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1") + } + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") + } + } + + test("column resolution scenarios with datasource table") { + val currentDb = spark.catalog.currentDatabase + withTempDatabase { db1 => + withTempDatabase { db2 => + withTempDir { f => + try { + val df = Seq(1).toDF() + val path = s"${f.getCanonicalPath}${File.separator}test1" + df.write.csv(path) + spark.catalog.setCurrentDatabase(db1) + + sql( + s""" + |CREATE TABLE t1(i1 INT) USING csv OPTIONS + |(path "$path", header "false") + """.stripMargin) + + spark.catalog.setCurrentDatabase(db2) + val df2 = Seq(20).toDF() + val path2 = s"${f.getCanonicalPath}${File.separator}test2" + df2.write.csv(path2) + + sql( + s""" + |CREATE TABLE t1(i1 INT) USING csv OPTIONS + |(path "$path2", header "false") + """.stripMargin) + + columnResolutionTests(db1, db2) + } finally { + spark.catalog.setCurrentDatabase (currentDb) + } + } + } + } + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala new file mode 100644 index 000000000000..ef65f074eba4 --- /dev/null +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.execution + +import java.io.File + +import org.apache.spark.sql.{AnalysisException, QueryTest, Row} +import org.apache.spark.sql.hive.test.TestHiveSingleton +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.test.SQLTestUtils + +class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSingleton { + + import spark.implicits._ + + def columnResolutionTests(db1: String, db2: String): Unit = { + spark.catalog.setCurrentDatabase(db1) + + checkAnswer(spark.sql("SELECT i1 FROM t1"), Row(1)) + checkAnswer(spark.sql(s"SELECT i1 FROM $db1.t1"), Row(1)) + + checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(1)) + checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1") + } + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") + } + + // Change current database to db2 + spark.catalog.setCurrentDatabase(db2) + checkAnswer(spark.sql("SELECT i1 FROM t1"), Row(20)) + checkAnswer(spark.sql(s"SELECT i1 FROM $db1.t1"), Row(1)) + + checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(20)) + checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) + + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1") + } + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") + } + } + + test("column resolution scenarios with non datasource table") { + val currentDb = spark.catalog.currentDatabase + withTempDatabase { db1 => + withTempDatabase { db2 => + try { + spark.catalog.setCurrentDatabase(db1) + spark.sql("CREATE TABLE t1(i1 INT)") + spark.sql("INSERT INTO t1 VALUES(1)") + spark.catalog.setCurrentDatabase(db2) + spark.sql("CREATE TABLE t1(i1 INT)") + spark.sql("INSERT INTO t1 VALUES(20)") + + columnResolutionTests(db1, db2) + } finally { + spark.catalog.setCurrentDatabase(currentDb) + } + } + } + } + + test("column resolution scenarios with datasource table") { + val currentDb = spark.catalog.currentDatabase + withTempDatabase { db1 => + withTempDatabase { db2 => + withTempDir { f => + try { + val df = Seq(1).toDF() + val path = s"${f.getCanonicalPath}${File.separator}test1" + df.write.csv(path) + spark.catalog.setCurrentDatabase(db1) + + sql( + s""" + |CREATE TABLE t1(i1 INT) USING csv OPTIONS + |(path "$path", header "false") + """.stripMargin) + + spark.catalog.setCurrentDatabase(db2) + val df2 = Seq(20).toDF() + val path2 = s"${f.getCanonicalPath}${File.separator}test2" + df2.write.csv(path2) + + sql( + s""" + |CREATE TABLE t1(i1 INT) USING csv OPTIONS + |(path "$path2", header "false") + """.stripMargin) + + columnResolutionTests(db1, db2) + } finally { + spark.catalog.setCurrentDatabase (currentDb) + } + } + } + } + } + + test("column resolution scenarios with ambiguous cases") { + val currentDb = spark.catalog.currentDatabase + withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") { + withTempDatabase { db1 => + withTempDatabase { db2 => + withTempPath { f => + try { + spark.catalog.setCurrentDatabase(db1) + spark.sql("CREATE TABLE t1(i1 INT)") + spark.sql("INSERT INTO t1 VALUES(1)") + spark.catalog.setCurrentDatabase(db2) + spark.sql("CREATE TABLE t1(i1 INT)") + spark.sql("INSERT INTO t1 VALUES(20)") + + spark.catalog.setCurrentDatabase(db1) + + intercept[AnalysisException] { + spark.sql(s"SELECT i1 FROM t1, $db1.t1") + } + + intercept[AnalysisException] { + spark.sql(s"SELECT t1.i1 FROM t1, $db1.t1") + } + + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db1.t1") + } + + intercept[AnalysisException] { + spark.sql(s"SELECT i1 FROM t1, $db2.t1") + } + + intercept[AnalysisException] { + spark.sql(s"SELECT t1.i1 FROM t1, $db2.t1") + } + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db2.t1") + } + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1, $db2.t1") + } + + spark.catalog.setCurrentDatabase(db2) + + intercept[AnalysisException] { + spark.sql(s"SELECT i1 FROM t1, $db1.t1") + } + + intercept[AnalysisException] { + spark.sql(s"SELECT t1.i1 FROM t1, $db1.t1") + } + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db1.t1") + } + + intercept[AnalysisException] { + spark.sql(s"SELECT i1 FROM t1, $db2.t1") + } + + intercept[AnalysisException] { + spark.sql(s"SELECT t1.i1 FROM t1, $db2.t1") + } + + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db2.t1") + } + } finally { + spark.catalog.setCurrentDatabase(currentDb) + } + } + } + } + } + } + + test("resolve fully qualified table name in star expansion ") { + val currentDb = spark.catalog.currentDatabase + withTempDatabase { db1 => + withTempDatabase { db2 => + withTempPath { f => + try { + val df = spark.range(1).toDF() + df.write.csv(f.getCanonicalPath) + spark.catalog.setCurrentDatabase(db1) + + sql( + s""" + |CREATE TABLE t1(i1 INT) USING csv OPTIONS + |(path "${f.getCanonicalPath}", header "false") + """.stripMargin) + + spark.catalog.setCurrentDatabase(db2) + spark.sql("CREATE TABLE t1(i1 INT)") + spark.sql("INSERT INTO t1 VALUES(20)") + + spark.catalog.setCurrentDatabase(db1) + checkAnswer(spark.sql("SELECT t1.* FROM t1"), Row(0)) + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.* FROM $db1.t1") + } + + checkAnswer(spark.sql(s"SELECT t1.* FROM $db1.t1"), Row(0)) + + spark.catalog.setCurrentDatabase(db2) + checkAnswer(spark.sql("SELECT t1.* FROM t1"), Row(20)) + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.* FROM $db1.t1") + } + + checkAnswer(spark.sql(s"SELECT t1.* FROM $db1.t1"), Row(0)) + checkAnswer(spark.sql(s"SELECT a.* FROM $db1.t1 AS a"), Row(0)) + + } finally { + spark.catalog.setCurrentDatabase(currentDb) + } + } + } + } + } + + test("resolve in case of subquery") { + val currentDb = spark.catalog.currentDatabase + withTempDatabase { db1 => + withTempDir { f => + try { + val df = Seq((4, 1), (3, 1)).toDF() + val path = s"${f.getCanonicalPath}${File.separator}test1" + df.write.csv(path) + spark.catalog.setCurrentDatabase(db1) + + sql( + s""" + |CREATE TABLE t3(c1 INT, c2 INT) USING csv OPTIONS + |(path "$path", header "false") + """.stripMargin) + + val df2 = Seq((4, 1), (2, 1)).toDF() + val path2 = s"${f.getCanonicalPath}${File.separator}test2" + df2.write.csv(path2) + + sql( + s""" + |CREATE TABLE t4(c2 INT, c3 INT) USING csv OPTIONS + |(path "$path2", header "false") + """.stripMargin) + + checkAnswer(spark.sql("SELECT * FROM t3 WHERE c1 IN " + + "(SELECT c2 FROM t4 WHERE t4.c3 = t3.c2)"), Row(4, 1)) + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT * FROM $db1.t3 WHERE c1 IN " + + s"(SELECT $db1.t4.c2 FROM $db1.t4 WHERE $db1.t4.c3 = $db1.t3.c2)") + } + + } finally { + spark.catalog.setCurrentDatabase(currentDb) + } + } + } + } + + test("col resolution - error case") { + val currentDb = spark.catalog.currentDatabase + withTempDatabase { db1 => + withTempPath { f => + try { + spark.catalog.setCurrentDatabase(db1) + spark.sql("CREATE TABLE t1(i1 INT)") + spark.sql("INSERT INTO t1 VALUES(1)") + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1 FROM t1") + } + intercept[AnalysisException] { + spark.sql(s"SELECT t1.x.y.* FROM t1") + } + intercept[AnalysisException] { + spark.sql(s"SELECT t1 FROM $db1.t1") + } + } finally { + spark.catalog.setCurrentDatabase(currentDb) + } + } + } + } + + test("Table with struct column") { + val currentDb = spark.catalog.currentDatabase + withTempDatabase { db1 => + try { + spark.catalog.setCurrentDatabase(db1) + spark.sql("CREATE TABLE t1(i1 INT, t1 struct)") + spark.sql("INSERT INTO t1 VALUES(1, (2, 3))") + checkAnswer(spark.sql(s"SELECT t1.i1 FROM t1"), Row(1)) + checkAnswer(spark.sql(s"SELECT t1.t1.i1 FROM t1"), Row(2)) + checkAnswer(spark.sql(s"SELECT t1.t1.i1 FROM $db1.t1"), Row(2)) + checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.t1.i1 FROM $db1.t1") + } + + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.t1.i2 FROM $db1.t1") + } + } finally { + spark.catalog.setCurrentDatabase(currentDb) + } + } + } +} \ No newline at end of file From 685a53d3b54a22221193b264475026bfaa005972 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Fri, 24 Feb 2017 14:01:16 -0800 Subject: [PATCH 02/19] Add tests for global temp views --- .../sql/execution/GlobalTempViewSuite.scala | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala index 5c63c6a414f9..122d3ce5ec80 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala @@ -150,6 +150,32 @@ class GlobalTempViewSuite extends QueryTest with SharedSQLContext { } } + test("column resolution scenarios with global temp view") { + val df = Seq(1).toDF("i1") + df.createGlobalTempView("t1") + try { + checkAnswer(spark.sql(s"SELECT * FROM $globalTempDB.t1"), Row(1)) + + // TODO: Fix this scenario + intercept[AnalysisException] { + checkAnswer(spark.sql(s"SELECT $globalTempDB.t1.* FROM $globalTempDB.t1"), Row(1)) + } + + checkAnswer(spark.sql(s"SELECT i1 FROM $globalTempDB.t1"), Row(1)) + + // TODO: Fix this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $globalTempDB.t1.i1 FROM $globalTempDB.t1") + } + + checkAnswer(spark.sql(s"SELECT t1.i1 FROM $globalTempDB.t1"), Row(1)) + checkAnswer(spark.sql(s"SELECT a.i1 FROM $globalTempDB.t1 AS a"), Row(1)) + checkAnswer(spark.sql(s"SELECT i1 FROM $globalTempDB.t1 AS a"), Row(1)) + } finally { + spark.catalog.dropGlobalTempView("t1") + } + } + test("public Catalog should recognize global temp view") { try { sql("CREATE GLOBAL TEMP VIEW src AS SELECT 1, 2") From 596b3e8c694818853b928bb2d1566c606da748f1 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Fri, 24 Feb 2017 14:15:41 -0800 Subject: [PATCH 03/19] Add test with local view --- .../apache/spark/sql/execution/SQLViewSuite.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala index 2d95cb6d64a8..1f22eca557c0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala @@ -52,6 +52,19 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils { } } + test("column resolution scenarios with local temp view") { + val df = Seq(2).toDF("i1") + df.createOrReplaceTempView("table1") + withTempView("table1") { + checkAnswer(spark.sql("SELECT table1.* FROM table1"), Row(2)) + checkAnswer(spark.sql("SELECT * FROM table1"), Row(2)) + checkAnswer(spark.sql("SELECT i1 FROM table1"), Row(2)) + checkAnswer(spark.sql("SELECT table1.i1 FROM table1"), Row(2)) + checkAnswer(spark.sql("SELECT a.i1 FROM table1 AS a"), Row(2)) + checkAnswer(spark.sql("SELECT i1 FROM table1 AS a"), Row(2)) + } + } + test("create a temp view on a permanent view") { withView("jtv1", "temp_jtv1") { sql("CREATE VIEW jtv1 AS SELECT * FROM jt WHERE id > 3") From cdd1468bfdc50837aaca92eef344ea149b2654a5 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Fri, 24 Feb 2017 17:14:52 -0800 Subject: [PATCH 04/19] newline in end --- .../apache/spark/sql/hive/execution/ColumnResolutionSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala index ef65f074eba4..56148d68e59c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala @@ -343,4 +343,4 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin } } } -} \ No newline at end of file +} From e95c33d43e10367db4081ddf6cfb01eb1bd2b573 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Fri, 24 Feb 2017 18:05:30 -0800 Subject: [PATCH 05/19] uppercase struct --- .../apache/spark/sql/hive/execution/ColumnResolutionSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala index 56148d68e59c..a051f29f77d0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala @@ -322,7 +322,7 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin withTempDatabase { db1 => try { spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1(i1 INT, t1 struct)") + spark.sql("CREATE TABLE t1(i1 INT, t1 STRUCT)") spark.sql("INSERT INTO t1 VALUES(1, (2, 3))") checkAnswer(spark.sql(s"SELECT t1.i1 FROM t1"), Row(1)) checkAnswer(spark.sql(s"SELECT t1.t1.i1 FROM t1"), Row(2)) From 715bc042f9285b6725858127905504d82cd53f99 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Sat, 25 Feb 2017 14:23:18 -0800 Subject: [PATCH 06/19] fix style and use ctas --- .../sql/execution/GlobalTempViewSuite.scala | 1 - .../execution/ColumnResolutionSuite.scala | 22 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala index 122d3ce5ec80..f30304145536 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala @@ -160,7 +160,6 @@ class GlobalTempViewSuite extends QueryTest with SharedSQLContext { intercept[AnalysisException] { checkAnswer(spark.sql(s"SELECT $globalTempDB.t1.* FROM $globalTempDB.t1"), Row(1)) } - checkAnswer(spark.sql(s"SELECT i1 FROM $globalTempDB.t1"), Row(1)) // TODO: Fix this scenario diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala index a051f29f77d0..a75cf4904847 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala @@ -130,12 +130,9 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin withTempPath { f => try { spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1(i1 INT)") - spark.sql("INSERT INTO t1 VALUES(1)") + spark.sql("CREATE TABLE t1 AS SELECT 1 AS i1") spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1(i1 INT)") - spark.sql("INSERT INTO t1 VALUES(20)") - + spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") spark.catalog.setCurrentDatabase(db1) intercept[AnalysisException] { @@ -219,8 +216,7 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin """.stripMargin) spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1(i1 INT)") - spark.sql("INSERT INTO t1 VALUES(20)") + spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") spark.catalog.setCurrentDatabase(db1) checkAnswer(spark.sql("SELECT t1.* FROM t1"), Row(0)) @@ -277,13 +273,17 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin |(path "$path2", header "false") """.stripMargin) - checkAnswer(spark.sql("SELECT * FROM t3 WHERE c1 IN " + - "(SELECT c2 FROM t4 WHERE t4.c3 = t3.c2)"), Row(4, 1)) + checkAnswer( + spark.sql("SELECT * FROM t3 WHERE c1 IN (SELECT c2 FROM t4 WHERE t4.c3 = t3.c2)"), + Row(4, 1)) // TODO: Support this scenario intercept[AnalysisException] { - spark.sql(s"SELECT * FROM $db1.t3 WHERE c1 IN " + - s"(SELECT $db1.t4.c2 FROM $db1.t4 WHERE $db1.t4.c3 = $db1.t3.c2)") + spark.sql( + s""" + |SELECT * FROM $db1.t3 WHERE c1 IN + |(SELECT $db1.t4.c2 FROM $db1.t4 WHERE $db1.t4.c3 = $db1.t3.c2) + """.stripMargin) } } finally { From 040e7dd25ce9ee2fa39516529c374b015440a475 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Mon, 27 Feb 2017 18:06:04 -0800 Subject: [PATCH 07/19] Add column resolution tests to the SQLQueryTestSuite and also separate out the -ve cases from +ve --- .../sql-tests/inputs/columnresolution.sql | 108 ++++ .../results/columnresolution.sql.out | 588 ++++++++++++++++++ .../spark/sql/ColumnResolutionSuite.scala | 4 - .../execution/ColumnResolutionSuite.scala | 100 +-- 4 files changed, 758 insertions(+), 42 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql new file mode 100644 index 000000000000..a545ddcd1a0a --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql @@ -0,0 +1,108 @@ +-- Scenario: column resolution scenarios with datasource table +CREATE DATABASE mydb1; +use mydb1; +CREATE TABLE t1 USING parquet as SELECT 1 as i1; + +CREATE DATABASE mydb2; +use mydb2; +CREATE TABLE t1 USING parquet as SELECT 20 as i1; + +USE mydb1; +SELECT i1 FROM t1; +SELECT i1 FROM mydb1.t1; +SELECT t1.i1 FROM t1; +SELECT t1.i1 FROM mydb1.t1; + +-- TODO: Support this scenario +SELECT mydb1.t1.i1 FROM t1; +-- TODO: Support this scenario +SELECT mydb1.t1.i1 FROM mydb1.t1; + +USE mydb2; +SELECT i1 FROM t1; +SELECT i1 FROM mydb1.t1; +SELECT t1.i1 FROM t1; +SELECT t1.i1 FROM mydb1.t1; + +-- TODO: Support this scenario +SELECT mydb1.t1.i1 FROM mydb1.t1; + + +-- Scenario: resolve fully qualified table name in star expansion +use mydb1; +SELECT t1.* FROM t1; +SELECT mydb1.t1.* FROM mydb1.t1; +SELECT t1.* FROM mydb1.t1; +use mydb2; +SELECT t1.* FROM t1; + +-- TODO: Support this scenario +SELECT mydb1.t1.* FROM mydb1.t1; + +SELECT t1.* FROM mydb1.t1; +SELECT a.* FROM mydb1.t1 AS a; + +-- Scenario: resolve in case of subquery + +USE mydb1; +CREATE TABLE t3 USING parquet AS SELECT * FROM VALUES (4,1), (3,1) AS t3(c1, c2); +CREATE TABLE t4 USING parquet AS SELECT * FROM VALUES (4,1), (2,1) AS t4(c2, c3); + +SELECT * FROM t3 WHERE c1 IN (SELECT c2 FROM t4 WHERE t4.c3 = t3.c2); + +-- TODO: Support this scenario +SELECT * FROM mydb1.t3 WHERE c1 IN + (SELECT mydb1.t4.c2 FROM mydb1.t4 WHERE mydb1.t4.c3 = mydb1.t3.c2); + +-- Scenario: column resolution scenarios in join queries +set spark.sql.crossJoin.enabled = true; + +-- TODO: Support this scenario +SELECT mydb1.t1.i1 FROM t1, mydb2.t1; + +-- TODO: Support this scenario +SELECT mydb1.t1.i1 FROM mydb1.t1, mydb2.t1; + +USE mydb2; +-- TODO: Support this scenario +SELECT mydb1.t1.i1 FROM t1, mydb1.t1; + +-- Scenario: Table with struct column +USE mydb1; +CREATE TABLE t5(i1 INT, t5 STRUCT) USING PARQUET; +INSERT INTO t5 VALUES(1, (2, 3)); +SELECT t5.i1 FROM t5; +SELECT t5.t5.i1 FROM t5; +SELECT t5.t5.i1 FROM mydb1.t5; +SELECT t5.i1 FROM mydb1.t5; +-- TODO: Support this scenario +SELECT mydb1.t5.t5.i1 FROM mydb1.t5; +-- TODO: Support this scenario +SELECT mydb1.t5.t5.i2 from mydb1.t5; + +-- Negative testcases for column resolution +USE mydb1; +SELECT mydb1.t1 FROM t1; +SELECT t1.x.y.* FROM t1; +SELECT t1 FROM mydb1.t1; +USE mydb2; +SELECT mydb1.t1.i1 FROM t1; +-- Negative tests: column resolution scenarios with ambiguous cases in join queries +set spark.sql.crossJoin.enabled = true; +USE mydb1; +SELECT i1 FROM t1, mydb1.t1; +SELECT t1.i1 FROM t1, mydb1.t1; +SELECT mydb1.t1.i1 FROM t1, mydb1.t1; +SELECT i1 FROM t1, mydb2.t1; +SELECT t1.i1 FROM t1, mydb2.t1; +USE mydb2; +SELECT i1 FROM t1, mydb1.t1; +SELECT t1.i1 FROM t1, mydb1.t1; +SELECT i1 FROM t1, mydb2.t1; +SELECT t1.i1 FROM t1, mydb2.t1; +SELECT db1.t1.i1 FROM t1, mydb2.t1; + +set spark.sql.crossJoin.enabled = true; +USE default; +DROP DATABASE mydb1 CASCADE; +DROP DATABASE mydb2 CASCADE; diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out new file mode 100644 index 000000000000..3460ebeb0309 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out @@ -0,0 +1,588 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 70 + + +-- !query 0 +CREATE DATABASE mydb1 +-- !query 0 schema +struct<> +-- !query 0 output + + + +-- !query 1 +use mydb1 +-- !query 1 schema +struct<> +-- !query 1 output + + + +-- !query 2 +CREATE TABLE t1 USING parquet as SELECT 1 as i1 +-- !query 2 schema +struct<> +-- !query 2 output + + + +-- !query 3 +CREATE DATABASE mydb2 +-- !query 3 schema +struct<> +-- !query 3 output + + + +-- !query 4 +use mydb2 +-- !query 4 schema +struct<> +-- !query 4 output + + + +-- !query 5 +CREATE TABLE t1 USING parquet as SELECT 20 as i1 +-- !query 5 schema +struct<> +-- !query 5 output + + + +-- !query 6 +USE mydb1 +-- !query 6 schema +struct<> +-- !query 6 output + + + +-- !query 7 +SELECT i1 FROM t1 +-- !query 7 schema +struct +-- !query 7 output +1 + + +-- !query 8 +SELECT i1 FROM mydb1.t1 +-- !query 8 schema +struct +-- !query 8 output +1 + + +-- !query 9 +SELECT t1.i1 FROM t1 +-- !query 9 schema +struct +-- !query 9 output +1 + + +-- !query 10 +SELECT t1.i1 FROM mydb1.t1 +-- !query 10 schema +struct +-- !query 10 output +1 + + +-- !query 11 +SELECT mydb1.t1.i1 FROM t1 +-- !query 11 schema +struct<> +-- !query 11 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 12 +SELECT mydb1.t1.i1 FROM mydb1.t1 +-- !query 12 schema +struct<> +-- !query 12 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 13 +USE mydb2 +-- !query 13 schema +struct<> +-- !query 13 output + + + +-- !query 14 +SELECT i1 FROM t1 +-- !query 14 schema +struct +-- !query 14 output +20 + + +-- !query 15 +SELECT i1 FROM mydb1.t1 +-- !query 15 schema +struct +-- !query 15 output +1 + + +-- !query 16 +SELECT t1.i1 FROM t1 +-- !query 16 schema +struct +-- !query 16 output +20 + + +-- !query 17 +SELECT t1.i1 FROM mydb1.t1 +-- !query 17 schema +struct +-- !query 17 output +1 + + +-- !query 18 +SELECT mydb1.t1.i1 FROM mydb1.t1 +-- !query 18 schema +struct<> +-- !query 18 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 19 +use mydb1 +-- !query 19 schema +struct<> +-- !query 19 output + + + +-- !query 20 +SELECT t1.* FROM t1 +-- !query 20 schema +struct +-- !query 20 output +1 + + +-- !query 21 +SELECT mydb1.t1.* FROM mydb1.t1 +-- !query 21 schema +struct<> +-- !query 21 output +org.apache.spark.sql.AnalysisException +cannot resolve 'mydb1.t1.*' give input columns 'i1'; + + +-- !query 22 +SELECT t1.* FROM mydb1.t1 +-- !query 22 schema +struct +-- !query 22 output +1 + + +-- !query 23 +use mydb2 +-- !query 23 schema +struct<> +-- !query 23 output + + + +-- !query 24 +SELECT t1.* FROM t1 +-- !query 24 schema +struct +-- !query 24 output +20 + + +-- !query 25 +SELECT mydb1.t1.* FROM mydb1.t1 +-- !query 25 schema +struct<> +-- !query 25 output +org.apache.spark.sql.AnalysisException +cannot resolve 'mydb1.t1.*' give input columns 'i1'; + + +-- !query 26 +SELECT t1.* FROM mydb1.t1 +-- !query 26 schema +struct +-- !query 26 output +1 + + +-- !query 27 +SELECT a.* FROM mydb1.t1 AS a +-- !query 27 schema +struct +-- !query 27 output +1 + + +-- !query 28 +USE mydb1 +-- !query 28 schema +struct<> +-- !query 28 output + + + +-- !query 29 +CREATE TABLE t3 USING parquet AS SELECT * FROM VALUES (4,1), (3,1) AS t3(c1, c2) +-- !query 29 schema +struct<> +-- !query 29 output + + + +-- !query 30 +CREATE TABLE t4 USING parquet AS SELECT * FROM VALUES (4,1), (2,1) AS t4(c2, c3) +-- !query 30 schema +struct<> +-- !query 30 output + + + +-- !query 31 +SELECT * FROM t3 WHERE c1 IN (SELECT c2 FROM t4 WHERE t4.c3 = t3.c2) +-- !query 31 schema +struct +-- !query 31 output +4 1 + + +-- !query 32 +SELECT * FROM mydb1.t3 WHERE c1 IN + (SELECT mydb1.t4.c2 FROM mydb1.t4 WHERE mydb1.t4.c3 = mydb1.t3.c2) +-- !query 32 schema +struct<> +-- !query 32 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t4.c3`' given input columns: [c2, c3]; line 2 pos 42 + + +-- !query 33 +set spark.sql.crossJoin.enabled = true +-- !query 33 schema +struct +-- !query 33 output +spark.sql.crossJoin.enabled + + +-- !query 34 +SELECT mydb1.t1.i1 FROM t1, mydb2.t1 +-- !query 34 schema +struct<> +-- !query 34 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 + + +-- !query 35 +SELECT mydb1.t1.i1 FROM mydb1.t1, mydb2.t1 +-- !query 35 schema +struct<> +-- !query 35 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 + + +-- !query 36 +USE mydb2 +-- !query 36 schema +struct<> +-- !query 36 output + + + +-- !query 37 +SELECT mydb1.t1.i1 FROM t1, mydb1.t1 +-- !query 37 schema +struct<> +-- !query 37 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 + + +-- !query 38 +USE mydb1 +-- !query 38 schema +struct<> +-- !query 38 output + + + +-- !query 39 +CREATE TABLE t5(i1 INT, t5 STRUCT) USING PARQUET +-- !query 39 schema +struct<> +-- !query 39 output + + + +-- !query 40 +INSERT INTO t5 VALUES(1, (2, 3)) +-- !query 40 schema +struct<> +-- !query 40 output + + + +-- !query 41 +SELECT t5.i1 FROM t5 +-- !query 41 schema +struct +-- !query 41 output +1 + + +-- !query 42 +SELECT t5.t5.i1 FROM t5 +-- !query 42 schema +struct +-- !query 42 output +2 + + +-- !query 43 +SELECT t5.t5.i1 FROM mydb1.t5 +-- !query 43 schema +struct +-- !query 43 output +2 + + +-- !query 44 +SELECT t5.i1 FROM mydb1.t5 +-- !query 44 schema +struct +-- !query 44 output +1 + + +-- !query 45 +SELECT mydb1.t5.t5.i1 FROM mydb1.t5 +-- !query 45 schema +struct<> +-- !query 45 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t5.t5.i1`' given input columns: [i1, t5]; line 1 pos 7 + + +-- !query 46 +SELECT mydb1.t5.t5.i2 from mydb1.t5 +-- !query 46 schema +struct<> +-- !query 46 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t5.t5.i2`' given input columns: [i1, t5]; line 1 pos 7 + + +-- !query 47 +USE mydb1 +-- !query 47 schema +struct<> +-- !query 47 output + + + +-- !query 48 +SELECT mydb1.t1 FROM t1 +-- !query 48 schema +struct<> +-- !query 48 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 49 +SELECT t1.x.y.* FROM t1 +-- !query 49 schema +struct<> +-- !query 49 output +org.apache.spark.sql.AnalysisException +cannot resolve 't1.x.y.*' give input columns 'i1'; + + +-- !query 50 +SELECT t1 FROM mydb1.t1 +-- !query 50 schema +struct<> +-- !query 50 output +org.apache.spark.sql.AnalysisException +cannot resolve '`t1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 51 +USE mydb2 +-- !query 51 schema +struct<> +-- !query 51 output + + + +-- !query 52 +SELECT mydb1.t1.i1 FROM t1 +-- !query 52 schema +struct<> +-- !query 52 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 53 +set spark.sql.crossJoin.enabled = true +-- !query 53 schema +struct +-- !query 53 output +spark.sql.crossJoin.enabled + + +-- !query 54 +USE mydb1 +-- !query 54 schema +struct<> +-- !query 54 output + + + +-- !query 55 +SELECT i1 FROM t1, mydb1.t1 +-- !query 55 schema +struct<> +-- !query 55 output +org.apache.spark.sql.AnalysisException +Reference 'i1' is ambiguous, could be: i1#575, i1#729.; line 1 pos 7 + + +-- !query 56 +SELECT t1.i1 FROM t1, mydb1.t1 +-- !query 56 schema +struct<> +-- !query 56 output +org.apache.spark.sql.AnalysisException +Reference 't1.i1' is ambiguous, could be: i1#575, i1#735.; line 1 pos 7 + + +-- !query 57 +SELECT mydb1.t1.i1 FROM t1, mydb1.t1 +-- !query 57 schema +struct<> +-- !query 57 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 + + +-- !query 58 +SELECT i1 FROM t1, mydb2.t1 +-- !query 58 schema +struct<> +-- !query 58 output +org.apache.spark.sql.AnalysisException +Reference 'i1' is ambiguous, could be: i1#575, i1#591.; line 1 pos 7 + + +-- !query 59 +SELECT t1.i1 FROM t1, mydb2.t1 +-- !query 59 schema +struct<> +-- !query 59 output +org.apache.spark.sql.AnalysisException +Reference 't1.i1' is ambiguous, could be: i1#575, i1#591.; line 1 pos 7 + + +-- !query 60 +USE mydb2 +-- !query 60 schema +struct<> +-- !query 60 output + + + +-- !query 61 +SELECT i1 FROM t1, mydb1.t1 +-- !query 61 schema +struct<> +-- !query 61 output +org.apache.spark.sql.AnalysisException +Reference 'i1' is ambiguous, could be: i1#591, i1#575.; line 1 pos 7 + + +-- !query 62 +SELECT t1.i1 FROM t1, mydb1.t1 +-- !query 62 schema +struct<> +-- !query 62 output +org.apache.spark.sql.AnalysisException +Reference 't1.i1' is ambiguous, could be: i1#591, i1#575.; line 1 pos 7 + + +-- !query 63 +SELECT i1 FROM t1, mydb2.t1 +-- !query 63 schema +struct<> +-- !query 63 output +org.apache.spark.sql.AnalysisException +Reference 'i1' is ambiguous, could be: i1#591, i1#761.; line 1 pos 7 + + +-- !query 64 +SELECT t1.i1 FROM t1, mydb2.t1 +-- !query 64 schema +struct<> +-- !query 64 output +org.apache.spark.sql.AnalysisException +Reference 't1.i1' is ambiguous, could be: i1#591, i1#767.; line 1 pos 7 + + +-- !query 65 +SELECT db1.t1.i1 FROM t1, mydb2.t1 +-- !query 65 schema +struct<> +-- !query 65 output +org.apache.spark.sql.AnalysisException +cannot resolve '`db1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 + + +-- !query 66 +set spark.sql.crossJoin.enabled = true +-- !query 66 schema +struct +-- !query 66 output +spark.sql.crossJoin.enabled + + +-- !query 67 +USE default +-- !query 67 schema +struct<> +-- !query 67 output + + + +-- !query 68 +DROP DATABASE mydb1 CASCADE +-- !query 68 schema +struct<> +-- !query 68 output + + + +-- !query 69 +DROP DATABASE mydb2 CASCADE +-- !query 69 schema +struct<> +-- !query 69 output + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala index 8b611669f143..87fcf98d81c3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala @@ -52,10 +52,6 @@ class ColumnResolutionSuite extends QueryTest with SharedSQLContext { checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(20)) checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1") - } - // TODO: Support this scenario intercept[AnalysisException] { spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala index a75cf4904847..60b9461cbfda 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala @@ -55,10 +55,6 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(20)) checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1") - } - // TODO: Support this scenario intercept[AnalysisException] { spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") @@ -71,11 +67,9 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin withTempDatabase { db2 => try { spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1(i1 INT)") - spark.sql("INSERT INTO t1 VALUES(1)") + spark.sql("CREATE TABLE t1 as SELECT 1 as i1") spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1(i1 INT)") - spark.sql("INSERT INTO t1 VALUES(20)") + spark.sql("CREATE TABLE t1 as SELECT 20 as i1") columnResolutionTests(db1, db2) } finally { @@ -122,7 +116,7 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin } } - test("column resolution scenarios with ambiguous cases") { + test("column resolution scenarios with ambiguous cases in join queries - negative cases") { val currentDb = spark.catalog.currentDatabase withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") { withTempDatabase { db1 => @@ -155,15 +149,6 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin spark.sql(s"SELECT t1.i1 FROM t1, $db2.t1") } - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db2.t1") - } - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1, $db2.t1") - } - spark.catalog.setCurrentDatabase(db2) intercept[AnalysisException] { @@ -174,22 +159,54 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin spark.sql(s"SELECT t1.i1 FROM t1, $db1.t1") } - // TODO: Support this scenario intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db1.t1") + spark.sql(s"SELECT i1 FROM t1, $db2.t1") } intercept[AnalysisException] { - spark.sql(s"SELECT i1 FROM t1, $db2.t1") + spark.sql(s"SELECT t1.i1 FROM t1, $db2.t1") } intercept[AnalysisException] { - spark.sql(s"SELECT t1.i1 FROM t1, $db2.t1") + spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db2.t1") } + } finally { + spark.catalog.setCurrentDatabase(currentDb) + } + } + } + } + } + } + + test("column resolution scenarios in join queries") { + val currentDb = spark.catalog.currentDatabase + withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") { + withTempDatabase { db1 => + withTempDatabase { db2 => + withTempPath { f => + try { + spark.catalog.setCurrentDatabase(db1) + spark.sql("CREATE TABLE t1 AS SELECT 1 AS i1") + spark.catalog.setCurrentDatabase(db2) + spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") + spark.catalog.setCurrentDatabase(db1) + // TODO: Support this scenario intercept[AnalysisException] { spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db2.t1") } + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1, $db2.t1") + } + + spark.catalog.setCurrentDatabase(db2) + // TODO: Support this scenario + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db1.t1") + } + } finally { spark.catalog.setCurrentDatabase(currentDb) } @@ -295,23 +312,30 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin test("col resolution - error case") { val currentDb = spark.catalog.currentDatabase - withTempDatabase { db1 => - withTempPath { f => - try { - spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1(i1 INT)") - spark.sql("INSERT INTO t1 VALUES(1)") - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1 FROM t1") - } - intercept[AnalysisException] { - spark.sql(s"SELECT t1.x.y.* FROM t1") - } - intercept[AnalysisException] { - spark.sql(s"SELECT t1 FROM $db1.t1") + withTempDatabase { db2 => + withTempDatabase { db1 => + withTempPath { f => + try { + spark.catalog.setCurrentDatabase(db1) + spark.sql("CREATE TABLE t1 as SELECT 1 as i1") + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1 FROM t1") + } + intercept[AnalysisException] { + spark.sql(s"SELECT t1.x.y.* FROM t1") + } + intercept[AnalysisException] { + spark.sql(s"SELECT t1 FROM $db1.t1") + } + + spark.catalog.setCurrentDatabase(db2) + spark.sql("CREATE TABLE t1 as SELECT 20 as i1") + intercept[AnalysisException] { + spark.sql(s"SELECT $db1.t1.i1 FROM t1") + } + } finally { + spark.catalog.setCurrentDatabase(currentDb) } - } finally { - spark.catalog.setCurrentDatabase(currentDb) } } } From feeb8a706bd15ab9c28b45da4d02b65e630488f3 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Mon, 27 Feb 2017 18:15:23 -0800 Subject: [PATCH 08/19] Remove the -ve cases from the SQLQueryTestSuite because the exprId can change in exception --- .../sql-tests/inputs/columnresolution.sql | 22 -- .../results/columnresolution.sql.out | 184 +--------------- .../execution/ColumnResolutionSuite.scala | 197 ------------------ 3 files changed, 9 insertions(+), 394 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql index a545ddcd1a0a..3cb4477207b5 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql @@ -80,28 +80,6 @@ SELECT mydb1.t5.t5.i1 FROM mydb1.t5; -- TODO: Support this scenario SELECT mydb1.t5.t5.i2 from mydb1.t5; --- Negative testcases for column resolution -USE mydb1; -SELECT mydb1.t1 FROM t1; -SELECT t1.x.y.* FROM t1; -SELECT t1 FROM mydb1.t1; -USE mydb2; -SELECT mydb1.t1.i1 FROM t1; --- Negative tests: column resolution scenarios with ambiguous cases in join queries -set spark.sql.crossJoin.enabled = true; -USE mydb1; -SELECT i1 FROM t1, mydb1.t1; -SELECT t1.i1 FROM t1, mydb1.t1; -SELECT mydb1.t1.i1 FROM t1, mydb1.t1; -SELECT i1 FROM t1, mydb2.t1; -SELECT t1.i1 FROM t1, mydb2.t1; -USE mydb2; -SELECT i1 FROM t1, mydb1.t1; -SELECT t1.i1 FROM t1, mydb1.t1; -SELECT i1 FROM t1, mydb2.t1; -SELECT t1.i1 FROM t1, mydb2.t1; -SELECT db1.t1.i1 FROM t1, mydb2.t1; - set spark.sql.crossJoin.enabled = true; USE default; DROP DATABASE mydb1 CASCADE; diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out index 3460ebeb0309..abe5128cc054 100644 --- a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 70 +-- Number of queries: 51 -- !query 0 @@ -391,198 +391,32 @@ cannot resolve '`mydb1.t5.t5.i2`' given input columns: [i1, t5]; line 1 pos 7 -- !query 47 -USE mydb1 +set spark.sql.crossJoin.enabled = true -- !query 47 schema -struct<> +struct -- !query 47 output - +spark.sql.crossJoin.enabled -- !query 48 -SELECT mydb1.t1 FROM t1 +USE default -- !query 48 schema struct<> -- !query 48 output -org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t1`' given input columns: [i1]; line 1 pos 7 + -- !query 49 -SELECT t1.x.y.* FROM t1 +DROP DATABASE mydb1 CASCADE -- !query 49 schema struct<> -- !query 49 output -org.apache.spark.sql.AnalysisException -cannot resolve 't1.x.y.*' give input columns 'i1'; + -- !query 50 -SELECT t1 FROM mydb1.t1 +DROP DATABASE mydb2 CASCADE -- !query 50 schema struct<> -- !query 50 output -org.apache.spark.sql.AnalysisException -cannot resolve '`t1`' given input columns: [i1]; line 1 pos 7 - - --- !query 51 -USE mydb2 --- !query 51 schema -struct<> --- !query 51 output - - - --- !query 52 -SELECT mydb1.t1.i1 FROM t1 --- !query 52 schema -struct<> --- !query 52 output -org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 - - --- !query 53 -set spark.sql.crossJoin.enabled = true --- !query 53 schema -struct --- !query 53 output -spark.sql.crossJoin.enabled - - --- !query 54 -USE mydb1 --- !query 54 schema -struct<> --- !query 54 output - - - --- !query 55 -SELECT i1 FROM t1, mydb1.t1 --- !query 55 schema -struct<> --- !query 55 output -org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: i1#575, i1#729.; line 1 pos 7 - - --- !query 56 -SELECT t1.i1 FROM t1, mydb1.t1 --- !query 56 schema -struct<> --- !query 56 output -org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: i1#575, i1#735.; line 1 pos 7 - - --- !query 57 -SELECT mydb1.t1.i1 FROM t1, mydb1.t1 --- !query 57 schema -struct<> --- !query 57 output -org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 - - --- !query 58 -SELECT i1 FROM t1, mydb2.t1 --- !query 58 schema -struct<> --- !query 58 output -org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: i1#575, i1#591.; line 1 pos 7 - - --- !query 59 -SELECT t1.i1 FROM t1, mydb2.t1 --- !query 59 schema -struct<> --- !query 59 output -org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: i1#575, i1#591.; line 1 pos 7 - - --- !query 60 -USE mydb2 --- !query 60 schema -struct<> --- !query 60 output - - - --- !query 61 -SELECT i1 FROM t1, mydb1.t1 --- !query 61 schema -struct<> --- !query 61 output -org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: i1#591, i1#575.; line 1 pos 7 - - --- !query 62 -SELECT t1.i1 FROM t1, mydb1.t1 --- !query 62 schema -struct<> --- !query 62 output -org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: i1#591, i1#575.; line 1 pos 7 - - --- !query 63 -SELECT i1 FROM t1, mydb2.t1 --- !query 63 schema -struct<> --- !query 63 output -org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: i1#591, i1#761.; line 1 pos 7 - - --- !query 64 -SELECT t1.i1 FROM t1, mydb2.t1 --- !query 64 schema -struct<> --- !query 64 output -org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: i1#591, i1#767.; line 1 pos 7 - - --- !query 65 -SELECT db1.t1.i1 FROM t1, mydb2.t1 --- !query 65 schema -struct<> --- !query 65 output -org.apache.spark.sql.AnalysisException -cannot resolve '`db1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 - - --- !query 66 -set spark.sql.crossJoin.enabled = true --- !query 66 schema -struct --- !query 66 output -spark.sql.crossJoin.enabled - - --- !query 67 -USE default --- !query 67 schema -struct<> --- !query 67 output - - - --- !query 68 -DROP DATABASE mydb1 CASCADE --- !query 68 schema -struct<> --- !query 68 output - - - --- !query 69 -DROP DATABASE mydb2 CASCADE --- !query 69 schema -struct<> --- !query 69 output diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala index 60b9461cbfda..0292a196fbd5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala @@ -17,8 +17,6 @@ package org.apache.spark.sql.hive.execution -import java.io.File - import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.hive.test.TestHiveSingleton import org.apache.spark.sql.internal.SQLConf @@ -79,43 +77,6 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin } } - test("column resolution scenarios with datasource table") { - val currentDb = spark.catalog.currentDatabase - withTempDatabase { db1 => - withTempDatabase { db2 => - withTempDir { f => - try { - val df = Seq(1).toDF() - val path = s"${f.getCanonicalPath}${File.separator}test1" - df.write.csv(path) - spark.catalog.setCurrentDatabase(db1) - - sql( - s""" - |CREATE TABLE t1(i1 INT) USING csv OPTIONS - |(path "$path", header "false") - """.stripMargin) - - spark.catalog.setCurrentDatabase(db2) - val df2 = Seq(20).toDF() - val path2 = s"${f.getCanonicalPath}${File.separator}test2" - df2.write.csv(path2) - - sql( - s""" - |CREATE TABLE t1(i1 INT) USING csv OPTIONS - |(path "$path2", header "false") - """.stripMargin) - - columnResolutionTests(db1, db2) - } finally { - spark.catalog.setCurrentDatabase (currentDb) - } - } - } - } - } - test("column resolution scenarios with ambiguous cases in join queries - negative cases") { val currentDb = spark.catalog.currentDatabase withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") { @@ -179,137 +140,6 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin } } - test("column resolution scenarios in join queries") { - val currentDb = spark.catalog.currentDatabase - withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") { - withTempDatabase { db1 => - withTempDatabase { db2 => - withTempPath { f => - try { - spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1 AS SELECT 1 AS i1") - spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") - spark.catalog.setCurrentDatabase(db1) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db2.t1") - } - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1, $db2.t1") - } - - spark.catalog.setCurrentDatabase(db2) - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db1.t1") - } - - } finally { - spark.catalog.setCurrentDatabase(currentDb) - } - } - } - } - } - } - - test("resolve fully qualified table name in star expansion ") { - val currentDb = spark.catalog.currentDatabase - withTempDatabase { db1 => - withTempDatabase { db2 => - withTempPath { f => - try { - val df = spark.range(1).toDF() - df.write.csv(f.getCanonicalPath) - spark.catalog.setCurrentDatabase(db1) - - sql( - s""" - |CREATE TABLE t1(i1 INT) USING csv OPTIONS - |(path "${f.getCanonicalPath}", header "false") - """.stripMargin) - - spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") - - spark.catalog.setCurrentDatabase(db1) - checkAnswer(spark.sql("SELECT t1.* FROM t1"), Row(0)) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.* FROM $db1.t1") - } - - checkAnswer(spark.sql(s"SELECT t1.* FROM $db1.t1"), Row(0)) - - spark.catalog.setCurrentDatabase(db2) - checkAnswer(spark.sql("SELECT t1.* FROM t1"), Row(20)) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.* FROM $db1.t1") - } - - checkAnswer(spark.sql(s"SELECT t1.* FROM $db1.t1"), Row(0)) - checkAnswer(spark.sql(s"SELECT a.* FROM $db1.t1 AS a"), Row(0)) - - } finally { - spark.catalog.setCurrentDatabase(currentDb) - } - } - } - } - } - - test("resolve in case of subquery") { - val currentDb = spark.catalog.currentDatabase - withTempDatabase { db1 => - withTempDir { f => - try { - val df = Seq((4, 1), (3, 1)).toDF() - val path = s"${f.getCanonicalPath}${File.separator}test1" - df.write.csv(path) - spark.catalog.setCurrentDatabase(db1) - - sql( - s""" - |CREATE TABLE t3(c1 INT, c2 INT) USING csv OPTIONS - |(path "$path", header "false") - """.stripMargin) - - val df2 = Seq((4, 1), (2, 1)).toDF() - val path2 = s"${f.getCanonicalPath}${File.separator}test2" - df2.write.csv(path2) - - sql( - s""" - |CREATE TABLE t4(c2 INT, c3 INT) USING csv OPTIONS - |(path "$path2", header "false") - """.stripMargin) - - checkAnswer( - spark.sql("SELECT * FROM t3 WHERE c1 IN (SELECT c2 FROM t4 WHERE t4.c3 = t3.c2)"), - Row(4, 1)) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql( - s""" - |SELECT * FROM $db1.t3 WHERE c1 IN - |(SELECT $db1.t4.c2 FROM $db1.t4 WHERE $db1.t4.c3 = $db1.t3.c2) - """.stripMargin) - } - - } finally { - spark.catalog.setCurrentDatabase(currentDb) - } - } - } - } - test("col resolution - error case") { val currentDb = spark.catalog.currentDatabase withTempDatabase { db2 => @@ -340,31 +170,4 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin } } } - - test("Table with struct column") { - val currentDb = spark.catalog.currentDatabase - withTempDatabase { db1 => - try { - spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1(i1 INT, t1 STRUCT)") - spark.sql("INSERT INTO t1 VALUES(1, (2, 3))") - checkAnswer(spark.sql(s"SELECT t1.i1 FROM t1"), Row(1)) - checkAnswer(spark.sql(s"SELECT t1.t1.i1 FROM t1"), Row(2)) - checkAnswer(spark.sql(s"SELECT t1.t1.i1 FROM $db1.t1"), Row(2)) - checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.t1.i1 FROM $db1.t1") - } - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.t1.i2 FROM $db1.t1") - } - } finally { - spark.catalog.setCurrentDatabase(currentDb) - } - } - } } From 129c0c4d2c213f0707dbed7560ee32b3d5f665ea Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Mon, 27 Feb 2017 18:18:16 -0800 Subject: [PATCH 09/19] Tests are moved to the SQLQueryTestSuite --- .../spark/sql/ColumnResolutionSuite.scala | 97 ------------------- 1 file changed, 97 deletions(-) delete mode 100644 sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala deleted file mode 100644 index 87fcf98d81c3..000000000000 --- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnResolutionSuite.scala +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql - -import java.io.File - -import org.apache.spark.sql.test.SharedSQLContext - - -class ColumnResolutionSuite extends QueryTest with SharedSQLContext { - - import testImplicits._ - - def columnResolutionTests(db1: String, db2: String): Unit = { - spark.catalog.setCurrentDatabase(db1) - checkAnswer(spark.sql("SELECT i1 FROM t1"), Row(1)) - checkAnswer(spark.sql(s"SELECT i1 FROM $db1.t1"), Row(1)) - - checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(1)) - checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1") - } - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") - } - - // Change current database to db2 - spark.catalog.setCurrentDatabase(db2) - checkAnswer(spark.sql("SELECT i1 FROM t1"), Row(20)) - checkAnswer(spark.sql(s"SELECT i1 FROM $db1.t1"), Row(1)) - - checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(20)) - checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") - } - } - - test("column resolution scenarios with datasource table") { - val currentDb = spark.catalog.currentDatabase - withTempDatabase { db1 => - withTempDatabase { db2 => - withTempDir { f => - try { - val df = Seq(1).toDF() - val path = s"${f.getCanonicalPath}${File.separator}test1" - df.write.csv(path) - spark.catalog.setCurrentDatabase(db1) - - sql( - s""" - |CREATE TABLE t1(i1 INT) USING csv OPTIONS - |(path "$path", header "false") - """.stripMargin) - - spark.catalog.setCurrentDatabase(db2) - val df2 = Seq(20).toDF() - val path2 = s"${f.getCanonicalPath}${File.separator}test2" - df2.write.csv(path2) - - sql( - s""" - |CREATE TABLE t1(i1 INT) USING csv OPTIONS - |(path "$path2", header "false") - """.stripMargin) - - columnResolutionTests(db1, db2) - } finally { - spark.catalog.setCurrentDatabase (currentDb) - } - } - } - } - } -} From 4700f39d1cff0bc843c5072792a026ed120fb639 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Mon, 27 Feb 2017 18:56:06 -0800 Subject: [PATCH 10/19] Remove empty lines --- .../src/test/resources/sql-tests/inputs/columnresolution.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql index 3cb4477207b5..082dc0364bb3 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql @@ -23,11 +23,9 @@ SELECT i1 FROM t1; SELECT i1 FROM mydb1.t1; SELECT t1.i1 FROM t1; SELECT t1.i1 FROM mydb1.t1; - -- TODO: Support this scenario SELECT mydb1.t1.i1 FROM mydb1.t1; - -- Scenario: resolve fully qualified table name in star expansion use mydb1; SELECT t1.* FROM t1; @@ -35,10 +33,8 @@ SELECT mydb1.t1.* FROM mydb1.t1; SELECT t1.* FROM mydb1.t1; use mydb2; SELECT t1.* FROM t1; - -- TODO: Support this scenario SELECT mydb1.t1.* FROM mydb1.t1; - SELECT t1.* FROM mydb1.t1; SELECT a.* FROM mydb1.t1 AS a; From 158016fae644763a2992a82a1659de0d16fa35f2 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Mon, 27 Feb 2017 20:24:28 -0800 Subject: [PATCH 11/19] Remove unused import --- .../apache/spark/sql/hive/execution/ColumnResolutionSuite.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala index 0292a196fbd5..e013107ef831 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala @@ -24,8 +24,6 @@ import org.apache.spark.sql.test.SQLTestUtils class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSingleton { - import spark.implicits._ - def columnResolutionTests(db1: String, db2: String): Unit = { spark.catalog.setCurrentDatabase(db1) From 1c5cc4b925027644d9489993d3eee46a12f62236 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Tue, 28 Feb 2017 11:50:30 -0800 Subject: [PATCH 12/19] Move view tests also to SQLQueryTestSuite --- .../inputs/columnresolution-views.sql | 25 ++++ .../sql-tests/inputs/columnresolution.sql | 22 +-- .../results/columnresolution-views.sql.out | 140 ++++++++++++++++++ .../results/columnresolution.sql.out | 20 +-- .../sql/execution/GlobalTempViewSuite.scala | 25 ---- .../spark/sql/execution/SQLViewSuite.scala | 13 -- 6 files changed, 187 insertions(+), 58 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/columnresolution-views.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/columnresolution-views.sql.out diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution-views.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution-views.sql new file mode 100644 index 000000000000..71f13e3cbcef --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution-views.sql @@ -0,0 +1,25 @@ +-- Tests for qualified column names for the view code-path +-- Test scenario with Temporary view +CREATE OR REPLACE TEMPORARY VIEW table1 AS SELECT 2 AS i1; +SELECT table1.* FROM table1; +SELECT * FROM table1; +SELECT table1.i1 FROM table1; +SELECT i1 FROM table1; +SELECT a.i1 FROM table1 AS a; +SELECT i1 FROM table1 AS a; +-- cleanup +DROP VIEW table1; + +-- Test scenario with Global Temp view +CREATE OR REPLACE GLOBAL TEMPORARY VIEW t1 as SELECT 1 as i1; +SELECT * FROM global_temp.t1; +-- TODO: Support this scenario +SELECT global_temp.t1.* FROM global_temp.t1; +SELECT i1 FROM global_temp.t1; +-- TODO: Support this scenario +SELECT global_temp.t1.i1 FROM global_temp.t1; +SELECT t1.i1 FROM global_temp.t1; +SELECT a.i1 FROM global_temp.t1 AS a; +SELECT i1 FROM global_temp.t1 AS a; +-- cleanup +DROP VIEW global_temp.t1; diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql index 082dc0364bb3..7941ed8a37e7 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql @@ -1,11 +1,12 @@ +-- Tests covering different scenarios with qualified column names -- Scenario: column resolution scenarios with datasource table CREATE DATABASE mydb1; -use mydb1; -CREATE TABLE t1 USING parquet as SELECT 1 as i1; +USE mydb1; +CREATE TABLE t1 USING parquet AS SELECT 1 AS i1; CREATE DATABASE mydb2; -use mydb2; -CREATE TABLE t1 USING parquet as SELECT 20 as i1; +USE mydb2; +CREATE TABLE t1 USING parquet AS SELECT 20 AS i1; USE mydb1; SELECT i1 FROM t1; @@ -27,11 +28,11 @@ SELECT t1.i1 FROM mydb1.t1; SELECT mydb1.t1.i1 FROM mydb1.t1; -- Scenario: resolve fully qualified table name in star expansion -use mydb1; +USE mydb1; SELECT t1.* FROM t1; SELECT mydb1.t1.* FROM mydb1.t1; SELECT t1.* FROM mydb1.t1; -use mydb2; +USE mydb2; SELECT t1.* FROM t1; -- TODO: Support this scenario SELECT mydb1.t1.* FROM mydb1.t1; @@ -51,7 +52,7 @@ SELECT * FROM mydb1.t3 WHERE c1 IN (SELECT mydb1.t4.c2 FROM mydb1.t4 WHERE mydb1.t4.c3 = mydb1.t3.c2); -- Scenario: column resolution scenarios in join queries -set spark.sql.crossJoin.enabled = true; +SET spark.sql.crossJoin.enabled = true; -- TODO: Support this scenario SELECT mydb1.t1.i1 FROM t1, mydb2.t1; @@ -65,7 +66,7 @@ SELECT mydb1.t1.i1 FROM t1, mydb1.t1; -- Scenario: Table with struct column USE mydb1; -CREATE TABLE t5(i1 INT, t5 STRUCT) USING PARQUET; +CREATE TABLE t5(i1 INT, t5 STRUCT) USING parquet; INSERT INTO t5 VALUES(1, (2, 3)); SELECT t5.i1 FROM t5; SELECT t5.t5.i1 FROM t5; @@ -74,9 +75,10 @@ SELECT t5.i1 FROM mydb1.t5; -- TODO: Support this scenario SELECT mydb1.t5.t5.i1 FROM mydb1.t5; -- TODO: Support this scenario -SELECT mydb1.t5.t5.i2 from mydb1.t5; +SELECT mydb1.t5.t5.i2 FROM mydb1.t5; -set spark.sql.crossJoin.enabled = true; +-- Cleanup and Reset +SET spark.sql.crossJoin.enabled = false; USE default; DROP DATABASE mydb1 CASCADE; DROP DATABASE mydb2 CASCADE; diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-views.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-views.sql.out new file mode 100644 index 000000000000..4ab35a0e76ce --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-views.sql.out @@ -0,0 +1,140 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 17 + + +-- !query 0 +CREATE OR REPLACE TEMPORARY VIEW table1 AS SELECT 2 AS i1 +-- !query 0 schema +struct<> +-- !query 0 output + + + +-- !query 1 +SELECT table1.* FROM table1 +-- !query 1 schema +struct +-- !query 1 output +2 + + +-- !query 2 +SELECT * FROM table1 +-- !query 2 schema +struct +-- !query 2 output +2 + + +-- !query 3 +SELECT table1.i1 FROM table1 +-- !query 3 schema +struct +-- !query 3 output +2 + + +-- !query 4 +SELECT i1 FROM table1 +-- !query 4 schema +struct +-- !query 4 output +2 + + +-- !query 5 +SELECT a.i1 FROM table1 AS a +-- !query 5 schema +struct +-- !query 5 output +2 + + +-- !query 6 +SELECT i1 FROM table1 AS a +-- !query 6 schema +struct +-- !query 6 output +2 + + +-- !query 7 +DROP VIEW table1 +-- !query 7 schema +struct<> +-- !query 7 output + + + +-- !query 8 +CREATE OR REPLACE GLOBAL TEMPORARY VIEW t1 as SELECT 1 as i1 +-- !query 8 schema +struct<> +-- !query 8 output + + + +-- !query 9 +SELECT * FROM global_temp.t1 +-- !query 9 schema +struct +-- !query 9 output +1 + + +-- !query 10 +SELECT global_temp.t1.* FROM global_temp.t1 +-- !query 10 schema +struct<> +-- !query 10 output +org.apache.spark.sql.AnalysisException +cannot resolve 'global_temp.t1.*' give input columns 'i1'; + + +-- !query 11 +SELECT i1 FROM global_temp.t1 +-- !query 11 schema +struct +-- !query 11 output +1 + + +-- !query 12 +SELECT global_temp.t1.i1 FROM global_temp.t1 +-- !query 12 schema +struct<> +-- !query 12 output +org.apache.spark.sql.AnalysisException +cannot resolve '`global_temp.t1.i1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 13 +SELECT t1.i1 FROM global_temp.t1 +-- !query 13 schema +struct +-- !query 13 output +1 + + +-- !query 14 +SELECT a.i1 FROM global_temp.t1 AS a +-- !query 14 schema +struct +-- !query 14 output +1 + + +-- !query 15 +SELECT i1 FROM global_temp.t1 AS a +-- !query 15 schema +struct +-- !query 15 output +1 + + +-- !query 16 +DROP VIEW global_temp.t1 +-- !query 16 schema +struct<> +-- !query 16 output + diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out index abe5128cc054..8d66de1cceef 100644 --- a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out @@ -11,7 +11,7 @@ struct<> -- !query 1 -use mydb1 +USE mydb1 -- !query 1 schema struct<> -- !query 1 output @@ -19,7 +19,7 @@ struct<> -- !query 2 -CREATE TABLE t1 USING parquet as SELECT 1 as i1 +CREATE TABLE t1 USING parquet AS SELECT 1 AS i1 -- !query 2 schema struct<> -- !query 2 output @@ -35,7 +35,7 @@ struct<> -- !query 4 -use mydb2 +USE mydb2 -- !query 4 schema struct<> -- !query 4 output @@ -43,7 +43,7 @@ struct<> -- !query 5 -CREATE TABLE t1 USING parquet as SELECT 20 as i1 +CREATE TABLE t1 USING parquet AS SELECT 20 AS i1 -- !query 5 schema struct<> -- !query 5 output @@ -158,7 +158,7 @@ cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 -- !query 19 -use mydb1 +USE mydb1 -- !query 19 schema struct<> -- !query 19 output @@ -191,7 +191,7 @@ struct -- !query 23 -use mydb2 +USE mydb2 -- !query 23 schema struct<> -- !query 23 output @@ -274,7 +274,7 @@ cannot resolve '`mydb1.t4.c3`' given input columns: [c2, c3]; line 2 pos 42 -- !query 33 -set spark.sql.crossJoin.enabled = true +SET spark.sql.crossJoin.enabled = true -- !query 33 schema struct -- !query 33 output @@ -325,7 +325,7 @@ struct<> -- !query 39 -CREATE TABLE t5(i1 INT, t5 STRUCT) USING PARQUET +CREATE TABLE t5(i1 INT, t5 STRUCT) USING parquet -- !query 39 schema struct<> -- !query 39 output @@ -382,7 +382,7 @@ cannot resolve '`mydb1.t5.t5.i1`' given input columns: [i1, t5]; line 1 pos 7 -- !query 46 -SELECT mydb1.t5.t5.i2 from mydb1.t5 +SELECT mydb1.t5.t5.i2 FROM mydb1.t5 -- !query 46 schema struct<> -- !query 46 output @@ -391,7 +391,7 @@ cannot resolve '`mydb1.t5.t5.i2`' given input columns: [i1, t5]; line 1 pos 7 -- !query 47 -set spark.sql.crossJoin.enabled = true +SET spark.sql.crossJoin.enabled = false -- !query 47 schema struct -- !query 47 output diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala index f30304145536..5c63c6a414f9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala @@ -150,31 +150,6 @@ class GlobalTempViewSuite extends QueryTest with SharedSQLContext { } } - test("column resolution scenarios with global temp view") { - val df = Seq(1).toDF("i1") - df.createGlobalTempView("t1") - try { - checkAnswer(spark.sql(s"SELECT * FROM $globalTempDB.t1"), Row(1)) - - // TODO: Fix this scenario - intercept[AnalysisException] { - checkAnswer(spark.sql(s"SELECT $globalTempDB.t1.* FROM $globalTempDB.t1"), Row(1)) - } - checkAnswer(spark.sql(s"SELECT i1 FROM $globalTempDB.t1"), Row(1)) - - // TODO: Fix this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $globalTempDB.t1.i1 FROM $globalTempDB.t1") - } - - checkAnswer(spark.sql(s"SELECT t1.i1 FROM $globalTempDB.t1"), Row(1)) - checkAnswer(spark.sql(s"SELECT a.i1 FROM $globalTempDB.t1 AS a"), Row(1)) - checkAnswer(spark.sql(s"SELECT i1 FROM $globalTempDB.t1 AS a"), Row(1)) - } finally { - spark.catalog.dropGlobalTempView("t1") - } - } - test("public Catalog should recognize global temp view") { try { sql("CREATE GLOBAL TEMP VIEW src AS SELECT 1, 2") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala index 1f22eca557c0..2d95cb6d64a8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala @@ -52,19 +52,6 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils { } } - test("column resolution scenarios with local temp view") { - val df = Seq(2).toDF("i1") - df.createOrReplaceTempView("table1") - withTempView("table1") { - checkAnswer(spark.sql("SELECT table1.* FROM table1"), Row(2)) - checkAnswer(spark.sql("SELECT * FROM table1"), Row(2)) - checkAnswer(spark.sql("SELECT i1 FROM table1"), Row(2)) - checkAnswer(spark.sql("SELECT table1.i1 FROM table1"), Row(2)) - checkAnswer(spark.sql("SELECT a.i1 FROM table1 AS a"), Row(2)) - checkAnswer(spark.sql("SELECT i1 FROM table1 AS a"), Row(2)) - } - } - test("create a temp view on a permanent view") { withView("jtv1", "temp_jtv1") { sql("CREATE VIEW jtv1 AS SELECT * FROM jt WHERE id > 3") From 14ae1251e83d8c2e28da4194b7667fddaa9d6cbd Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Tue, 28 Feb 2017 14:18:38 -0800 Subject: [PATCH 13/19] Rebase with master --- .../test/resources/sql-tests/results/columnresolution.sql.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out index 8d66de1cceef..b28b750c810d 100644 --- a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out @@ -278,7 +278,7 @@ SET spark.sql.crossJoin.enabled = true -- !query 33 schema struct -- !query 33 output -spark.sql.crossJoin.enabled +spark.sql.crossJoin.enabled true -- !query 34 @@ -395,7 +395,7 @@ SET spark.sql.crossJoin.enabled = false -- !query 47 schema struct -- !query 47 output -spark.sql.crossJoin.enabled +spark.sql.crossJoin.enabled false -- !query 48 From 1accdcfb4ceb1e571f549f0b63b06e62565d51f8 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Tue, 28 Feb 2017 15:19:11 -0800 Subject: [PATCH 14/19] upper casing SQL --- .../spark/sql/hive/execution/ColumnResolutionSuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala index e013107ef831..7024b37b0fcf 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala @@ -63,9 +63,9 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin withTempDatabase { db2 => try { spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1 as SELECT 1 as i1") + spark.sql("CREATE TABLE t1 AS SELECT 1 AS i1") spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1 as SELECT 20 as i1") + spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") columnResolutionTests(db1, db2) } finally { @@ -145,7 +145,7 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin withTempPath { f => try { spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1 as SELECT 1 as i1") + spark.sql("CREATE TABLE t1 AS SELECT 1 AS i1") intercept[AnalysisException] { spark.sql(s"SELECT $db1.t1 FROM t1") } @@ -157,7 +157,7 @@ class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSin } spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1 as SELECT 20 as i1") + spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") intercept[AnalysisException] { spark.sql(s"SELECT $db1.t1.i1 FROM t1") } From 102e555c421950844aab9168844e734c1803790e Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Wed, 1 Mar 2017 14:48:51 -0800 Subject: [PATCH 15/19] Add -ve tests to the SQLQueryTestSuite framework --- .../inputs/columnresolution-negative.sql | 36 +++ .../results/columnresolution-negative.sql.out | 240 ++++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/columnresolution-negative.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution-negative.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution-negative.sql new file mode 100644 index 000000000000..4afb2cf6915c --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution-negative.sql @@ -0,0 +1,36 @@ +-- Negative testcases for column resolution +CREATE DATABASE mydb1; +USE mydb1; +CREATE TABLE t1 USING parquet AS SELECT 1 AS i1; + +CREATE DATABASE mydb2; +USE mydb2; +CREATE TABLE t1 USING parquet AS SELECT 20 AS i1; + +-- Negative tests: column resolution scenarios with ambiguous cases in join queries +set spark.sql.crossJoin.enabled = true; +USE mydb1; +SELECT i1 FROM t1, mydb1.t1; +SELECT t1.i1 FROM t1, mydb1.t1; +SELECT mydb1.t1.i1 FROM t1, mydb1.t1; +SELECT i1 FROM t1, mydb2.t1; +SELECT t1.i1 FROM t1, mydb2.t1; +USE mydb2; +SELECT i1 FROM t1, mydb1.t1; +SELECT t1.i1 FROM t1, mydb1.t1; +SELECT i1 FROM t1, mydb2.t1; +SELECT t1.i1 FROM t1, mydb2.t1; +SELECT db1.t1.i1 FROM t1, mydb2.t1; + +-- Negative tests +USE mydb1; +SELECT mydb1.t1 FROM t1; +SELECT t1.x.y.* FROM t1; +SELECT t1 FROM mydb1.t1; +USE mydb2; +SELECT mydb1.t1.i1 FROM t1; + +-- reset +set spark.sql.crossJoin.enabled = false; +DROP DATABASE mydb1 CASCADE; +DROP DATABASE mydb2 CASCADE; \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out new file mode 100644 index 000000000000..98200e8f9052 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out @@ -0,0 +1,240 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 28 + + +-- !query 0 +CREATE DATABASE mydb1 +-- !query 0 schema +struct<> +-- !query 0 output + + + +-- !query 1 +USE mydb1 +-- !query 1 schema +struct<> +-- !query 1 output + + + +-- !query 2 +CREATE TABLE t1 USING parquet AS SELECT 1 AS i1 +-- !query 2 schema +struct<> +-- !query 2 output + + + +-- !query 3 +CREATE DATABASE mydb2 +-- !query 3 schema +struct<> +-- !query 3 output + + + +-- !query 4 +USE mydb2 +-- !query 4 schema +struct<> +-- !query 4 output + + + +-- !query 5 +CREATE TABLE t1 USING parquet AS SELECT 20 AS i1 +-- !query 5 schema +struct<> +-- !query 5 output + + + +-- !query 6 +set spark.sql.crossJoin.enabled = true +-- !query 6 schema +struct +-- !query 6 output +spark.sql.crossJoin.enabled true + + +-- !query 7 +USE mydb1 +-- !query 7 schema +struct<> +-- !query 7 output + + + +-- !query 8 +SELECT i1 FROM t1, mydb1.t1 +-- !query 8 schema +struct<> +-- !query 8 output +org.apache.spark.sql.AnalysisException +Reference 'i1' is ambiguous, could be: i1#588, i1#589.; line 1 pos 7 + + +-- !query 9 +SELECT t1.i1 FROM t1, mydb1.t1 +-- !query 9 schema +struct<> +-- !query 9 output +org.apache.spark.sql.AnalysisException +Reference 't1.i1' is ambiguous, could be: i1#592, i1#593.; line 1 pos 7 + + +-- !query 10 +SELECT mydb1.t1.i1 FROM t1, mydb1.t1 +-- !query 10 schema +struct<> +-- !query 10 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 + + +-- !query 11 +SELECT i1 FROM t1, mydb2.t1 +-- !query 11 schema +struct<> +-- !query 11 output +org.apache.spark.sql.AnalysisException +Reference 'i1' is ambiguous, could be: i1#601, i1#602.; line 1 pos 7 + + +-- !query 12 +SELECT t1.i1 FROM t1, mydb2.t1 +-- !query 12 schema +struct<> +-- !query 12 output +org.apache.spark.sql.AnalysisException +Reference 't1.i1' is ambiguous, could be: i1#605, i1#606.; line 1 pos 7 + + +-- !query 13 +USE mydb2 +-- !query 13 schema +struct<> +-- !query 13 output + + + +-- !query 14 +SELECT i1 FROM t1, mydb1.t1 +-- !query 14 schema +struct<> +-- !query 14 output +org.apache.spark.sql.AnalysisException +Reference 'i1' is ambiguous, could be: i1#610, i1#611.; line 1 pos 7 + + +-- !query 15 +SELECT t1.i1 FROM t1, mydb1.t1 +-- !query 15 schema +struct<> +-- !query 15 output +org.apache.spark.sql.AnalysisException +Reference 't1.i1' is ambiguous, could be: i1#614, i1#615.; line 1 pos 7 + + +-- !query 16 +SELECT i1 FROM t1, mydb2.t1 +-- !query 16 schema +struct<> +-- !query 16 output +org.apache.spark.sql.AnalysisException +Reference 'i1' is ambiguous, could be: i1#618, i1#619.; line 1 pos 7 + + +-- !query 17 +SELECT t1.i1 FROM t1, mydb2.t1 +-- !query 17 schema +struct<> +-- !query 17 output +org.apache.spark.sql.AnalysisException +Reference 't1.i1' is ambiguous, could be: i1#622, i1#623.; line 1 pos 7 + + +-- !query 18 +SELECT db1.t1.i1 FROM t1, mydb2.t1 +-- !query 18 schema +struct<> +-- !query 18 output +org.apache.spark.sql.AnalysisException +cannot resolve '`db1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 + + +-- !query 19 +USE mydb1 +-- !query 19 schema +struct<> +-- !query 19 output + + + +-- !query 20 +SELECT mydb1.t1 FROM t1 +-- !query 20 schema +struct<> +-- !query 20 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 21 +SELECT t1.x.y.* FROM t1 +-- !query 21 schema +struct<> +-- !query 21 output +org.apache.spark.sql.AnalysisException +cannot resolve 't1.x.y.*' give input columns 'i1'; + + +-- !query 22 +SELECT t1 FROM mydb1.t1 +-- !query 22 schema +struct<> +-- !query 22 output +org.apache.spark.sql.AnalysisException +cannot resolve '`t1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 23 +USE mydb2 +-- !query 23 schema +struct<> +-- !query 23 output + + + +-- !query 24 +SELECT mydb1.t1.i1 FROM t1 +-- !query 24 schema +struct<> +-- !query 24 output +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 + + +-- !query 25 +set spark.sql.crossJoin.enabled = false +-- !query 25 schema +struct +-- !query 25 output +spark.sql.crossJoin.enabled false + + +-- !query 26 +DROP DATABASE mydb1 CASCADE +-- !query 26 schema +struct<> +-- !query 26 output + + + +-- !query 27 +DROP DATABASE mydb2 CASCADE +-- !query 27 schema +struct<> +-- !query 27 output + From f01c884d8146a23bfb6b64ec4d4b38f17067e947 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Wed, 1 Mar 2017 14:57:59 -0800 Subject: [PATCH 16/19] This file isnt updated properly in Spark-19766 --- .../src/test/resources/sql-tests/results/inner-join.sql.out | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/inner-join.sql.out b/sql/core/src/test/resources/sql-tests/results/inner-join.sql.out index aa20537d449e..8d56ebe9fd3b 100644 --- a/sql/core/src/test/resources/sql-tests/results/inner-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/inner-join.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 13 +-- Number of queries: 7 -- !query 0 @@ -65,4 +65,3 @@ struct 1 a 1 b 1 b - From 14fc6fdf30547c702015c717f99a7f15ff6503e3 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Wed, 1 Mar 2017 15:37:05 -0800 Subject: [PATCH 17/19] Change to SQLQueryTestSuite to mask the id --- .../results/columnresolution-negative.sql.out | 16 ++++++++-------- .../org/apache/spark/sql/SQLQueryTestSuite.scala | 5 +++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out index 98200e8f9052..df4733d4ff1a 100644 --- a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out @@ -72,7 +72,7 @@ SELECT i1 FROM t1, mydb1.t1 struct<> -- !query 8 output org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: i1#588, i1#589.; line 1 pos 7 +Reference 'i1' is ambiguous, could be: i1#x, i1#x.; line 1 pos 7 -- !query 9 @@ -81,7 +81,7 @@ SELECT t1.i1 FROM t1, mydb1.t1 struct<> -- !query 9 output org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: i1#592, i1#593.; line 1 pos 7 +Reference 't1.i1' is ambiguous, could be: i1#x, i1#x.; line 1 pos 7 -- !query 10 @@ -99,7 +99,7 @@ SELECT i1 FROM t1, mydb2.t1 struct<> -- !query 11 output org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: i1#601, i1#602.; line 1 pos 7 +Reference 'i1' is ambiguous, could be: i1#x, i1#x.; line 1 pos 7 -- !query 12 @@ -108,7 +108,7 @@ SELECT t1.i1 FROM t1, mydb2.t1 struct<> -- !query 12 output org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: i1#605, i1#606.; line 1 pos 7 +Reference 't1.i1' is ambiguous, could be: i1#x, i1#x.; line 1 pos 7 -- !query 13 @@ -125,7 +125,7 @@ SELECT i1 FROM t1, mydb1.t1 struct<> -- !query 14 output org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: i1#610, i1#611.; line 1 pos 7 +Reference 'i1' is ambiguous, could be: i1#x, i1#x.; line 1 pos 7 -- !query 15 @@ -134,7 +134,7 @@ SELECT t1.i1 FROM t1, mydb1.t1 struct<> -- !query 15 output org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: i1#614, i1#615.; line 1 pos 7 +Reference 't1.i1' is ambiguous, could be: i1#x, i1#x.; line 1 pos 7 -- !query 16 @@ -143,7 +143,7 @@ SELECT i1 FROM t1, mydb2.t1 struct<> -- !query 16 output org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: i1#618, i1#619.; line 1 pos 7 +Reference 'i1' is ambiguous, could be: i1#x, i1#x.; line 1 pos 7 -- !query 17 @@ -152,7 +152,7 @@ SELECT t1.i1 FROM t1, mydb2.t1 struct<> -- !query 17 output org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: i1#622, i1#623.; line 1 pos 7 +Reference 't1.i1' is ambiguous, could be: i1#x, i1#x.; line 1 pos 7 -- !query 18 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 0b3da9aa8fbe..f7d554209afb 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -228,12 +228,13 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted) } catch { - case a: AnalysisException if a.plan.nonEmpty => + case a: AnalysisException => // Do not output the logical plan tree which contains expression IDs. // Also implement a crude way of masking expression IDs in the error message // with a generic pattern "###". + val msg = if (a.plan.nonEmpty) a.getSimpleMessage else a.getMessage (StructType(Seq.empty), - Seq(a.getClass.getName, a.getSimpleMessage.replaceAll("#\\d+", "#x"))) + Seq(a.getClass.getName, msg.replaceAll("#\\d+", "#x"))) case NonFatal(e) => // If there is an exception, put the exception class followed by the message. (StructType(Seq.empty), Seq(e.getClass.getName, e.getMessage)) From 5594eb0864376bbac617bf744755330f1e7bff49 Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Wed, 1 Mar 2017 15:44:59 -0800 Subject: [PATCH 18/19] Delete the ColumnResolutionSuite --- .../execution/ColumnResolutionSuite.scala | 171 ------------------ 1 file changed, 171 deletions(-) delete mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala deleted file mode 100644 index 7024b37b0fcf..000000000000 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ColumnResolutionSuite.scala +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.{AnalysisException, QueryTest, Row} -import org.apache.spark.sql.hive.test.TestHiveSingleton -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.test.SQLTestUtils - -class ColumnResolutionSuite extends QueryTest with SQLTestUtils with TestHiveSingleton { - - def columnResolutionTests(db1: String, db2: String): Unit = { - spark.catalog.setCurrentDatabase(db1) - - checkAnswer(spark.sql("SELECT i1 FROM t1"), Row(1)) - checkAnswer(spark.sql(s"SELECT i1 FROM $db1.t1"), Row(1)) - - checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(1)) - checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1") - } - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") - } - - // Change current database to db2 - spark.catalog.setCurrentDatabase(db2) - checkAnswer(spark.sql("SELECT i1 FROM t1"), Row(20)) - checkAnswer(spark.sql(s"SELECT i1 FROM $db1.t1"), Row(1)) - - checkAnswer(spark.sql("SELECT t1.i1 FROM t1"), Row(20)) - checkAnswer(spark.sql(s"SELECT t1.i1 FROM $db1.t1"), Row(1)) - - // TODO: Support this scenario - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM $db1.t1") - } - } - - test("column resolution scenarios with non datasource table") { - val currentDb = spark.catalog.currentDatabase - withTempDatabase { db1 => - withTempDatabase { db2 => - try { - spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1 AS SELECT 1 AS i1") - spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") - - columnResolutionTests(db1, db2) - } finally { - spark.catalog.setCurrentDatabase(currentDb) - } - } - } - } - - test("column resolution scenarios with ambiguous cases in join queries - negative cases") { - val currentDb = spark.catalog.currentDatabase - withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") { - withTempDatabase { db1 => - withTempDatabase { db2 => - withTempPath { f => - try { - spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1 AS SELECT 1 AS i1") - spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") - spark.catalog.setCurrentDatabase(db1) - - intercept[AnalysisException] { - spark.sql(s"SELECT i1 FROM t1, $db1.t1") - } - - intercept[AnalysisException] { - spark.sql(s"SELECT t1.i1 FROM t1, $db1.t1") - } - - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db1.t1") - } - - intercept[AnalysisException] { - spark.sql(s"SELECT i1 FROM t1, $db2.t1") - } - - intercept[AnalysisException] { - spark.sql(s"SELECT t1.i1 FROM t1, $db2.t1") - } - - spark.catalog.setCurrentDatabase(db2) - - intercept[AnalysisException] { - spark.sql(s"SELECT i1 FROM t1, $db1.t1") - } - - intercept[AnalysisException] { - spark.sql(s"SELECT t1.i1 FROM t1, $db1.t1") - } - - intercept[AnalysisException] { - spark.sql(s"SELECT i1 FROM t1, $db2.t1") - } - - intercept[AnalysisException] { - spark.sql(s"SELECT t1.i1 FROM t1, $db2.t1") - } - - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1, $db2.t1") - } - } finally { - spark.catalog.setCurrentDatabase(currentDb) - } - } - } - } - } - } - - test("col resolution - error case") { - val currentDb = spark.catalog.currentDatabase - withTempDatabase { db2 => - withTempDatabase { db1 => - withTempPath { f => - try { - spark.catalog.setCurrentDatabase(db1) - spark.sql("CREATE TABLE t1 AS SELECT 1 AS i1") - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1 FROM t1") - } - intercept[AnalysisException] { - spark.sql(s"SELECT t1.x.y.* FROM t1") - } - intercept[AnalysisException] { - spark.sql(s"SELECT t1 FROM $db1.t1") - } - - spark.catalog.setCurrentDatabase(db2) - spark.sql("CREATE TABLE t1 AS SELECT 20 AS i1") - intercept[AnalysisException] { - spark.sql(s"SELECT $db1.t1.i1 FROM t1") - } - } finally { - spark.catalog.setCurrentDatabase(currentDb) - } - } - } - } - } -} From b2e411c859f4266428db6b4fbf55d89a101e50bf Mon Sep 17 00:00:00 2001 From: Sunitha Kambhampati Date: Thu, 2 Mar 2017 09:11:17 -0800 Subject: [PATCH 19/19] address comments --- .../inputs/columnresolution-negative.sql | 6 +- .../inputs/columnresolution-views.sql | 34 ++++---- .../sql-tests/inputs/columnresolution.sql | 6 +- .../results/columnresolution-negative.sql.out | 38 ++++----- .../results/columnresolution-views.sql.out | 38 ++++----- .../results/columnresolution.sql.out | 85 ++++++++++++------- .../apache/spark/sql/SQLQueryTestSuite.scala | 3 +- 7 files changed, 119 insertions(+), 91 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution-negative.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution-negative.sql index 4afb2cf6915c..1caa45c66749 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/columnresolution-negative.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution-negative.sql @@ -8,7 +8,7 @@ USE mydb2; CREATE TABLE t1 USING parquet AS SELECT 20 AS i1; -- Negative tests: column resolution scenarios with ambiguous cases in join queries -set spark.sql.crossJoin.enabled = true; +SET spark.sql.crossJoin.enabled = true; USE mydb1; SELECT i1 FROM t1, mydb1.t1; SELECT t1.i1 FROM t1, mydb1.t1; @@ -21,6 +21,7 @@ SELECT t1.i1 FROM t1, mydb1.t1; SELECT i1 FROM t1, mydb2.t1; SELECT t1.i1 FROM t1, mydb2.t1; SELECT db1.t1.i1 FROM t1, mydb2.t1; +SET spark.sql.crossJoin.enabled = false; -- Negative tests USE mydb1; @@ -31,6 +32,5 @@ USE mydb2; SELECT mydb1.t1.i1 FROM t1; -- reset -set spark.sql.crossJoin.enabled = false; DROP DATABASE mydb1 CASCADE; -DROP DATABASE mydb2 CASCADE; \ No newline at end of file +DROP DATABASE mydb2 CASCADE; diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution-views.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution-views.sql index 71f13e3cbcef..d3f928751757 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/columnresolution-views.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution-views.sql @@ -1,25 +1,25 @@ -- Tests for qualified column names for the view code-path -- Test scenario with Temporary view -CREATE OR REPLACE TEMPORARY VIEW table1 AS SELECT 2 AS i1; -SELECT table1.* FROM table1; -SELECT * FROM table1; -SELECT table1.i1 FROM table1; -SELECT i1 FROM table1; -SELECT a.i1 FROM table1 AS a; -SELECT i1 FROM table1 AS a; +CREATE OR REPLACE TEMPORARY VIEW view1 AS SELECT 2 AS i1; +SELECT view1.* FROM view1; +SELECT * FROM view1; +SELECT view1.i1 FROM view1; +SELECT i1 FROM view1; +SELECT a.i1 FROM view1 AS a; +SELECT i1 FROM view1 AS a; -- cleanup -DROP VIEW table1; +DROP VIEW view1; -- Test scenario with Global Temp view -CREATE OR REPLACE GLOBAL TEMPORARY VIEW t1 as SELECT 1 as i1; -SELECT * FROM global_temp.t1; +CREATE OR REPLACE GLOBAL TEMPORARY VIEW view1 as SELECT 1 as i1; +SELECT * FROM global_temp.view1; -- TODO: Support this scenario -SELECT global_temp.t1.* FROM global_temp.t1; -SELECT i1 FROM global_temp.t1; +SELECT global_temp.view1.* FROM global_temp.view1; +SELECT i1 FROM global_temp.view1; -- TODO: Support this scenario -SELECT global_temp.t1.i1 FROM global_temp.t1; -SELECT t1.i1 FROM global_temp.t1; -SELECT a.i1 FROM global_temp.t1 AS a; -SELECT i1 FROM global_temp.t1 AS a; +SELECT global_temp.view1.i1 FROM global_temp.view1; +SELECT view1.i1 FROM global_temp.view1; +SELECT a.i1 FROM global_temp.view1 AS a; +SELECT i1 FROM global_temp.view1 AS a; -- cleanup -DROP VIEW global_temp.t1; +DROP VIEW global_temp.view1; diff --git a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql index 7941ed8a37e7..79e90ad3de91 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/columnresolution.sql @@ -63,6 +63,7 @@ SELECT mydb1.t1.i1 FROM mydb1.t1, mydb2.t1; USE mydb2; -- TODO: Support this scenario SELECT mydb1.t1.i1 FROM t1, mydb1.t1; +SET spark.sql.crossJoin.enabled = false; -- Scenario: Table with struct column USE mydb1; @@ -72,13 +73,16 @@ SELECT t5.i1 FROM t5; SELECT t5.t5.i1 FROM t5; SELECT t5.t5.i1 FROM mydb1.t5; SELECT t5.i1 FROM mydb1.t5; +SELECT t5.* FROM mydb1.t5; +SELECT t5.t5.* FROM mydb1.t5; -- TODO: Support this scenario SELECT mydb1.t5.t5.i1 FROM mydb1.t5; -- TODO: Support this scenario SELECT mydb1.t5.t5.i2 FROM mydb1.t5; +-- TODO: Support this scenario +SELECT mydb1.t5.* FROM mydb1.t5; -- Cleanup and Reset -SET spark.sql.crossJoin.enabled = false; USE default; DROP DATABASE mydb1 CASCADE; DROP DATABASE mydb2 CASCADE; diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out index df4733d4ff1a..60bd8e9cc99d 100644 --- a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out @@ -51,7 +51,7 @@ struct<> -- !query 6 -set spark.sql.crossJoin.enabled = true +SET spark.sql.crossJoin.enabled = true -- !query 6 schema struct -- !query 6 output @@ -165,63 +165,63 @@ cannot resolve '`db1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 -- !query 19 -USE mydb1 +SET spark.sql.crossJoin.enabled = false -- !query 19 schema -struct<> +struct -- !query 19 output - +spark.sql.crossJoin.enabled false -- !query 20 -SELECT mydb1.t1 FROM t1 +USE mydb1 -- !query 20 schema struct<> -- !query 20 output -org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t1`' given input columns: [i1]; line 1 pos 7 + -- !query 21 -SELECT t1.x.y.* FROM t1 +SELECT mydb1.t1 FROM t1 -- !query 21 schema struct<> -- !query 21 output org.apache.spark.sql.AnalysisException -cannot resolve 't1.x.y.*' give input columns 'i1'; +cannot resolve '`mydb1.t1`' given input columns: [i1]; line 1 pos 7 -- !query 22 -SELECT t1 FROM mydb1.t1 +SELECT t1.x.y.* FROM t1 -- !query 22 schema struct<> -- !query 22 output org.apache.spark.sql.AnalysisException -cannot resolve '`t1`' given input columns: [i1]; line 1 pos 7 +cannot resolve 't1.x.y.*' give input columns 'i1'; -- !query 23 -USE mydb2 +SELECT t1 FROM mydb1.t1 -- !query 23 schema struct<> -- !query 23 output - +org.apache.spark.sql.AnalysisException +cannot resolve '`t1`' given input columns: [i1]; line 1 pos 7 -- !query 24 -SELECT mydb1.t1.i1 FROM t1 +USE mydb2 -- !query 24 schema struct<> -- !query 24 output -org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 + -- !query 25 -set spark.sql.crossJoin.enabled = false +SELECT mydb1.t1.i1 FROM t1 -- !query 25 schema -struct +struct<> -- !query 25 output -spark.sql.crossJoin.enabled false +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t1.i1`' given input columns: [i1]; line 1 pos 7 -- !query 26 diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-views.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-views.sql.out index 4ab35a0e76ce..616421d6f2b2 100644 --- a/sql/core/src/test/resources/sql-tests/results/columnresolution-views.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-views.sql.out @@ -3,7 +3,7 @@ -- !query 0 -CREATE OR REPLACE TEMPORARY VIEW table1 AS SELECT 2 AS i1 +CREATE OR REPLACE TEMPORARY VIEW view1 AS SELECT 2 AS i1 -- !query 0 schema struct<> -- !query 0 output @@ -11,7 +11,7 @@ struct<> -- !query 1 -SELECT table1.* FROM table1 +SELECT view1.* FROM view1 -- !query 1 schema struct -- !query 1 output @@ -19,7 +19,7 @@ struct -- !query 2 -SELECT * FROM table1 +SELECT * FROM view1 -- !query 2 schema struct -- !query 2 output @@ -27,7 +27,7 @@ struct -- !query 3 -SELECT table1.i1 FROM table1 +SELECT view1.i1 FROM view1 -- !query 3 schema struct -- !query 3 output @@ -35,7 +35,7 @@ struct -- !query 4 -SELECT i1 FROM table1 +SELECT i1 FROM view1 -- !query 4 schema struct -- !query 4 output @@ -43,7 +43,7 @@ struct -- !query 5 -SELECT a.i1 FROM table1 AS a +SELECT a.i1 FROM view1 AS a -- !query 5 schema struct -- !query 5 output @@ -51,7 +51,7 @@ struct -- !query 6 -SELECT i1 FROM table1 AS a +SELECT i1 FROM view1 AS a -- !query 6 schema struct -- !query 6 output @@ -59,7 +59,7 @@ struct -- !query 7 -DROP VIEW table1 +DROP VIEW view1 -- !query 7 schema struct<> -- !query 7 output @@ -67,7 +67,7 @@ struct<> -- !query 8 -CREATE OR REPLACE GLOBAL TEMPORARY VIEW t1 as SELECT 1 as i1 +CREATE OR REPLACE GLOBAL TEMPORARY VIEW view1 as SELECT 1 as i1 -- !query 8 schema struct<> -- !query 8 output @@ -75,7 +75,7 @@ struct<> -- !query 9 -SELECT * FROM global_temp.t1 +SELECT * FROM global_temp.view1 -- !query 9 schema struct -- !query 9 output @@ -83,16 +83,16 @@ struct -- !query 10 -SELECT global_temp.t1.* FROM global_temp.t1 +SELECT global_temp.view1.* FROM global_temp.view1 -- !query 10 schema struct<> -- !query 10 output org.apache.spark.sql.AnalysisException -cannot resolve 'global_temp.t1.*' give input columns 'i1'; +cannot resolve 'global_temp.view1.*' give input columns 'i1'; -- !query 11 -SELECT i1 FROM global_temp.t1 +SELECT i1 FROM global_temp.view1 -- !query 11 schema struct -- !query 11 output @@ -100,16 +100,16 @@ struct -- !query 12 -SELECT global_temp.t1.i1 FROM global_temp.t1 +SELECT global_temp.view1.i1 FROM global_temp.view1 -- !query 12 schema struct<> -- !query 12 output org.apache.spark.sql.AnalysisException -cannot resolve '`global_temp.t1.i1`' given input columns: [i1]; line 1 pos 7 +cannot resolve '`global_temp.view1.i1`' given input columns: [i1]; line 1 pos 7 -- !query 13 -SELECT t1.i1 FROM global_temp.t1 +SELECT view1.i1 FROM global_temp.view1 -- !query 13 schema struct -- !query 13 output @@ -117,7 +117,7 @@ struct -- !query 14 -SELECT a.i1 FROM global_temp.t1 AS a +SELECT a.i1 FROM global_temp.view1 AS a -- !query 14 schema struct -- !query 14 output @@ -125,7 +125,7 @@ struct -- !query 15 -SELECT i1 FROM global_temp.t1 AS a +SELECT i1 FROM global_temp.view1 AS a -- !query 15 schema struct -- !query 15 output @@ -133,7 +133,7 @@ struct -- !query 16 -DROP VIEW global_temp.t1 +DROP VIEW global_temp.view1 -- !query 16 schema struct<> -- !query 16 output diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out index b28b750c810d..764cad0e3943 100644 --- a/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 51 +-- Number of queries: 54 -- !query 0 @@ -317,15 +317,15 @@ cannot resolve '`mydb1.t1.i1`' given input columns: [i1, i1]; line 1 pos 7 -- !query 38 -USE mydb1 +SET spark.sql.crossJoin.enabled = false -- !query 38 schema -struct<> +struct -- !query 38 output - +spark.sql.crossJoin.enabled false -- !query 39 -CREATE TABLE t5(i1 INT, t5 STRUCT) USING parquet +USE mydb1 -- !query 39 schema struct<> -- !query 39 output @@ -333,7 +333,7 @@ struct<> -- !query 40 -INSERT INTO t5 VALUES(1, (2, 3)) +CREATE TABLE t5(i1 INT, t5 STRUCT) USING parquet -- !query 40 schema struct<> -- !query 40 output @@ -341,23 +341,23 @@ struct<> -- !query 41 -SELECT t5.i1 FROM t5 +INSERT INTO t5 VALUES(1, (2, 3)) -- !query 41 schema -struct +struct<> -- !query 41 output -1 + -- !query 42 -SELECT t5.t5.i1 FROM t5 +SELECT t5.i1 FROM t5 -- !query 42 schema struct -- !query 42 output -2 +1 -- !query 43 -SELECT t5.t5.i1 FROM mydb1.t5 +SELECT t5.t5.i1 FROM t5 -- !query 43 schema struct -- !query 43 output @@ -365,58 +365,83 @@ struct -- !query 44 -SELECT t5.i1 FROM mydb1.t5 +SELECT t5.t5.i1 FROM mydb1.t5 -- !query 44 schema struct -- !query 44 output -1 +2 -- !query 45 -SELECT mydb1.t5.t5.i1 FROM mydb1.t5 +SELECT t5.i1 FROM mydb1.t5 -- !query 45 schema -struct<> +struct -- !query 45 output -org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t5.t5.i1`' given input columns: [i1, t5]; line 1 pos 7 +1 -- !query 46 -SELECT mydb1.t5.t5.i2 FROM mydb1.t5 +SELECT t5.* FROM mydb1.t5 -- !query 46 schema -struct<> +struct> -- !query 46 output -org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t5.t5.i2`' given input columns: [i1, t5]; line 1 pos 7 +1 {"i1":2,"i2":3} -- !query 47 -SET spark.sql.crossJoin.enabled = false +SELECT t5.t5.* FROM mydb1.t5 -- !query 47 schema -struct +struct -- !query 47 output -spark.sql.crossJoin.enabled false +2 3 -- !query 48 -USE default +SELECT mydb1.t5.t5.i1 FROM mydb1.t5 -- !query 48 schema struct<> -- !query 48 output - +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t5.t5.i1`' given input columns: [i1, t5]; line 1 pos 7 -- !query 49 -DROP DATABASE mydb1 CASCADE +SELECT mydb1.t5.t5.i2 FROM mydb1.t5 -- !query 49 schema struct<> -- !query 49 output - +org.apache.spark.sql.AnalysisException +cannot resolve '`mydb1.t5.t5.i2`' given input columns: [i1, t5]; line 1 pos 7 -- !query 50 -DROP DATABASE mydb2 CASCADE +SELECT mydb1.t5.* FROM mydb1.t5 -- !query 50 schema struct<> -- !query 50 output +org.apache.spark.sql.AnalysisException +cannot resolve 'mydb1.t5.*' give input columns 'i1, t5'; + + +-- !query 51 +USE default +-- !query 51 schema +struct<> +-- !query 51 output + + + +-- !query 52 +DROP DATABASE mydb1 CASCADE +-- !query 52 schema +struct<> +-- !query 52 output + + + +-- !query 53 +DROP DATABASE mydb2 CASCADE +-- !query 53 schema +struct<> +-- !query 53 output diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index f7d554209afb..68ababcd1102 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -233,8 +233,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { // Also implement a crude way of masking expression IDs in the error message // with a generic pattern "###". val msg = if (a.plan.nonEmpty) a.getSimpleMessage else a.getMessage - (StructType(Seq.empty), - Seq(a.getClass.getName, msg.replaceAll("#\\d+", "#x"))) + (StructType(Seq.empty), Seq(a.getClass.getName, msg.replaceAll("#\\d+", "#x"))) case NonFatal(e) => // If there is an exception, put the exception class followed by the message. (StructType(Seq.empty), Seq(e.getClass.getName, e.getMessage))