Skip to content

[SPARK-41181][SQL] Migrate the map options errors onto error classes #38730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
panbingkun wants to merge 6 commits into from
Closed

[SPARK-41181][SQL] Migrate the map options errors onto error classes #38730

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d23e7d4
[SPARK-41181][SQL] Migrate the map options errors onto error classes
panbingkun Nov 21, 2022
22662ee
[SPARK-41181][SQL] Migrate the map options errors onto error classes
panbingkun Nov 21, 2022
4fef9fd
[SPARK-41181][SQL] Migrate the map options errors onto error classes
panbingkun Nov 24, 2022
3c28cd8
[SPARK-41181][SQL] Migrate the map options errors onto error classes
panbingkun Nov 24, 2022
677b977
[SPARK-41181][SQL] Migrate the map options errors onto error classes
panbingkun Nov 24, 2022
8becd41
[SPARK-41181][SQL] Migrate the map options errors onto error classes
panbingkun Nov 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 17 additions & 10 deletions core/src/main/resources/error/error-classes.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -730,6 +730,23 @@
"The <joinType> JOIN with LATERAL correlation is not allowed because an OUTER subquery cannot correlate to its join partner. Remove the LATERAL correlation or use an INNER JOIN, or LEFT OUTER JOIN instead."
]
},
"INVALID_OPTIONS" : {
"message" : [
"Invalid options:"
],
"subClass" : {
"NON_STRING_TYPE" : {
"message" : [
"A type of keys and values in map() must be string, but got <map>."
Copy link
Member

@MaxGekk MaxGekk Nov 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: map -> mapType

]
},
"NOT_MAP_FUNCTION" : {
Copy link
Member

@MaxGekk MaxGekk Nov 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not NON_MAP_FUNCTION?

"message" : [
"Must use a map() function for options."
Copy link
Member

@MaxGekk MaxGekk Nov 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"Must use a map() function for options."
"Must use the `map()` function for options."

]
}
}
},
"INVALID_PANDAS_UDF_PLACEMENT" : {
"message" : [
"The group aggregate pandas UDF <functionList> cannot be invoked together with as other, non-pandas aggregate functions."
Expand Down Expand Up @@ -2185,16 +2202,6 @@
"Schema should be struct type but got <dataType>."
]
},
"_LEGACY_ERROR_TEMP_1095" : {
"message" : [
"A type of keys and values in map() must be string, but got <map>."
]
},
"_LEGACY_ERROR_TEMP_1096" : {
"message" : [
"Must use a map() function for options."
]
},
"_LEGACY_ERROR_TEMP_1097" : {
"message" : [
"The field for corrupt records must be string type and nullable."
Expand Down
6 changes: 3 additions & 3 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1013,13 +1013,13 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase {

def keyValueInMapNotStringError(m: CreateMap): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1095",
messageParameters = Map("map" -> m.dataType.catalogString))
errorClass = "INVALID_OPTIONS.NON_STRING_TYPE",
messageParameters = Map("map" -> toSQLType(m.dataType)))
}

def nonMapFunctionNotAllowedError(): Throwable = {
Copy link
Member

@MaxGekk MaxGekk Nov 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Be sure the names are consistent. not or non

new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1096",
errorClass = "INVALID_OPTIONS.NOT_MAP_FUNCTION",
messageParameters = Map.empty)
}

Expand Down
13 changes: 7 additions & 6 deletions sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ struct<>
-- !query output
org.apache.spark.sql.AnalysisException
{
"errorClass" : "_LEGACY_ERROR_TEMP_1096",
"errorClass" : "INVALID_OPTIONS.NOT_MAP_FUNCTION",
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
Expand All @@ -84,9 +84,9 @@ struct<>
-- !query output
org.apache.spark.sql.AnalysisException
{
"errorClass" : "_LEGACY_ERROR_TEMP_1095",
"errorClass" : "INVALID_OPTIONS.NON_STRING_TYPE",
"messageParameters" : {
"map" : "map<string,int>"
"map" : "\"MAP<STRING, INT>\""
},
"queryContext" : [ {
"objectType" : "",
Expand Down Expand Up @@ -222,7 +222,7 @@ struct<>
-- !query output
org.apache.spark.sql.AnalysisException
{
"errorClass" : "_LEGACY_ERROR_TEMP_1096",
"errorClass" : "INVALID_OPTIONS.NOT_MAP_FUNCTION",
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
Expand All @@ -233,16 +233,17 @@ org.apache.spark.sql.AnalysisException
}



-- !query
select to_csv(named_struct('a', 1, 'b', 2), map('mode', 1))
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
{
"errorClass" : "_LEGACY_ERROR_TEMP_1095",
"errorClass" : "INVALID_OPTIONS.NON_STRING_TYPE",
"messageParameters" : {
"map" : "map<string,int>"
"map" : "\"MAP<STRING, INT>\""
},
"queryContext" : [ {
"objectType" : "",
Expand Down
12 changes: 6 additions & 6 deletions sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ struct<>
-- !query output
org.apache.spark.sql.AnalysisException
{
"errorClass" : "_LEGACY_ERROR_TEMP_1096",
"errorClass" : "INVALID_OPTIONS.NOT_MAP_FUNCTION",
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
Expand All @@ -88,9 +88,9 @@ struct<>
-- !query output
org.apache.spark.sql.AnalysisException
{
"errorClass" : "_LEGACY_ERROR_TEMP_1095",
"errorClass" : "INVALID_OPTIONS.NON_STRING_TYPE",
"messageParameters" : {
"map" : "map<string,int>"
"map" : "\"MAP<STRING, INT>\""
},
"queryContext" : [ {
"objectType" : "",
Expand Down Expand Up @@ -192,7 +192,7 @@ struct<>
-- !query output
org.apache.spark.sql.AnalysisException
{
"errorClass" : "_LEGACY_ERROR_TEMP_1096",
"errorClass" : "INVALID_OPTIONS.NOT_MAP_FUNCTION",
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
Expand All @@ -210,9 +210,9 @@ struct<>
-- !query output
org.apache.spark.sql.AnalysisException
{
"errorClass" : "_LEGACY_ERROR_TEMP_1095",
"errorClass" : "INVALID_OPTIONS.NON_STRING_TYPE",
"messageParameters" : {
"map" : "map<string,int>"
"map" : "\"MAP<STRING, INT>\""
},
"queryContext" : [ {
"objectType" : "",
Expand Down
66 changes: 48 additions & 18 deletions sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -395,16 +395,31 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
df2.selectExpr("to_json(a, map('timestampFormat', 'dd/MM/yyyy HH:mm'))"),
Row("""{"_1":"26/08/2015 18:00"}""") :: Nil)

val errMsg1 = intercept[AnalysisException] {
df2.selectExpr("to_json(a, named_struct('a', 1))")
}
assert(errMsg1.getMessage.startsWith("Must use a map() function for options"))
checkError(
exception = intercept[AnalysisException] {
df2.selectExpr("to_json(a, named_struct('a', 1))")
},
errorClass = "INVALID_OPTIONS.NOT_MAP_FUNCTION",
parameters = Map.empty,
context = ExpectedContext(
fragment = "to_json(a, named_struct('a', 1))",
start = 0,
stop = 31
)
)

val errMsg2 = intercept[AnalysisException] {
df2.selectExpr("to_json(a, map('a', 1))")
}
assert(errMsg2.getMessage.startsWith(
"A type of keys and values in map() must be string, but got"))
checkError(
exception = intercept[AnalysisException] {
df2.selectExpr("to_json(a, map('a', 1))")
},
errorClass = "INVALID_OPTIONS.NON_STRING_TYPE",
parameters = Map("map" -> "\"MAP<STRING, INT>\""),
context = ExpectedContext(
fragment = "to_json(a, map('a', 1))",
start = 0,
stop = 22
)
)
}

test("SPARK-19967 Support from_json in SQL") {
Expand Down Expand Up @@ -441,15 +456,30 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
df3.selectExpr("""from_json(value, 'time InvalidType')""")
}
assert(errMsg2.getMessage.contains("DataType invalidtype is not supported"))
val errMsg3 = intercept[AnalysisException] {
df3.selectExpr("from_json(value, 'time Timestamp', named_struct('a', 1))")
}
assert(errMsg3.getMessage.startsWith("Must use a map() function for options"))
val errMsg4 = intercept[AnalysisException] {
df3.selectExpr("from_json(value, 'time Timestamp', map('a', 1))")
}
assert(errMsg4.getMessage.startsWith(
"A type of keys and values in map() must be string, but got"))
checkError(
exception = intercept[AnalysisException] {
df3.selectExpr("from_json(value, 'time Timestamp', named_struct('a', 1))")
},
errorClass = "INVALID_OPTIONS.NOT_MAP_FUNCTION",
parameters = Map.empty,
context = ExpectedContext(
fragment = "from_json(value, 'time Timestamp', named_struct('a', 1))",
start = 0,
stop = 55
)
)
checkError(
exception = intercept[AnalysisException] {
df3.selectExpr("from_json(value, 'time Timestamp', map('a', 1))")
},
errorClass = "INVALID_OPTIONS.NON_STRING_TYPE",
parameters = Map("map" -> "\"MAP<STRING, INT>\""),
context = ExpectedContext(
fragment = "from_json(value, 'time Timestamp', map('a', 1))",
start = 0,
stop = 46
)
)
}

test("SPARK-24027: from_json - map<string, int>") {
Expand Down