diff --git a/core/pom.xml b/core/pom.xml index af7687854ed54..f31fbed404fc1 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -224,6 +224,10 @@ org.scala-lang scala-library + + org.scala-lang + scala-reflect + org.json4s json4s-jackson_${scala.binary.version} diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala index ef5a7e35ad562..97b689cdadd5f 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala @@ -36,7 +36,7 @@ import org.apache.spark.util.Utils * (2) the Spark version of the client / server * (3) an optional message */ -@JsonInclude(Include.NON_NULL) +@JsonInclude(Include.NON_ABSENT) @JsonAutoDetect(getterVisibility = Visibility.ANY, setterVisibility = Visibility.ANY) @JsonPropertyOrder(alphabetic = true) private[rest] abstract class SubmitRestProtocolMessage { diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala index 53d69ba26811f..3abb2d8a11f35 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala @@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging * There is no particular relationship between an operation scope and a stage or a job. * A scope may live inside one stage (e.g. map) or span across multiple jobs (e.g. take). */ -@JsonInclude(Include.NON_NULL) +@JsonInclude(Include.NON_ABSENT) @JsonPropertyOrder(Array("id", "name", "parent")) private[spark] class RDDOperationScope( val name: String, diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala b/core/src/main/scala/org/apache/spark/status/KVUtils.scala index 99b1843d8e1c0..45348be5c98b9 100644 --- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala +++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala @@ -42,7 +42,7 @@ private[spark] object KVUtils extends Logging { private[spark] class KVStoreScalaSerializer extends KVStoreSerializer { mapper.registerModule(DefaultScalaModule) - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) + mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT) } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala index 4560d300cb0c8..50a286d0d3b0f 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala @@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{ } mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule) mapper.enable(SerializationFeature.INDENT_OUTPUT) - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) + mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT) mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat) override def isWriteable( diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index b4fad23a4b114..dd149f2ed81e1 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -86,16 +86,16 @@ htrace-core/3.0.4//htrace-core-3.0.4.jar httpclient/4.5.6//httpclient-4.5.6.jar httpcore/4.4.10//httpcore-4.4.10.jar ivy/2.4.0//ivy-2.4.0.jar -jackson-annotations/2.6.7//jackson-annotations-2.6.7.jar +jackson-annotations/2.9.10//jackson-annotations-2.9.10.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar -jackson-core/2.6.7//jackson-core-2.6.7.jar -jackson-databind/2.6.7.3//jackson-databind-2.6.7.3.jar -jackson-dataformat-yaml/2.6.7//jackson-dataformat-yaml-2.6.7.jar +jackson-core/2.9.10//jackson-core-2.9.10.jar +jackson-databind/2.9.10.5//jackson-databind-2.9.10.5.jar +jackson-dataformat-yaml/2.9.10//jackson-dataformat-yaml-2.9.10.jar jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations/2.6.7//jackson-module-jaxb-annotations-2.6.7.jar -jackson-module-paranamer/2.7.9//jackson-module-paranamer-2.7.9.jar -jackson-module-scala_2.11/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar +jackson-module-jaxb-annotations/2.9.10//jackson-module-jaxb-annotations-2.9.10.jar +jackson-module-paranamer/2.9.10//jackson-module-paranamer-2.9.10.jar +jackson-module-scala_2.11/2.9.10//jackson-module-scala_2.11-2.9.10.jar jackson-xc/1.9.13//jackson-xc-1.9.13.jar janino/3.0.16//janino-3.0.16.jar javassist/3.18.1-GA//javassist-3.18.1-GA.jar @@ -178,7 +178,7 @@ shapeless_2.11/2.3.2//shapeless_2.11-2.3.2.jar shims/0.7.45//shims-0.7.45.jar slf4j-api/1.7.16//slf4j-api-1.7.16.jar slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar -snakeyaml/1.15//snakeyaml-1.15.jar +snakeyaml/1.23//snakeyaml-1.23.jar snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar snappy/0.2//snappy-0.2.jar spire-macros_2.11/0.13.0//spire-macros_2.11-0.13.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 3dcb4d724cf26..2c265fd77a761 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -86,16 +86,16 @@ htrace-core/3.1.0-incubating//htrace-core-3.1.0-incubating.jar httpclient/4.5.6//httpclient-4.5.6.jar httpcore/4.4.10//httpcore-4.4.10.jar ivy/2.4.0//ivy-2.4.0.jar -jackson-annotations/2.6.7//jackson-annotations-2.6.7.jar +jackson-annotations/2.9.10//jackson-annotations-2.9.10.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar -jackson-core/2.6.7//jackson-core-2.6.7.jar -jackson-databind/2.6.7.3//jackson-databind-2.6.7.3.jar -jackson-dataformat-yaml/2.6.7//jackson-dataformat-yaml-2.6.7.jar +jackson-core/2.9.10//jackson-core-2.9.10.jar +jackson-databind/2.9.10.5//jackson-databind-2.9.10.5.jar +jackson-dataformat-yaml/2.9.10//jackson-dataformat-yaml-2.9.10.jar jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations/2.6.7//jackson-module-jaxb-annotations-2.6.7.jar -jackson-module-paranamer/2.7.9//jackson-module-paranamer-2.7.9.jar -jackson-module-scala_2.11/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar +jackson-module-jaxb-annotations/2.9.10//jackson-module-jaxb-annotations-2.9.10.jar +jackson-module-paranamer/2.9.10//jackson-module-paranamer-2.9.10.jar +jackson-module-scala_2.11/2.9.10//jackson-module-scala_2.11-2.9.10.jar jackson-xc/1.9.13//jackson-xc-1.9.13.jar janino/3.0.16//janino-3.0.16.jar javassist/3.18.1-GA//javassist-3.18.1-GA.jar @@ -179,7 +179,7 @@ shapeless_2.11/2.3.2//shapeless_2.11-2.3.2.jar shims/0.7.45//shims-0.7.45.jar slf4j-api/1.7.16//slf4j-api-1.7.16.jar slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar -snakeyaml/1.15//snakeyaml-1.15.jar +snakeyaml/1.23//snakeyaml-1.23.jar snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar snappy/0.2//snappy-0.2.jar spire-macros_2.11/0.13.0//spire-macros_2.11-0.13.0.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 5d033460ed0f0..67651b51b44b2 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -86,17 +86,17 @@ htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar httpclient/4.5.6//httpclient-4.5.6.jar httpcore/4.4.10//httpcore-4.4.10.jar ivy/2.4.0//ivy-2.4.0.jar -jackson-annotations/2.6.7//jackson-annotations-2.6.7.jar +jackson-annotations/2.9.10//jackson-annotations-2.9.10.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar -jackson-core/2.6.7//jackson-core-2.6.7.jar -jackson-databind/2.6.7.3//jackson-databind-2.6.7.3.jar -jackson-dataformat-yaml/2.6.7//jackson-dataformat-yaml-2.6.7.jar +jackson-core/2.9.10//jackson-core-2.9.10.jar +jackson-databind/2.9.10.5//jackson-databind-2.9.10.5.jar +jackson-dataformat-yaml/2.9.10//jackson-dataformat-yaml-2.9.10.jar jackson-jaxrs-base/2.7.8//jackson-jaxrs-base-2.7.8.jar jackson-jaxrs-json-provider/2.7.8//jackson-jaxrs-json-provider-2.7.8.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations/2.6.7//jackson-module-jaxb-annotations-2.6.7.jar -jackson-module-paranamer/2.7.9//jackson-module-paranamer-2.7.9.jar -jackson-module-scala_2.11/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar +jackson-module-jaxb-annotations/2.9.10//jackson-module-jaxb-annotations-2.9.10.jar +jackson-module-paranamer/2.9.10//jackson-module-paranamer-2.9.10.jar +jackson-module-scala_2.11/2.9.10//jackson-module-scala_2.11-2.9.10.jar janino/3.0.16//janino-3.0.16.jar javassist/3.18.1-GA//javassist-3.18.1-GA.jar javax.annotation-api/1.2//javax.annotation-api-1.2.jar @@ -199,7 +199,7 @@ shapeless_2.11/2.3.2//shapeless_2.11-2.3.2.jar shims/0.7.45//shims-0.7.45.jar slf4j-api/1.7.16//slf4j-api-1.7.16.jar slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar -snakeyaml/1.15//snakeyaml-1.15.jar +snakeyaml/1.23//snakeyaml-1.23.jar snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar snappy/0.2//snappy-0.2.jar spire-macros_2.11/0.13.0//spire-macros_2.11-0.13.0.jar diff --git a/pom.xml b/pom.xml index eccd45aacb2ca..c59cc1f0f3ab7 100644 --- a/pom.xml +++ b/pom.xml @@ -158,9 +158,9 @@ 2.11.12 2.11 1.9.13 - 2.6.7 - 2.6.7.1 - 2.6.7.3 + 2.9.10 + 2.9.10 + 2.9.10.5 1.1.7.5 1.1.2 1.2.0-incubating diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 7ee079cf053a5..915f811df3a7f 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -553,7 +553,7 @@ object DependencyOverrides { lazy val settings = Seq( dependencyOverrides += "com.google.guava" % "guava" % "14.0.1", dependencyOverrides += "commons-io" % "commons-io" % "2.4", - dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.7.3", + dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.10.5", dependencyOverrides += "jline" % "jline" % "2.14.6") } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala index 5592aa6e3346b..86136602f7ee7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala @@ -72,12 +72,13 @@ object JSONBenchmarks { } /* - Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz + Java HotSpot(TM) 64-Bit Server VM 1.8.0_241-b07 on Mac OS X 10.15.5 + Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz - JSON schema inferring: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - -------------------------------------------------------------------------------------------- - No encoding 38902 / 39282 2.6 389.0 1.0X - UTF-8 is set 56959 / 57261 1.8 569.6 0.7X + JSON schema inferring: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + --------------------------------------------------------------------------------------------- + No encoding 7664 / 7686 13.0 76.6 1.0X + UTF-8 is set 9981 / 10180 10.0 99.8 0.8X */ benchmark.run() } @@ -113,12 +114,13 @@ object JSONBenchmarks { } /* - Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz + Java HotSpot(TM) 64-Bit Server VM 1.8.0_241-b07 on Mac OS X 10.15.5 + Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz - JSON per-line parsing: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - -------------------------------------------------------------------------------------------- - No encoding 25947 / 26188 3.9 259.5 1.0X - UTF-8 is set 46319 / 46417 2.2 463.2 0.6X + JSON per-line parsing: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + --------------------------------------------------------------------------------------------- + No encoding 5832 / 5915 17.1 58.3 1.0X + UTF-8 is set 8687 / 8884 11.5 86.9 0.7X */ benchmark.run() } @@ -161,12 +163,13 @@ object JSONBenchmarks { } /* - Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz + Java HotSpot(TM) 64-Bit Server VM 1.8.0_241-b07 on Mac OS X 10.15.5 + Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz - JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - -------------------------------------------------------------------------------------------- - No encoding 45543 / 45660 0.2 4554.3 1.0X - UTF-8 is set 65737 / 65957 0.2 6573.7 0.7X + JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + --------------------------------------------------------------------------------------------- + No encoding 9367 / 9475 1.1 936.7 1.0X + UTF-8 is set 14211 / 14397 0.7 1421.1 0.7X */ benchmark.run() } @@ -196,12 +199,14 @@ object JSONBenchmarks { } /* - Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz + Java HotSpot(TM) 64-Bit Server VM 1.8.0_241-b07 on Mac OS X 10.15.5 + Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz Count a dataset with 10 columns: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative --------------------------------------------------------------------------------------------- - Select 10 columns + count() 9961 / 10006 1.0 996.1 1.0X - Select 1 column + count() 8355 / 8470 1.2 835.5 1.2X + Select 10 columns + count() 2074 / 2112 4.8 207.4 1.0X + Select 1 column + count() 1753 / 1792 5.7 175.3 1.2X + */ benchmark.run() }