Upgrade jackson to 2.9.x

core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala

@@ -36,7 +36,7 @@ import org.apache.spark.util.Utils
  *   (2) the Spark version of the client / server
  *   (3) an optional message
  */
-@JsonInclude(Include.NON_NULL)
+@JsonInclude(Include.NON_ABSENT)
 @JsonAutoDetect(getterVisibility = Visibility.ANY, setterVisibility = Visibility.ANY)
 @JsonPropertyOrder(alphabetic = true)
 private[rest] abstract class SubmitRestProtocolMessage {

core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala

@@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging
  * There is no particular relationship between an operation scope and a stage or a job.
  * A scope may live inside one stage (e.g. map) or span across multiple jobs (e.g. take).
  */
-@JsonInclude(Include.NON_NULL)
+@JsonInclude(Include.NON_ABSENT)
 @JsonPropertyOrder(Array("id", "name", "parent"))
 private[spark] class RDDOperationScope(
     val name: String,

core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala

@@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{
   }
   mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
   mapper.enable(SerializationFeature.INDENT_OUTPUT)
-  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
+  mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
   mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)
 
   override def isWriteable(

dev/deps/spark-deps-hadoop-palantir

@@ -49,12 +49,12 @@ core-1.1.2.jar
 curator-client-2.7.1.jar
 curator-framework-2.7.1.jar
 curator-recipes-2.7.1.jar
-dropwizard-jackson-0.9.2.jar
-dropwizard-lifecycle-0.9.2.jar
-dropwizard-metrics-0.9.2.jar
-dropwizard-metrics-influxdb-1.1.8.jar
-dropwizard-util-0.9.2.jar
-dropwizard-validation-0.9.2.jar
+dropwizard-jackson-1.0.0.jar
+dropwizard-lifecycle-1.0.0.jar
+dropwizard-metrics-1.0.0.jar
+dropwizard-metrics-influxdb-1.2.2.jar
+dropwizard-util-1.0.0.jar
+dropwizard-validation-1.0.0.jar
 fastutil-7.0.13.jar
 flatbuffers-1.2.0-3f79e055.jar
 generex-1.0.1.jar
@@ -79,7 +79,7 @@ hadoop-yarn-client-2.8.2-palantir.2.jar
 hadoop-yarn-common-2.8.2-palantir.2.jar
 hadoop-yarn-server-common-2.8.2-palantir.2.jar
 hadoop-yarn-server-web-proxy-2.8.2-palantir.2.jar
-hibernate-validator-5.2.2.Final.jar
+hibernate-validator-5.2.4.Final.jar
 hk2-api-2.5.0-b32.jar
 hk2-locator-2.5.0-b32.jar
 hk2-utils-2.5.0-b32.jar
@@ -89,21 +89,22 @@ httpclient-4.5.4.jar
 httpcore-4.4.8.jar
 ion-java-1.0.1.jar
 ivy-2.4.0.jar
-jackson-annotations-2.6.7.jar
-jackson-core-2.6.7.jar
+jackson-annotations-2.9.5.jar
+jackson-core-2.9.5.jar
 jackson-core-asl-1.9.13.jar
-jackson-databind-2.6.7.1.jar
-jackson-dataformat-cbor-2.6.7.jar
-jackson-dataformat-yaml-2.6.7.jar
-jackson-datatype-guava-2.6.7.jar
-jackson-datatype-jdk7-2.6.7.jar
-jackson-datatype-joda-2.6.7.jar
+jackson-databind-2.9.5.jar
+jackson-dataformat-cbor-2.9.5.jar
+jackson-dataformat-yaml-2.9.5.jar
+jackson-datatype-guava-2.9.5.jar
+jackson-datatype-jdk8-2.9.5.jar
+jackson-datatype-joda-2.9.5.jar
+jackson-datatype-jsr310-2.9.5.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-afterburner-2.6.7.jar
-jackson-module-jaxb-annotations-2.6.7.jar
-jackson-module-paranamer-2.7.9.jar
-jackson-module-scala_2.11-2.6.7.1.jar
+jackson-module-afterburner-2.9.5.jar
+jackson-module-jaxb-annotations-2.9.5.jar
+jackson-module-paranamer-2.9.5.jar
+jackson-module-scala_2.11-2.9.5.jar
 jackson-xc-1.9.13.jar
 janino-3.0.8.jar
 java-xmlbuilder-1.1.jar
@@ -140,6 +141,7 @@ jsp-api-2.1.jar
 jsr305-3.0.1.jar
 jtransforms-2.4.0.jar
 jul-to-slf4j-1.7.25.jar
+kafka-clients-0.11.0.0.jar
 kryo-shaded-3.0.3.jar
 kubernetes-client-3.0.0.jar
 kubernetes-model-2.0.0.jar
@@ -151,7 +153,7 @@ machinist_2.11-0.6.1.jar
 macro-compat_2.11-1.1.1.jar
 metrics-core-3.2.5.jar
 metrics-graphite-3.2.5.jar
-metrics-influxdb-1.1.8.jar
+metrics-influxdb-1.2.2.jar
 metrics-json-3.2.5.jar
 metrics-jvm-3.2.5.jar
 minlog-1.3.0.jar
@@ -185,7 +187,7 @@ scala-xml_2.11-1.0.5.jar
 shapeless_2.11-2.3.2.jar
 slf4j-api-1.7.25.jar
 slf4j-log4j12-1.7.25.jar
-snakeyaml-1.15.jar
+snakeyaml-1.18.jar
 snappy-java-1.1.7.1.jar
 spire-macros_2.11-0.13.0.jar
 spire_2.11-0.13.0.jar

pom.xml

@@ -145,8 +145,8 @@
     <ivy.version>2.4.0</ivy.version>
     <oro.version>2.0.8</oro.version>
     <codahale.metrics.version>3.2.5</codahale.metrics.version>
-    <dropwizard.version>0.9.2</dropwizard.version>
-    <spark-influx-sink.version>0.3.12</spark-influx-sink.version>
+    <dropwizard.version>1.0.0</dropwizard.version>
+    <spark-influx-sink.version>0.4.0</spark-influx-sink.version>
     <avro.version>1.8.1</avro.version>
     <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
     <jets3t.version>0.9.4</jets3t.version>
@@ -168,8 +168,8 @@
     <scala.version>2.11.8</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
-    <fasterxml.jackson.version>2.6.7</fasterxml.jackson.version>
-    <fasterxml.jackson.databind.version>2.6.7.1</fasterxml.jackson.databind.version>
+    <fasterxml.jackson.version>2.9.5</fasterxml.jackson.version>
+    <fasterxml.jackson.databind.version>2.9.5</fasterxml.jackson.databind.version>
     <snappy.version>1.1.7.1</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
     <calcite.version>1.2.0-incubating</calcite.version>
@@ -786,6 +786,11 @@
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
           </exclusion>
+          <!-- Old lz4 dependency with different coordinates -->
+          <exclusion>
+            <groupId>net.jpountz.lz4</groupId>
+            <artifactId>lz4</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -848,7 +853,7 @@
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.datatype</groupId>
-        <artifactId>jackson-datatype-jdk7</artifactId>
+        <artifactId>jackson-datatype-jdk8</artifactId>
         <version>${fasterxml.jackson.version}</version>
       </dependency>
       <dependency>